31:Tc58,create_parameter_form = (dr_method) => { switch (dr_method) { case "ISOMAP": case "LLE": case "LTSA": return html``; break; case "TSNE": return html``; break; case "UMAP": return html``; break; case "TriMap": return html``; break; case "LSP": return html``; case "SQDMDS": return html``; break; case "LDA": return html``; break; default: return html``; break; } }32:[] 2f:{"title":"Embeddings playground (WIP)","cells":[{"id":0,"value":"# Embeddings playground (WIP)\n\nA simple example to play with sentence embeddings and dimensionality reduction","pinned":false,"hidden":false,"mode":"md"},{"id":7,"value":"viewof query = Inputs.text({label: \"Semantic search using embeddings\", value: \"\", submit: \"Submit\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":47,"value":"{\n // Execute after these two\n updateHistory;\n resetResults;\n\n console.log(\"Do navio\");\n \n return mutable history.length && navio(mutable history, {\n showSelectedAttrib: false,\n showSequenceIDAttrib: false,\n attribWidth: 2,\n addAllAttribsRecursionLevel: 0,\n height: 200,\n y0: 10\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":67,"value":"viewof reset = Inputs.button(\"reset\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":385,"value":"## Dimensionality Reduction","pinned":false,"hidden":false,"mode":"md"},{"id":318,"value":"vl\n .markText({size: 18, fontFamily: \"Sans Serif\"})\n .encode(\n vl.text().fieldN(\"query\"),\n vl.x().fieldQ(\"x\").scale({zero: null}).axis(null),\n vl.y().fieldQ(\"y\").scale({zero: null}).axis(null)\n )\n .data(\n mutable history.map((d, i) => ({\n query: d.query,\n x: positions[i][0],\n y: positions[i][1]\n }))\n )\n .width(400)\n .height(400)\n .render()","pinned":false,"hidden":false,"mode":"ojs"},{"id":447,"value":"viewof druid_method","pinned":false,"hidden":false,"mode":"ojs"},{"id":497,"value":"viewof reducerParams = form(create_parameter_form(druid_method))\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":277,"value":"### Query History","pinned":false,"hidden":false,"mode":"md"},{"id":8,"value":"queryEmbedding = await extractor(query, {\n pooling: \"mean\",\n normalize: true\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":10,"value":"extractor = await transformers.pipeline(\n \"feature-extraction\",\n \"Xenova/all-MiniLM-L6-v2\",\n // \"Xenova/bert-base-uncased\",\n // \"Xenova/gte-small\",\n // \"allenai/specter2\",\n { pooling: \"mean\", quantized: false }\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":551,"value":"history","pinned":true,"hidden":false,"mode":"ojs"},{"id":42,"value":"mutable history = []","pinned":true,"hidden":false,"mode":"ojs"},{"id":453,"value":"// positions = {\n// history;\n// druid_method;\n// reducerParams;\n// history\n// const data = mutable history.map(d => Array.from(d.embedding))\n// console.log(\"yield dataForUmap \", data);\n// if (data.length) {\n// yield* druidGenerator(data, druid_method, reducerParams);\n// } else {\n// yield [];\n// }\n// }","pinned":true,"hidden":false,"mode":"ojs"},{"id":43,"value":"updateHistory = {\n console.log(\"Do embedding\");\n if (query) {\n const embeddingsObj = Object.fromEntries(\n Array.from(queryEmbedding.data).map((d, i) => [i, d])\n );\n mutable history.push({ ...embeddingsObj, query, embedding: queryEmbedding.data });\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":78,"value":"resetResults = {\n reset;\n console.log(\"Reset history\");\n mutable history = [];\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":312,"value":"## UMAP","pinned":false,"hidden":false,"mode":"md"},{"id":338,"value":"viewof nNeighbors = Inputs.range([0, 10], {value: 1, step: 0.01, label: \"UMAP neighborhoods\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":342,"value":"viewof nComponents = Inputs.range([0, 10], {value: 2, step: 1, label: \"UMAP nComponents\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":346,"value":"viewof minDist = Inputs.range([0, 20], {value: 0.1, label: \"UMAP minDist\", step: 0.01})","pinned":false,"hidden":false,"mode":"ojs"},{"id":310,"value":"config = ({\n nComponents,\n minDist,\n nNeighbors\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":299,"value":"// positions = {\n// updateHistory;\n// resetResults;\n \n// return Generators.observe(\n// worker(\n// fit,\n// { data: dataForUmap, show_dynamic: true, config },\n// `\n// const window = {};\n// importScripts(\"https://unpkg.com/umap-js@1.3.3/lib/umap-js.js\");\n// importScripts(\"https://unpkg.com/d3-random@2\");\n// const UMAP = window.UMAP;\n// Math.random = d3.randomLcg(${23});\n// `\n// )\n// );\n// }","pinned":true,"hidden":false,"mode":"ojs"},{"id":538,"value":"positions = {\n updateHistory;\n resetResults;\n\n const dataForUmap = mutable history.map((d) => Array.from(d.embedding));\n\n yield* Generators.observe(\n worker(\n function* dr({ method, data, parameters }) {\n console.log(\"Initializing druid...\", data.length);\n yield [];\n yield* self.druid[method].generator(data, parameters); // show the steps\n },\n {\n method: druid_method,\n data: dataForUmap,\n parameters: reducerParams\n },\n\n `\n importScripts(${JSON.stringify(\n await require.resolve(\"@saehrimnir/druidjs@^0.7.3\")\n )});\n `\n )\n );\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":463,"value":"dataForUmap = mutable history.map(d => Array.from(d.embedding))","pinned":true,"hidden":false,"mode":"ojs"},{"id":587,"value":"default_method = \"PCA\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":425,"value":"import {\n druidGenerator,\n chart as chartDR,\n viewof druid_method,\n \n} with { dataForUmap as dataAsMatrix , default_method} from \"@john-guerra/druidjs-playground\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":19,"value":"{\n // Execute after these two\n updateHistory;\n query;\n \n return Inputs.table(mutable history);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"transformers = import(\"https://cdn.jsdelivr.net/npm/@xenova/transformers@2.6.0\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":263,"value":"import {chart} from \"@john-guerra/umap-playground\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":264,"value":"import {\n worker,\n fit\n} with { dataForUmap as data } from \"@fil/umap-js-worker\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":17,"value":"import { navio } from \"@john-guerra/navio\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":496,"value":"$31","pinned":true,"hidden":false,"mode":"ojs"},{"id":500,"value":"import { form } from \"@mbostock/form-input\"","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}