I love to build dataviz for insight discovery. I also love to put technology to the service of humanity
Public
Edited
Mar 6
1 star
queryEmbedding = await extractor(query, {
pooling: "mean",
normalize: true
})
extractor = await transformers.pipeline(
"feature-extraction",
"Xenova/all-MiniLM-L6-v2",
// "Xenova/bert-base-uncased",
// "Xenova/gte-small",
// "allenai/specter2",
{ pooling: "mean", quantized: false }
)
history
mutable history = []
// positions = {
// history;
// druid_method;
// reducerParams;
// history
// const data = mutable history.map(d => Array.from(d.embedding))
// console.log("yield dataForUmap ", data);
// if (data.length) {
// yield* druidGenerator(data, druid_method, reducerParams);
// } else {
// yield [];
// }
// }
updateHistory = {
console.log("Do embedding");
if (query) {
const embeddingsObj = Object.fromEntries(
Array.from(queryEmbedding.data).map((d, i) => [i, d])
);
mutable history.push({ ...embeddingsObj, query, embedding: queryEmbedding.data });
}
}
resetResults = {
reset;
console.log("Reset history");
mutable history = [];
}
config = ({
nComponents,
minDist,
nNeighbors
})
// positions = {
// updateHistory;
// resetResults;
// return Generators.observe(
// worker(
// fit,
// { data: dataForUmap, show_dynamic: true, config },
// `
// const window = {};
// importScripts("https://unpkg.com/umap-js@1.3.3/lib/umap-js.js");
// importScripts("https://unpkg.com/d3-random@2");
// const UMAP = window.UMAP;
// Math.random = d3.randomLcg(${23});
// `
// )
// );
// }
positions = {
updateHistory;
resetResults;
const dataForUmap = mutable history.map((d) => Array.from(d.embedding));
yield* Generators.observe(
worker(
function* dr({ method, data, parameters }) {
console.log("Initializing druid...", data.length);
yield [];
yield* self.druid[method].generator(data, parameters); // show the steps
},
{
method: druid_method,
data: dataForUmap,
parameters: reducerParams
},
`
importScripts(${JSON.stringify(
await require.resolve("@saehrimnir/druidjs@^0.7.3")
)});
`
)
);
}
dataForUmap = mutable history.map(d => Array.from(d.embedding))
default_method = "PCA"
import {
druidGenerator,
chart as chartDR,
viewof druid_method,
} with { dataForUmap as dataAsMatrix , default_method} from "@john-guerra/druidjs-playground"
transformers = import("https://cdn.jsdelivr.net/npm/@xenova/transformers@2.6.0")
import {chart} from "@john-guerra/umap-playground"
import {
worker,
fit
} with { dataForUmap as data } from "@fil/umap-js-worker"
import { navio } from "@john-guerra/navio"
create_parameter_form = (dr_method) => {
switch (dr_method) {
case "ISOMAP":
case "LLE":
case "LTSA":
return html`<form>
<label><input name="neighbors" type="range" min=10 max=300 value=30 /> neighbors <i>k</i></label><br />
<label><input name="seed" type="number" value=1212 /> seed <i>seed</i></label>
</form>`;
break;
case "TSNE":
return html`<form>
<label><input name="perplexity" type="range" min=2 max=100 value=50 /> <i>perplexity</i></label><br>
<label><input name="epsilon" type="range" min=1 max=100 value=5 /> <i>epsilon</i></label><br />
<label><input name="seed" type="number" value=1212 /> seed <i>seed</i></label>
</form>`;
break;
case "UMAP":
return html`<form>
<label><input name="n_neighbors" type="range" min=1 max=50 value=2 /> number of neighbors <i>n_neighbors</i></label><br>
<label><input name="local_connectivity" type="range" min=1 max=10 value=1 /> local connectivity <i>local_conn</i></label><br>
<label><input name="min_dist" type="range" min=0.05 max=1.5 step=.05 value=1 /> minimum distance <i>min_dist</i></label><br />
<label><input name="seed" type="number" value=1212 /> seed <i>seed</i></label>
</form>`;
break;
case "TriMap":
return html`<form>
<label><input name="weight_adj" type="range" min=100 max=100000 value=100 /> scaling factor <i>weight_adj</i></label><br>
<label><input name="c" type="range" min=1 max=10 value=5 /> number of triplets multiplier <i>c</i></label><br />
<label><input name="seed" type="number" value=1212 /> seed <i>seed</i></label>
</form>`;
break;
case "LSP":
return html`<form>
<label><input name="neighbors" type="range" min=1 max=300 value=30 /> number of neighbors to consider <i>k</i></label><br>
<label><input name="control_points" type="range" min=10 max=100 value=20 /> number of controlpoints <i>c</i></label><br />
<label><input name="seed" type="number" value=1212 /> seed <i>seed</i></label>
</form>`;
case "SQDMDS":
return html`<form>
<label><input name="decay_start" type="range" min=0 max=1 step=.01 value="0.1" /> Percentage of iterations using exaggeration phase</label><br>
<label><input name="decay_cte" type="range" min=0 max=1 step=.01 value="0.31" /> Controls the decay of the learning parameter</label><br />
<label><input name="seed" type="number" value=1212 /> seed <i>seed</i></label>
</form>`;
break;
case "LDA":
return html`<form>
<label>
<select name="clusters">
<option value="islands">islands</option>
<option value="species">species</option>
</select>
<i>class labels</i></label><br />
<label><input name="seed" type="number" value=1212 /> seed <i>seed</i></label>
</form>`;
break;
default:
return html`<form>
<label><input name="seed" type="number" value=1212 /> seed <i>seed</i></label>
</form>`;
break;
}
}
import { form } from "@mbostock/form-input"
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.