I love to build dataviz for insight discovery. I also love to put technology to the service of humanity
Public
Edited
Oct 23, 2023
Paused
3 forks
Importers
28 stars
navio(embeddings.map(d => d.data), {attribWidth: 2, height: 600})
computeEmbeddings && DOM.download(new Blob(
[JSON.stringify(embeddings_computed)],
{type: "application/json"}
), "embeddings.json", "Download Embeddings")
viewof embeddings_computed = {
if (computeEmbeddings) {
const before = performance.now();
restart;
const values = [];
const bar = progress({ interval: 50, invalidation });
yield bar;
let i =0;
for (let row of sample) {
values.push(
await extractor(embeddingAccessor(row), {
pooling: "mean",
normalize: true
})
);
await bar.progress(i++, sample.length);
// console.log("i", i);
}
console.log("progress completed in", performance.now() - before);
bar.resolve(values);
} else {
// Not computing embeddgins, return nothing
yield pleaseEnableComputeEmbeddings();
}
}
function pleaseEnableComputeEmbeddings() {
return Object.assign(htl.html`Please enable compute embeddings`, {
value: []
});
}
sample = selected
embeddings = {
if (computeEmbeddings) {
return embeddings_computed;
} else {
return embeddings_cached;
}
}
embeddings_cached = FileAttachment("embeddings_v2.json").json().then(res => {
for (let row of res)
row.data = new Float32Array(Array.from(Object.values(row.data)));
return res;
})
embeddingAccessor = (d) =>
// [d.paper_type].join("\n\n")
computeEmbeddingsUsing.map(a => d[a]).join("\n\n")
papersHighlighted = drSelection.drag
? dataToPlot.filter(
(d) =>
d.x >= drSelection.drag.x[0] &&
d.x <= drSelection.drag.x[1] &&
d.y >= drSelection.drag.y[0] &&
d.y <= drSelection.drag.y[1]
)
: dataToPlot
vegaScatterPlot = {
let chart = vl
.markCircle({ size: 100 })
.encode(
vl.x().fieldQ("x").axis(null),
vl.y().fieldQ("y").axis(null),
vl.color().fieldN(colorBy)
)
.width(Math.max(320, width - 150))
.height(500)
.data(dataToPlot);
// console.log("vega spec interactive =", interactive);
if (interactive) {
// console.log("vega interactive!", interactive);
const hover = vl
.selectPoint("hover")
.nearest(true)
.on("mouseover,pointerover,click,pointerup")
.init({ x: [], y: [] });
const drag = vl
.selectInterval("drag")
.init({ x: [], y: [] });
chart = chart
.params(hover, drag)
.encode(
vl.stroke().if(hover, vl.value("black")).value(null),
vl.size().if(vl.or(hover, drag), vl.value(100)).value(50),
vl.tooltip([
"title",
"paper_type_name",
"event_title",
"session_title",
"abstract"
])
);
}
return chart;
}
reducerResult = Generators.observe(
worker(
function* dr({ method, data, parameters }) {
yield* self.druid[method].generator(data, {}); // show the steps
},
{
method: druid_method,
data: embeddings.map((row) => row.data),
parameters: reducerParams
},
`
importScripts(${JSON.stringify(
await require.resolve("@saehrimnir/druidjs@^0.7.3")
)});
`
)
)
dataToPlot = sample.map((row, id) => ({
...row,
x: reducerResult[id][0],
y: reducerResult[id][1]
}))
create_parameter_form = (dr_method) => {
switch (dr_method) {
case "ISOMAP":
case "LLE":
case "LTSA":
return html`<form>
<label><input name="neighbors" type="range" min=10 max=300 value=10 /> neighbors <i>neighbors</i></label>
</form>`;
break;
case "SAMMON":
return html`<form>
<label><input name="magic" type="range" min=0 max=1 value=0.1 step=0.1 /> magic <i>magic</i></label>
</form>`;
break;
case "TSNE":
return html`<form>
<label><input name="perplexity" type="range" min=2 max=100 value=50 /> <i>perplexity</i></label><br>
<label><input name="epsilon" type="range" min=1 max=100 value=5 /> <i>epsilon</i></label>
</form>`;
break;
case "UMAP":
return html`<form>
<label><input name="n_neighbors" type="range" min=2 max=100 value=20 /> number of neighbors <i>n_neighbors</i></label><br>
<label><input name="min_dist" type="range" min=0.05 max=1.5 step=.05 value=0.1 /> minimum distance <i>min_dist</i></label>
</form>`;
break;
case "TriMap":
return html`<form>
<label><input name="weight_adj" type="range" min=100 max=100000 value=100 /> scaling factor <i>weight_adj</i></label><br>
<label><input name="c" type="range" min=1 max=10 value=5 /> number of triplets multiplier <i>c</i></label>
</form>`;
break;
case "LSP":
return html`<form>
<label><input name="neighbors" type="range" min=10 max=300 value=30 /> number of neighbors to consider <i>k</i></label><br>
<label><input name="control_points" type="range" min=10 max=100 value=20 /> number of controlpoints <i>control_points</i></label>
</form>`;
break;
case "LDA":
return html`<form>
<label>
<select name="clusters">
<option value="islands">islands</option>
<option value="species">species</option>
</select>
<i>class labels</i>
</form>`;
break;
default:
return html`<form>
${dr_method} has no parameters!
</form>`;
break;
}
}
druid = "🤯" // placeholder for Observable to allow the *dr* function to exist; the worker uses importScripts
import {conditionalShow} from "@john-guerra/conditional-show"
import {vegaSelected} from "@john-guerra/vega-selected"
import { worker } from "@fil/worker"
import { form } from "@mbostock/form-input"
import {FacetedSearch} from "@john-guerra/faceted-search"
import {progress} from "@mootari/displaying-progress"
import {navio} from "@john-guerra/navio"
import {vl} from "@john-guerra/vega-lite-api-v5"
transformers = import("https://cdn.jsdelivr.net/npm/@xenova/transformers@2.6.0")
extractor = await transformers.pipeline(
"feature-extraction",
"Xenova/all-MiniLM-L6-v2",
// "allenai/specter2",
{ quantized: false }
)
dicPapers = FileAttachment("paper_list_ieeevis_2023_from_github@1.json")
.json()
// .then((res) =>
// Array.from(Object.entries(res)).map(([UID, d]) => {
// d.UID = UID;
// d.time_stamp_parsed = new Date(d.time_stamp);
// d.thumb = `https://ieeevis.b-cdn.net/vis_2023/paper_images_small/${d.UID}.png`;
// return d;
// })
// )
papers = FileAttachment("papers_vis2023@2.json")
.json()
.then((res) =>
res.map((d) => {
d.time_stamp_parsed = new Date(d.time_stamp);
d.thumb = `https://ieeevis.b-cdn.net/vis_2023/paper_images_small/${d.UID}.png`;
d.url = `https://content.ieeevis.org/year/2023/paper_${d.UID}.html`
// Override any attributes with the original values from github
// This is for getting the actual abstract
return Object.assign(d, dicPapers[d.UID]);
})
)
// embeddings2 = {
// const before = performance.now();
// const res = await Promise.all(
// sample.map((row) =>
// extractor(embeddingAccessor(row), {
// pooling: "mean",
// normalize: true
// })
// )
// );
// console.log("completed in", performance.now() - before);
// return res;
// }
// Generators.observe((notify) => {
// const before = performance.now();
// const res = [];
// notify(res);
// sample.map((row, i) => {
// extractor(embeddingAccessor(row), {
// pooling: "mean",
// normalize: true
// }).then((rowRes) => {
// console.log("got result ", i)
// res.push(rowRes);
// if (res.length> sample.length-1)
// console.log("Generator duration", performance.now()-before);
// notify(res);
// });
// });
// })
// {
// const plt = Plot.plot({
// marginTop: 50,
// marginBottom: 50,
// inset: 50,
// width: width,
// height: width,
// marks: [
// Plot.image(dataToPlot, {
// x: "x",
// y: "y",
// src: (x) => x.thumb1,
// width: 50,
// title: "x"
// }),
// Plot.tip(
// dataToPlot,
// Plot.pointer({
// stroke: "paper_type",
// // channels: { Note: "Note", sport: "sport" },
// x: "x",
// y: "y",
// title: embeddingAccessor
// })
// )
// ]
// });
// const selection = d3.select(plt);
// selection.on("mousemove", (evt) => {
// const tipdata = plt.value;
// if (tipdata) {
// const image = selection.select(`image[href='${tipdata.thumb1}']`);
// image.raise();
// }
// });
// return plt;
// }
// // Plot viz
// projection = {
// const projection = await Plot.plot({
// marginTop: 10,
// inset: 10,
// color: {
// // scheme: "Paired",
// legend: true,
// columns: 3
// },
// legend: {
// marginTop: 50
// },
// width: width,
// height: 400,
// x: {axis: null},
// y: {axis: null},
// marks: [
// Plot.dot(dataToPlot, {
// x: "x",
// y: "y",
// fill: colorBy,
// r: 5,
// opacity: 0.8
// // strokeWidth: 4,
// // stroke: "Category"
// }),
// Plot.tip(
// dataToPlot,
// Plot.pointer({
// stroke: "paper_type",
// // channels: { Note: "Note", sport: "sport" },
// x: "x",
// y: "y",
// title: embeddingAccessor
// })
// )
// ]
// });
// Array.from(projection.querySelectorAll("svg")).map(ele => ele.style.overflow="visible");
// console.log( projection.querySelectorAll("svg"));
// return projection;
// }
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.