I love to build dataviz for insight discovery. I also love to put technology to the service of humanity
Public
Edited
Oct 23, 2023
Paused
3 forks
Importers
28 stars
navio(embeddings.map(d => d.data), {attribWidth: 2, height: 600})
computeEmbeddings && DOM.download(new Blob(
[JSON.stringify(embeddings_computed)],
{type: "application/json"}
), "embeddings.json", "Download Embeddings")
viewof embeddings_computed = {
if (computeEmbeddings) {
const before = performance.now();
restart;
const values = [];
const bar = progress({ interval: 50, invalidation });
yield bar;
let i =0;
for (let row of sample) {
values.push(
await extractor(embeddingAccessor(row), {
pooling: "mean",
normalize: true
})
);
await bar.progress(i++, sample.length);
// console.log("i", i);
}
console.log("progress completed in", performance.now() - before);
bar.resolve(values);
} else {
// Not computing embeddgins, return nothing
yield pleaseEnableComputeEmbeddings();
}
}
function pleaseEnableComputeEmbeddings() {
return Object.assign(htl.html`Please enable compute embeddings`, {
value: []
});
}
sample = selected
embeddings = {
if (computeEmbeddings) {
return embeddings_computed;
} else {
return embeddings_cached;
}
}
embeddings_cached = FileAttachment("embeddings_v2.json").json().then(res => {
for (let row of res)
row.data = new Float32Array(Array.from(Object.values(row.data)));
return res;
})
embeddingAccessor = (d) =>
// [d.paper_type].join("\n\n")
computeEmbeddingsUsing.map(a => d[a]).join("\n\n")
papersHighlighted = drSelection.drag
? dataToPlot.filter(
(d) =>
d.x >= drSelection.drag.x[0] &&
d.x <= drSelection.drag.x[1] &&
d.y >= drSelection.drag.y[0] &&
d.y <= drSelection.drag.y[1]
)
: dataToPlot
vegaScatterPlot = {
let chart = vl
.markCircle({ size: 100 })
.encode(
vl.x().fieldQ("x").axis(null),
vl.y().fieldQ("y").axis(null),
vl.color().fieldN(colorBy)
)
.width(Math.max(320, width - 150))
.height(500)
.data(dataToPlot);
// console.log("vega spec interactive =", interactive);
if (interactive) {
// console.log("vega interactive!", interactive);
const hover = vl
.selectPoint("hover")
.nearest(true)
.on("mouseover,pointerover,click,pointerup")
.init({ x: [], y: [] });
const drag = vl
.selectInterval("drag")
.init({ x: [], y: [] });
chart = chart
.params(hover, drag)
.encode(
vl.stroke().if(hover, vl.value("black")).value(null),
vl.size().if(vl.or(hover, drag), vl.value(100)).value(50),
vl.tooltip([
"title",
"paper_type_name",
"event_title",
"session_title",
"abstract"
])
);
}
return chart;
}
reducerResult = Generators.observe(
worker(
function* dr({ method, data, parameters }) {
yield* self.druid[method].generator(data, {}); // show the steps
},
{
method: druid_method,
data: embeddings.map((row) => row.data),
parameters: reducerParams
},
`
importScripts(${JSON.stringify(
await require.resolve("@saehrimnir/druidjs@^0.7.3")
)});
`
)
)
dataToPlot = sample.map((row, id) => ({
...row,
x: reducerResult[id][0],
y: reducerResult[id][1]
}))
create_parameter_form = (dr_method) => {
switch (dr_method) {
case "ISOMAP":
case "LLE":
case "LTSA":
return html`<form>
<label><input name="neighbors" type="range" min=10 max=300 value=10 /> neighbors <i>neighbors</i></label>
</form>`;
break;
case "SAMMON":
return html`<form>
<label><input name="magic" type="range" min=0 max=1 value=0.1 step=0.1 /> magic <i>magic</i></label>
</form>`;
break;
case "TSNE":
return html`<form>
<label><input name="perplexity" type="range" min=2 max=100 value=50 /> <i>perplexity</i></label><br>
<label><input name="epsilon" type="range" min=1 max=100 value=5 /> <i>epsilon</i></label>
</form>`;
break;
case "UMAP":
return html`<form>
<label><input name="n_neighbors" type="range" min=2 max=100 value=20 /> number of neighbors <i>n_neighbors</i></label><br>
<label><input name="min_dist" type="range" min=0.05 max=1.5 step=.05 value=0.1 /> minimum distance <i>min_dist</i></label>
</form>`;
break;
case "TriMap":
return html`<form>
<label><input name="weight_adj" type="range" min=100 max=100000 value=100 /> scaling factor <i>weight_adj</i></label><br>
<label><input name="c" type="range" min=1 max=10 value=5 /> number of triplets multiplier <i>c</i></label>
</form>`;
break;
case "LSP":
return html`<form>
<label><input name="neighbors" type="range" min=10 max=300 value=30 /> number of neighbors to consider <i>k</i></label><br>
<label><input name="control_points" type="range" min=10 max=100 value=20 /> number of controlpoints <i>control_points</i></label>
</form>`;
break;
case "LDA":
return html`<form>
<label>
<select name="clusters">
<option value="islands">islands</option>
<option value="species">species</option>
</select>
<i>class labels</i>
</form>`;
break;
default:
return html`<form>
${dr_method} has no parameters!
</form>`;
break;
}
}
druid = "🤯" // placeholder for Observable to allow the *dr* function to exist; the worker uses importScripts
import {conditionalShow} from "@john-guerra/conditional-show"
import {vegaSelected} from "@john-guerra/vega-selected"
import { worker } from "@fil/worker"
import { form } from "@mbostock/form-input"
import {FacetedSearch} from "@john-guerra/faceted-search"
import {progress} from "@mootari/displaying-progress"
import {navio} from "@john-guerra/navio"
import {vl} from "@john-guerra/vega-lite-api-v5"
transformers = import("https://cdn.jsdelivr.net/npm/@xenova/transformers@2.6.0")
extractor = await transformers.pipeline(
"feature-extraction",
"Xenova/all-MiniLM-L6-v2",
// "allenai/specter2",
{ quantized: false }
)
dicPapers = FileAttachment("paper_list_ieeevis_2023_from_github@1.json")
.json()
// .then((res) =>
// Array.from(Object.entries(res)).map(([UID, d]) => {
// d.UID = UID;
// d.time_stamp_parsed = new Date(d.time_stamp);
// d.thumb = `https://ieeevis.b-cdn.net/vis_2023/paper_images_small/${d.UID}.png`;
// return d;
// })
// )
papers = FileAttachment("papers_vis2023@2.json")
.json()
.then((res) =>
res.map((d) => {
d.time_stamp_parsed = new Date(d.time_stamp);
d.thumb = `https://ieeevis.b-cdn.net/vis_2023/paper_images_small/${d.UID}.png`;
d.url = `https://content.ieeevis.org/year/2023/paper_${d.UID}.html`
// Override any attributes with the original values from github
// This is for getting the actual abstract
return Object.assign(d, dicPapers[d.UID]);
})
)
// embeddings2 = {
// const before = performance.now();
// const res = await Promise.all(
// sample.map((row) =>
// extractor(embeddingAccessor(row), {
// pooling: "mean",
// normalize: true
// })
// )
// );
// console.log("completed in", performance.now() - before);
// return res;
// }
// Generators.observe((notify) => {
// const before = performance.now();
// const res = [];
// notify(res);
// sample.map((row, i) => {
// extractor(embeddingAccessor(row), {
// pooling: "mean",
// normalize: true
// }).then((rowRes) => {
// console.log("got result ", i)
// res.push(rowRes);
// if (res.length> sample.length-1)
// console.log("Generator duration", performance.now()-before);
// notify(res);
// });
// });
// })
// {
// const plt = Plot.plot({
// marginTop: 50,
// marginBottom: 50,
// inset: 50,
// width: width,
// height: width,
// marks: [
// Plot.image(dataToPlot, {
// x: "x",
// y: "y",
// src: (x) => x.thumb1,
// width: 50,
// title: "x"
// }),
// Plot.tip(
// dataToPlot,
// Plot.pointer({
// stroke: "paper_type",
// // channels: { Note: "Note", sport: "sport" },
// x: "x",
// y: "y",
// title: embeddingAccessor
// })
// )
// ]
// });
// const selection = d3.select(plt);
// selection.on("mousemove", (evt) => {
// const tipdata = plt.value;
// if (tipdata) {
// const image = selection.select(`image[href='${tipdata.thumb1}']`);
// image.raise();
// }
// });
// return plt;
// }
// // Plot viz
// projection = {
// const projection = await Plot.plot({
// marginTop: 10,
// inset: 10,
// color: {
// // scheme: "Paired",
// legend: true,
// columns: 3
// },
// legend: {
// marginTop: 50
// },
// width: width,
// height: 400,
// x: {axis: null},
// y: {axis: null},
// marks: [
// Plot.dot(dataToPlot, {
// x: "x",
// y: "y",
// fill: colorBy,
// r: 5,
// opacity: 0.8
// // strokeWidth: 4,
// // stroke: "Category"
// }),
// Plot.tip(
// dataToPlot,
// Plot.pointer({
// stroke: "paper_type",
// // channels: { Note: "Note", sport: "sport" },
// x: "x",
// y: "y",
// title: embeddingAccessor
// })
// )
// ]
// });
// Array.from(projection.querySelectorAll("svg")).map(ele => ele.style.overflow="visible");
// console.log( projection.querySelectorAll("svg"));
// return projection;
// }
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
