I love to build dataviz for insight discovery. I also love to put technology to the service of humanity
viewof includeKeywords = Inputs.toggle({label: "Include Keywords"})
paperClicked = drSelection.clicked
papersHighlighted = drSelection.brushed
sessionsToAttend = {
query;
semanticSearchResults;
return d3
.groups(
navioSelected.filter((d) => d.score > minScore/100),
// (v) => v.length,
(d) => d.firstTimestamp.toLocaleDateString(...dateOptions),
(d) =>
`${d.firstTimestamp.toLocaleTimeString(...dateOptions)} ${d.firstRoomName} - ${d.firstSessionName}`
)
.sort((a, b) => d3.ascending(a[0], b[0]))
.map(([g, v]) => [g, v.sort((a, b) => d3.ascending(a[0], b[0]))]);
}
dateOptions = ["en-US", { timeZone: "utc" }]
roomsToGo = {
query;
semanticSearchResults;
return d3
.groups(
navioSelected.filter((d) => d.score > minScore / 100),
// (v) => v.length,
(d) => d.firstTimestamp.toLocaleDateString(...dateOptions),
(d) => `${d.firstRoomName}`
// (d) => d.title
)
.sort((a, b) => d3.ascending(a[0], b[0]))
.map(([g, v]) => [g, v.sort((a, b) => b[1].length - a[1].length)]);
}
viewof maxPapersSpinner = Inputs.bind(htl.html`<input type="number" style="width: 3rem" min=1 value=10>`, viewof maxPapers)
// navio(embeddings.map(d => d.data), {attribWidth: 2, height: 600})
computeEmbeddings && DOM.download(new Blob(
[JSON.stringify(embeddings_computed)],
{type: "application/json"}
), "embeddings.json", "Download Embeddings")
viewof embeddings_computed = {
if (computeEmbeddings) {
const before = performance.now();
restart;
const values = [];
const bar = progress({ interval: 50, invalidation });
yield bar;
let i =0;
for (let row of sample) {
values.push({id: row.id,
embedding: await extractor(embeddingAccessor(row), extractorOptions)
});
await bar.progress(i++, sample.length);
// console.log("i", i);
}
console.log("progress completed in", performance.now() - before);
bar.resolve(values);
} else {
// Not computing embeddgins, return nothing
yield pleaseEnableComputeEmbeddings();
}
}
embeddings = {
if (computeEmbeddings) {
return embeddings_computed;
} else {
return embeddings_cached;
}
}
embeddings_cached = (
// embeddingsFileSelected === "Title and Abstract May 13" ?
(await FileAttachment("CHI_2025_embeddings.json.zip").zip()) .file("CHI_2025_embeddings.json")
// :
// (await FileAttachment("chi2024papers-all-componentized_embeddings.json.zip").zip()).file("chi2024papers-all-componentized_embeddings.json")
)
.json()
.then((res) => {
for (let row of res) {
row.embedding.data = new Float32Array(
Array.from(Object.values(row.embedding.data))
);
// newRes.push({
// data: new Float32Array(Array.from(Object.values(row.data)))
// });
}
return res;
})
viewof embeddingsFileSelected = Inputs.select(
[
"Title and Abstract May 13",
"Joel's componentized_embeddings",
],
{ label: "Procomputed embeddings file"}
)
embeddingAccessor = (d) =>
// [d.paper_type].join("\n\n")
computeEmbeddingsUsing.map(a => d[a]).join("\n\n")
embeddingsHash = {
console.log("computing embHash");
return new Map(embeddings.map((d, i) => [d.id, d.embedding]))
}
function renderItem(p) {
return htl.html`
<div style="width: 150px; flex: 1; padding-right: 10px; padding-bottom: 15px">
<strong><a href=${p.url}>${p.title}</a></strong>
<div>Similarity score: ${(p.score * 100).toFixed(2)}% ${
p.awards ? p.awards : ""
}</div>
<div style="font-style: italic; max-height: 4em; overflow: auto;">${p.authorsExpanded
.map((a) => `${a.firstName} ${a.lastName}`)
.join(", ")}</div>
<div>${p.track} - ${p.firstSessionName}</div>
<div style="margin-top: 0.5em; max-height: 70px; overflow: auto">${
p.abstract
}</div>
</div>
`;
}
facetedSelected = facetedSelectedPhase1.filter(
(d) =>
tracksSelected.includes(d.track) &&
(!authorsSelected.length ||
authorsSelected.some((a) => d.authorsList.includes(a))) &&
(!affiliationsSelected.length ||
affiliationsSelected.some((a) => d.affiliationsList.includes(a.toLocaleLowerCase())))
)
function pleaseEnableComputeEmbeddings() {
return Object.assign(htl.html`Please enable compute embeddings`, {
value: []
});
}
attrs = Object.keys(papers[0]).concat(["score"])
reducerResultComputed = {
embeddings;
console.log(
"Launching DR Worker",
[...embeddingsHash.keys()].length,
sample.length,
sample.at(-1),
tracksSelected
);
if (useCachedDR) return [];
restartDR;
return DruidGenerator(
sample.map((d) => embeddingsHash.get(d.id).data),
druid_method,
druid_params
);
}
reducerResult = {
console.log("got reducerResult", sample.length)
return useCachedDR
? reducerResultCached
: Object.fromEntries(
sample.map((d, id) => [d.id, reducerResultComputed[id]])
)
}
// DOWNLOAD UMAP
sample.map((d, i) => {
const [x, y] = reducerResultComputed[i];
return ({id: d.id, x, y });
})
reducerResultCached = FileAttachment("CHI_2025_umap.csv")
.csv({ typed: true })
.then((res) => Object.fromEntries(res.map((d) => [d.id, [d.x, d.y]])))
dataToPlot = {
// Recompute data to plot when clustering;
computeClusters;
maxClusters;
return (semanticSearchResults ? semanticSearchResults : navioSelected).map(
(row, id) => ({
...row,
x: reducerResult[row.id] && reducerResult[row.id][0],
y: reducerResult[row.id] && reducerResult[row.id][1]
})
);
}
sample = navioSelected
semanticSearchResults = cosineSimilarity(query, sample)
// https://github.com/xenova/transformers.js/blob/main/examples/semantic-image-search-client/src/app/worker.js#L46
async function cosineSimilarity(query, data) {
if (!query) {
for (let row of data) {
row.score = 0;
}
return;
}
const query_embeds = (await extractor(query, extractorOptions)).data;
// const database_embeds = sample.map((d) => embeddingsHash.get(d.id).data);
for (let row of data) {
const dbVector = embeddingsHash.get(row.id)?.data;
if (dbVector) {
row.score = pairCosineSimilarity(query_embeds, dbVector);
} else {
console.log("no embedding found for", row.id);
}
}
return data;
}
// Computes the cosine similarity between two embeddings
function pairCosineSimilarity(embeddingA, embeddingB) {
const EMBED_DIM = embeddingA.length;
let dotProduct = 0;
let normEmbeds = 0;
let normDB = 0;
for (let j = 0; j < EMBED_DIM; ++j) {
const embedValue = embeddingA[j];
const dbValue = embeddingB[j];
dotProduct += embedValue * dbValue;
normEmbeds += embedValue * embedValue;
normDB += dbValue * dbValue;
}
const score = dotProduct / (Math.sqrt(normEmbeds) * Math.sqrt(normDB));
return score;
}
import {
DruidGenerator,
viewof druid_method,
viewof druid_params
} with { default_method } from "@john-guerra/druidjs-generator"
default_method = "UMAP"
import {conditionalShow} from "@john-guerra/conditional-show"
import {FacetedSearch} from "@john-guerra/faceted-search"
import {progress} from "@mootari/displaying-progress"
import {navio} from "@john-guerra/navio"
import {scentedCheckbox} from "@john-guerra/scented-checkbox"
import { BrushableScatterPlot } with { getVegaView } from "@john-guerra/brushable-scatterplot"
transformers = import("https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.1")
extractor = await transformers.pipeline(
"feature-extraction",
"Xenova/all-MiniLM-L6-v2",
// "allenai/specter2",
// { quantize: false }
extractorOptions
)
extractorOptions = ({
pooling: "mean",
quantize: false,
// normalize: true,
// quantize: true,
// precision: "binary"
})
papers = chiConference.contents.map((d) => {
d.id = +d.id;
d.sessions = d.sessionIds.map((s) => {
const session = maps.get("sessions").get(s);
session.room = maps.get("rooms").get(session.roomId);
session.timeSlot = maps.get("timeSlots").get(session.timeSlotId);
return session;
});
d.firstSessionName = d.sessions[0].name;
d.firstRoomName = d.sessions[0].room?.name;
d.firstTimestamp = new Date(d.sessions[0].timeSlot?.startDate);
d.firstSessionNameWord = d.sessions[0].name.split(" ")[0];
d.track = maps.get("tracks").get(d.trackId).name;
d.authorsExpanded = d.authors.map((a) => maps.get("people").get(a.personId));
d.authorsList = d.authorsExpanded.map((a) =>
`${a.firstName?.trim()} ${a.lastName?.trim()}`.toLocaleLowerCase()
);
d.authorNames = d.authorsList.join(", ");
d.affiliationsList = [
...new Set(
d.authors
.map((a) =>
a.affiliations.map((f) => f.institution.trim().toLocaleLowerCase())
)
.flat()
)
];
d.url = `https://programs.sigchi.org/chi/2025/program/content/${d.id}`;
return d;
})
maps = {
const attrs = [
// "conference",
// "publicationInfo",
"sponsors",
"sponsorLevels",
"floors",
"rooms",
"tracks",
"contentTypes",
"timeSlots",
"sessions",
"events",
"contents",
"people",
"recognitions"
];
const maps = new Map(
attrs.map((a) => [a, new Map(chiConference[a].map((d) => [d.id, d]))])
);
return maps;
}
chiConference = (await FileAttachment("CHI_2025_program.json.zip").zip())
.file("CHI_2025_program.json")
.json()
kmeans = import('https://cdn.skypack.dev/ml-kmeans@6.0.0?min')
embeddings
clusters = {
if (!computeClusters) return null;
console.log("Computing clusters");
const before = performance.now();
const clusters = kmeans.kmeans(
embeddings.map((d) => d.embedding.data),
maxClusters
);
clusters.clusters.map((c, i) => {
const paper = mapPaperId.get(embeddings[i].id);
if (paper) paper.cluster = c;
else {
console.log("paper", embeddings[i].id, " not found in map");
}
});
console.log("🤡 clusters computed", (performance.now() - before) / 1000);
return clusters;
}
clusterPapers = {
if (!computeClusters) return [];
const indexes = clusters.clusters.reduce((p, d, i) => {
if (d === cluster) p.push(i);
return p;
}, []);
return indexes.map((i) => papers[i]);
}
viewof cluster = Inputs.range([0, maxClusters], {step: 1})
Inputs.table(clusterPapers)
authors = [...new Set(papers.map(d => d.authorsList).flat())].sort()
affiliations = [...new Set(papers.map(d => d.authors.map( a=> a.affiliations[0]?.institution.trim().toLocaleLowerCase())).flat())].sort()
viewof initialAuthors = {
const widget = await PersistInput(
"authors",
Inputs.checkbox(authors, { maxHeight: 100 })
);
widget.style.height = "100px";
widget.style.overflow = "scroll";
return widget;
}
viewof initialAffiliations = {
const widget = PersistInput(
"affiliations",
Inputs.checkbox(affiliations, { maxHeight: "100px" })
);
widget.style.height = "100px";
widget.style.overflow = "scroll";
return widget;
}
import {multiAutoSelect} from "@john-guerra/multi-auto-select@262"
import {PersistInput} from '@john-guerra/persist-input'
function getVegaView(dataToPlot, options) {
const {
interactive,
colorScheme,
color,
shape,
x,
y,
size,
id,
tooltip,
colorDomain,
title,
shapeDomain
} = options;
// Make it depend on clustering
clusters;
let colorField;
if (computeClusters) {
colorField = vl
.color()
.if("datum.cluster == undefined", vl.value("#ccc7"))
.fieldN(color)
.scale({
domain: colorDomain,
scheme: colorScheme || options.colorSchemeQuantitative
});
} else {
colorField = vl
.color()
.if("datum.cluster == undefined", vl.value("#ccc7"))
.fieldQ(color)
.scale({
domain: colorDomain,
scheme: colorScheme || options.colorSchemeQuantitative
});
}
let base = vl
.markPoint({ opacity: 0.6, filled: true })
.encode(vl.x().fieldQ(x).axis(null), vl.y().fieldQ(y).axis(null))
.data(dataToPlot.filter((d) => d.type !== "keyword"));
let baseWithParams;
if (color) base = base.encode(colorField);
if (size)
base = base.encode(
vl.size().fieldQ(size).scale({ range: options.sizeRange }),
vl.order().fieldQ(size)
);
if (shape) {
let shapeEncoding = vl.shape().fieldN(shape);
if (shapeDomain) {
shapeEncoding = shapeEncoding.scale({ domain: shapeDomain });
}
base = base.encode(shapeEncoding);
}
const boxes = vl
.markRect({ filled: false })
.encode(
vl.x().fieldQ("bbox.minX"),
vl.y().fieldQ("bbox.minY"),
vl.x2().fieldQ("bbox.maxX"),
vl.y2().fieldQ("bbox.maxY")
)
// .width(Math.max(320, width - 100))
// .height(500)
.data(hdbscanClusters);
// console.log("chart", chart.toObject());
const keywords = vl
.markText({ fontSize: 8, tooltip: true, opacity: 0.7 })
.encode(vl.x().fieldQ("x"), vl.y().fieldQ("y"), vl.text().fieldN("title"))
.data(dataToPlot.filter((d) => d.type === "keyword"));
const events = "mouseover,pointerover,touchmove,click";
if (interactive) {
// console.log("vega interactive!", interactive);
const hover = vl
.selectSingle("hover")
.nearest(true)
.on(events)
.clear("none")
.init({ x: [], y: [] });
const drag = vl.selectInterval("drag");
const click = vl
.selectPoint("click")
.fields([id])
.nearest(true)
.on("click")
.init({ id: [] });
const shiftClick = vl
.selectPoint("shiftClick")
.fields([id])
.on("click[event.altKey]")
.toggle("false")
.init({ id: [] });
baseWithParams = base.params(click, hover, drag, shiftClick).encode(
vl
.stroke()
.condition({ param: "click", value: "black", empty: false })
// .condition({param: "hover", value: "grey", empty: false})
.value(null),
// vl.size().if(vl.or(hover, drag), vl.value(80)).value(50),
vl.tooltip(tooltip)
);
if (color)
baseWithParams = baseWithParams.encode(
vl
.color()
// .if(computeClusters ? "!datum.cluster": "false", vl.value("grey"))
.if(vl.or(hover, drag), colorField)
.value("grey")
);
}
let chart = vl.layer(
...[
baseWithParams,
clusteringSpace === "Reduced Space" && clusteringType === "HDBScan"
? boxes
: null,
includeKeywords ? keywords : null
].filter((d) => d)
);
// console.log("chart", dataToPlot, chart.toObject());
return chart.title(title).width(options.width).height(options.height);
}
Clustering = import('https://cdn.skypack.dev/hdbscanjs@1.0.12?min').then(res => res.default)
clusteredTree = {
if (!computeClusters) return {};
const dataset = Object.entries(reducerResult).map(([k, v]) => ({
data: v,
id: k
}));
// two distance measure functions are supported:
// 1) euclidean
// 2) geoDist (take inputs as lonlat points)
const distFunc = Clustering.distFunc.euclidean;
const before = performance.now();
const cluster = new Clustering(dataset, distFunc);
const treeNode = cluster.getTree();
console.log("HDBScan run in ", performance.now() - before);
return treeNode;
// const filtered = treeNode.filter((val) => val.data.length === 2, {
// minX: 0,
// maxX: width,
// minY: 0,
// maxY: innerHeight
// });
// filtered.forEach((x) => console.log(x.toString()));
// const filterFunc = val => ...;
// const bbox = {minX:.., maxX:.., minY:.., maxY:..};
// const filteredNodes = treeNode.filter(filterFunc, bbox);
}
Object.entries(reducerResult).map(([k, v]) => ({
data: v,
id: k
}));
convertTree(clusteredTree)
Tree(convertTree(clusteredTree), { height: 500, width, fit: true, r: 0.5 })
import {Tree} from "@john-guerra/tree-fit"
onClustersChangeColor = {
if (computeClusters === true) {
viewof colorBy.value = "cluster";
viewof colorBy.dispatchEvent(new CustomEvent("input"), { bubbles: true });
viewof colorScheme.value = "turbo";
viewof colorScheme.dispatchEvent(new CustomEvent("input"), { bubbles: true });
} else {
viewof colorBy.value = "score";
viewof colorBy.dispatchEvent(new CustomEvent("input"), { bubbles: true });
viewof colorScheme.value = "brownbluegreen";
viewof colorScheme.dispatchEvent(new CustomEvent("input"), { bubbles: true });
}
}
dataToPlotWithClusters = {
console.log("🏘️ Recompute dataToPlotWithClusters", clusters);
clusters;
return dataToPlot;
}
// clusterHDBScanChart = {
// const edges = vl
// .markLine({ filled: false })
// .encode(
// vl.x().fieldQ("edge[0][0]"),
// vl.y().fieldQ("edge[0][1]"),
// vl.x2().fieldQ("edge[1][0]"),
// vl.y2().fieldQ("edge[1][1]")
// )
// .width(600)
// .height(400)
// .data(hdbscanClusters);
// const boxes = vl
// .markRect({ filled: false })
// .encode(
// vl.x().fieldQ("bbox.minX"),
// vl.y().fieldQ("bbox.minY"),
// vl.x2().fieldQ("bbox.maxX"),
// vl.y2().fieldQ("bbox.maxY")
// )
// .width(Math.max(320, width - 100))
// .height(500)
// .data(hdbscanClusters);
// const mapClusters = new Map(
// hdbscanClusters.map((d, c) => d.opt.map((e) => [e, c])).flat()
// );
// const points = vl
// .markPoint({ tooltip: true })
// .encode(
// vl.x().fieldQ("x"),
// vl.y().fieldQ("y"),
// vl.shape().fieldN("type"),
// vl.size().fieldQ("score"),
// vl.tooltip(["title", "authors", "abstract", "id", "score", "cluster"]),
// vl
// .color()
// .if("datum.cluster == null", vl.value("#ccc7"))
// .fieldQ("cluster")
// .scale({ scheme: "turbo" })
// )
// .data(
// dataToPlotWithClusters.map((d) => ({
// ...d,
// cluster: mapClusters.get(d.id)
// }))
// );
// return vl
// .layer(
// ...[
// edges,
// clusteringSpace === "Reduced Space" ? boxes : null,
// points
// ].filter((d) => d)
// )
// .render();
// }
clusteredTreeViz = Tree(convertTree(hdbScanClusteredTree), {
height: 400,
width,
fit: true,
r: 1,
tree: d3.cluster,
sort: (a,b) => d3.descending(a.data.data.length, b.data.data.length)
})
// viewof computeClusters = Inputs.toggle({
// label:
// "Compute Clusters (WIP)",
// value: false
// })
clustersKmeans = {
if (
!computeClusters ||
clusteringType !== "Kmeans"
)
return {};
const before = performance.now();
let dataset;
if (clusteringSpace === "Embeddings") {
dataset = navioSelected.map(({ id }) => embeddingsHash.get(id).data);
} else {
dataset = dataToPlot.map(({ x, y }) => [x, y]);
// dataset = Object.entries(reducerResult).map(([k, v]) => v);
}
const clusters = await kmeans.kmeans(dataset, maxClusters);
clusters.clusters.map((c, i) => {
dataToPlot[i].cluster = c;
});
console.log(
"🦜🦜🦜 Kmeans clusters computed in",
(performance.now() - before) / 1000
);
return clusters;
}
HDBScanJS = import('https://cdn.skypack.dev/hdbscanjs@1.0.12?min').then(res => res.default)
hdbScanClusteredTree?.dist || 1
// function enoughTimeSinceLastDRResult() {
// let diff;
// console.log(
// "⏰ enoughTimeSince edit",
// (diff = performance.now() - lastReducerTimestamp),
// diff > minTimeDeltaForClustering
// );
// return (
// performance.now() - lastReducerTimestamp > minTimeDeltaForClustering
// );
// }
hdbScanClusteredTree = {
reducerResult;
if (
!computeClusters ||
clusteringType != "HDBScan"
)
return {};
const before = performance.now();
console.log("🦜 Running HDBScan");
let dataset, distFunc;
if (clusteringSpace === "Embeddings") {
// Cluster embeddings
dataset = dataToPlot.map(({ id }) => ({
data: embeddingsHash.get(id).data,
opt: id
}));
distFunc = pairCosineSimilarity;
} else {
// two distance measure functions are supported:
// 1) euclidean
// 2) geoDist (take inputs as lonlat points)
distFunc = HDBScanJS.distFunc.euclidean;
// // Cluster DR
dataset = dataToPlot.map(({ id, x, y }) => ({ data: [x, y], opt: id }));
// dataset = Object.entries(reducerResult).map(([k, v]) => ({
// data: v,
// opt: k
// }));
}
const cluster = new HDBScanJS(dataset, distFunc);
const treeNode = cluster.getTree();
console.log("🦜 HDBScan ran in ", performance.now() - before);
return treeNode;
}
hdbscanClusters = {
hdbScanClusteredTree;
console.log("🦜🤯 Trying to copy HDBScans clusters");
if (!computeClusters || clusteringType != "HDBScan") return [];
const filterCond = (val) =>
val.data.length >= papersPerClusterRange[0] &&
val.data.length < papersPerClusterRange[1] &&
val.dist <= maxClusteringDistance;
const hdbscanClusters = hdbScanClusteredTree.filter(filterCond);
const mapClusters = new Map(
hdbscanClusters.map((d, c) => d.opt.map((e) => [e, c])).flat()
);
dataToPlot.forEach((d) => {
d.cluster = mapClusters.get(d.id);
// d.cluster = d.cluster === null ? -1 : d.cluster;
});
console.log("🦜🤯 HDBScans clusters copied!");
return hdbscanClusters;
}
corpus = {
if (!computeClusters) return {};
const groupedClusters = hdbscanClusters.map((c, i) =>
c.opt.map((id) => mapPaperId.get(id))
);
return new tfidf.Corpus(
groupedClusters.map((_, i) => i),
groupedClusters.map((c) =>
c
.map(
(d) => d?.title
+ d?.abstract
)
.join("\n")
),
true,
[],
5,
0.9
);
}
groupedClusters = hdbscanClusters.map((c, i) =>
c.opt.map((id) => mapPaperId.get(id))
);
mapPaperId = new Map(dataToPlot.map(d => [d.id, d]))
function convertTree(t) {
let children = [];
if (t?.left) children = [convertTree(t.left)];
if (t?.right) children = [...children, convertTree(t.right)];
return {
...t,
children
};
}
import {interval} from '@mootari/range-slider'
tfidf = import('https://unpkg.com/tiny-tfidf@0.9.1/index.js?module')
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.