Digital Humanist. Manhattan-based.
async function create_duckdb_copy(tile) {
// Builds a copy on duckdb of the data in a tile.
// the new table will have the same name as the tile.
const key = tile.key;
const table_exists_already = db
.query(
`SELECT * FROM information_schema.tables WHERE table_name = '${key}'`
)
.then((d) => d.numRows > 0);
const table_is_downloaded = tile.download;
const [is_there, is_ready_to_post] = await Promise.all([
table_exists_already,
table_is_downloaded
]);
if (is_there) {
return key;
}
const tb = new arrow.Table(tile.record_batch.schema, tile.record_batch);
if (tb.length === 0) {
throw new Error("humbug");
}
const ipc = arrow.tableToIPC(tb);
const name = key;
const databadse = await db.db();
const conn = await databadse.connect();
await conn.query(`DROP TABLE IF EXISTS "${name}"`);
await conn.insertArrowFromIPCStream(ipc, {
name,
schema: "main", // Not sure what this is.
options: {} // no options.
});
await conn.close();
return key;
}
listener = {
// This being observable, the code below will run every time 'searchterm' changes.
// It creates a new async transformation on deepscatter that returns `1` for each entry where the regex matches, and `0` for each where it doesn't.
if (searchterm == "") {
return;
}
scatterplot._root.transformations[searchterm] = async function (tile) {
// First ensure it exists in duckdb.
await create_duckdb_copy(tile);
const key = tile.key;
// Get the regex query out of duckdb.
const value = await db.query(
`SELECT regexp_matches(lyric_line, '${searchterm}')::FLOAT AS "${searchterm}" FROM "${key}"`
);
// Grab just the column called 'searchterm' and convert it to
// a floating point array. DuckDB returns multi-record-batch chunks,
// so we have to use the Float32Array rather than inserting the arrow
// column directly.
const asArray = value.getChild(searchterm).toArray();
return asArray;
};
}
transformation_plot = {
listener;
if (searchterm != "") {
//
scatterplot.plotAPI({
background_options: {
size: [0.4, 2] // Background half normal size, foreground double normal size.
},
encoding: {
foreground: {
field: searchterm,
op: "eq",
a: 1
}
}
});
} else {
scatterplot.plotAPI({
background_options: {
size: [0.4, 1] // Bring the foreground size back down to normal.
},
encoding: {
foreground: null
}
});
}
}
scatterplot = {
const plot = new deepscatter.default("#plot", 900, 600);
await plot.plotAPI({
source_url: "https://davidnmora.github.io/lyric-viz/deepscatter-tiles",
point_size: 1,
max_points: 1e6,
zoom_balance: 0.35,
alpha: 40,
background_color: "#FFFAF2",
encoding: {
x: {
field: "x",
transform: "literal"
},
y: {
field: "y",
transform: "literal"
},
color: {
field: "generic_genre",
range: "category10"
}
}
});
invalidation.then(() => plot.destroy());
return plot;
}
html`
<style type="text/css">
.tooltip {
background-color: black;
width: 400px;
font-family: sans-serif;
}
.rect {
fill: white;
}
</style>
`
deepscatter = import("https://benschmidt.org/deepscatter@2.9.3")
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
arrow = scatterplot.arrow
db = {
const db = await DuckDBClient.of({});
return db;
}
import {
DuckDBClient,
} from "@bmschmidt/duckdb-client-1-24-0-arrow-11-0-0"
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
