Digital Humanist. Manhattan-based.
async function create_duckdb_copy(tile) {
// Builds a copy on duckdb of the data in a tile.
// the new table will have the same name as the tile.
const key = tile.key;
const table_exists_already = db
.query(
`SELECT * FROM information_schema.tables WHERE table_name = '${key}'`
)
.then((d) => d.numRows > 0);
const table_is_downloaded = tile.download;
const [is_there, is_ready_to_post] = await Promise.all([
table_exists_already,
table_is_downloaded
]);
if (is_there) {
return key;
}
const tb = new arrow.Table(tile.record_batch.schema, tile.record_batch);
if (tb.length === 0) {
throw new Error("humbug");
}
const ipc = arrow.tableToIPC(tb);
const name = key;
const databadse = await db.db();
const conn = await databadse.connect();
await conn.query(`DROP TABLE IF EXISTS "${name}"`);
await conn.insertArrowFromIPCStream(ipc, {
name,
schema: "main", // Not sure what this is.
options: {} // no options.
});
await conn.close();
return key;
}
listener = {
// This being observable, the code below will run every time 'searchterm' changes.
// It creates a new async transformation on deepscatter that returns `1` for each entry where the regex matches, and `0` for each where it doesn't.
if (searchterm == "") {
return;
}
scatterplot._root.transformations[searchterm] = async function (tile) {
// First ensure it exists in duckdb.
await create_duckdb_copy(tile);
const key = tile.key;
// Get the regex query out of duckdb.
const value = await db.query(
`SELECT regexp_matches(lyric_line, '${searchterm}')::FLOAT AS "${searchterm}" FROM "${key}"`
);
// Grab just the column called 'searchterm' and convert it to
// a floating point array. DuckDB returns multi-record-batch chunks,
// so we have to use the Float32Array rather than inserting the arrow
// column directly.
const asArray = value.getChild(searchterm).toArray();
return asArray;
};
}
transformation_plot = {
listener;
if (searchterm != "") {
//
scatterplot.plotAPI({
background_options: {
size: [0.4, 2] // Background half normal size, foreground double normal size.
},
encoding: {
foreground: {
field: searchterm,
op: "eq",
a: 1
}
}
});
} else {
scatterplot.plotAPI({
background_options: {
size: [0.4, 1] // Bring the foreground size back down to normal.
},
encoding: {
foreground: null
}
});
}
}
scatterplot = {
const plot = new deepscatter.default("#plot", 900, 600);
await plot.plotAPI({
source_url: "https://davidnmora.github.io/lyric-viz/deepscatter-tiles",
point_size: 1,
max_points: 1e6,
zoom_balance: 0.35,
alpha: 40,
background_color: "#FFFAF2",
encoding: {
x: {
field: "x",
transform: "literal"
},
y: {
field: "y",
transform: "literal"
},
color: {
field: "generic_genre",
range: "category10"
}
}
});
invalidation.then(() => plot.destroy());
return plot;
}
html`
<style type="text/css">
.tooltip {
background-color: black;
width: 400px;
font-family: sans-serif;
}
.rect {
fill: white;
}
</style>
`
deepscatter = import("https://benschmidt.org/deepscatter@2.9.3")
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
arrow = scatterplot.arrow
db = {
const db = await DuckDBClient.of({});
return db;
}
import {
DuckDBClient,
} from "@bmschmidt/duckdb-client-1-24-0-arrow-11-0-0"
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.