client
query_script = `select * exclude (wow)`
// query_script = `select * from range(0, 10)`
// query_script = query_script_examples[query_example_index]
query_ast = JSON.parse(query_ast_string)
<!-- Token color -->
<div style='font-family: monospace'>
<div style='font-family: monospace'>
${query_script}
</div>
<div>
${[...Array(rebuilt_query_length).keys()].map(i => {
return htl.html`<span style='opacity: ${ast_query_location_indices.has(i) ? 1 : 0}'>▼</span>`
})}
</div>
<div>
${query_token_strings.map((d,i) => {
return htl.html`<span style='background-color: ${token_color_scale(i)}'>${d}</span>`
})}
</div>
</div>
{
return htl.html`
<div style='font-family: monospace'>
${query_token_strings.map((d,i) => {
return htl.html`
<span>${i} -></span>
<span style='background-color: ${token_color_scale(i)}'>${d}</span>
<span>-></span>
<span>${query_token_types[i]}</span>
<br/>
`
})}
</div>
`
}
{
// Now do the above but for everything that has a query location
}
rebuilt_query_ast = JSON.parse(rebuilt_query_ast_string)
query_ast_string = await parse_sql_to_json(query_script)
rebuilt_query_ast_string = await parse_sql_to_json(rebuilt_query_script)
rebuilt_query_script = {
const inner = `select json_deserialize_sql('${query_ast_string}') as sql`
const res = await client.queryRow(inner)
return res.sql
}
// query_tokens = await db_instance.tokenize(query_script)
query_tokens = await db_instance.tokenize(rebuilt_query_script)
query_token_strings = {
const start_offset = query_tokens.offsets;
const end_offset = [...start_offset.slice(1), undefined]
return start_offset.map((_,i) => rebuilt_query_script.substring(start_offset[i], end_offset[i]))
}
rebuilt_query_length = {
return _.sum(query_token_strings.map(d => d.length))
}
query_token_types = query_tokens.types.map(d => token_type_enum[d])
ast_query_location_indices = {
let data = []
mapKeysDeep(rebuilt_query_ast, (v,k) => {
if (k === 'query_location') {
data.push(v)
}
return k
})
// data.sort((a,b) => a - b)
return new Set(data)
}
typeof ast_query_location_indices[0]
token_type_enum = ['IDENTIFIER', 'NUMERIC_CONSTANT', 'STRING_CONSTANT', 'OPERATOR', 'KEYWORD', 'COMMENT']
token_color_scale = d3.scaleOrdinal().range(
d3.schemeCategory10.map(d => chroma(d).brighten(2))
)
chroma = require("chroma-js@2.4.2")
function escape_sql(sql) {
return sql.replaceAll(`"`, `""`).replaceAll(`'`, `''`)
}
async function parse_sql_to_json(sql) {
const inner = `select json_serialize_sql('${escape_sql(sql)}') as ast`
const res = await client.queryRow(inner)
return res.ast
}
function mapKeysDeep(obj, cb) {
return _.mapValues(
_.mapKeys(obj, cb),
val => (_.isObject(val) ? mapKeysDeep(val, cb) : val),
)
}
mapKeysDeep({'a': 1, 'b': {'c': 2, 'd': 3}}, (v,k) => k.toUpperCase())
db_instance.getVersion()
duckdb = import(
"https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@1.28.1-dev232.0/+esm"
)
bundle = {
const bundles = duckdb.getJsDelivrBundles()
if (duckdb_bundle === 'Auto') {
return duckdb.selectBundle(bundles)
} else {
const bun = bundles[duckdb_bundle]
bun['pthreadWorker'] = null;
return bun;
}
// return bundles['mvp']
}
async function makeDB() {
const logger = new duckdb.ConsoleLogger();
const worker = await duckdb.createWorker(bundle.mainWorker);
const db = new duckdb.AsyncDuckDB(logger, worker);
await db.instantiate(bundle.mainModule);
return db
}
db_instance = makeDB()
client = {
const client_class = (duckdb_client === 'Compatibility') ? DuckDBClientCompat : DuckDBClient;
const c = new client_class(db_instance);
return c;
}
class DuckDBClientCompat extends DuckDBClient {
async queryStream(query, params) {
const connection = await this._db.connect();
let reader, batch;
try {
if (params?.length > 0) {
const statement = await connection.prepare(query);
reader = await statement.send(...params);
} else {
reader = await connection.send(query);
}
batch = await reader.next();
if (batch.done) throw new Error("missing first batch");
} catch (error) {
await connection.close();
throw error;
}
// Mosaic utility: convert Arrow value to Javascript value
const converters = {}
batch.value.schema.fields.forEach(d => {
console.log('Type for ', d.name, d.type)
converters[d.name] = convertArrowValue(d.type)
})
return {
schema: getArrowTableSchema(batch.value),
async *readRows() {
try {
while (!batch.done) {
let batch_array = batch.value.toArray();
// Convert all values to Javascript version
let object_array = []
for (let i = 0; i < batch_array.length; i++) {
const d_proxy = batch_array[i];
const d_obj = {}
for (let k of Object.keys(converters)) {
d_obj[k] = converters[k](d_proxy[k])
}
object_array.push(d_obj)
}
yield object_array;
batch = await reader.next();
}
} finally {
await connection.close();
}
}
};
}
}
getArrowTableSchema = observable_stdlib.getArrowTableSchema
convertArrowValue = mosaic_core.convertArrowValue
observable_stdlib = await import('https://cdn.jsdelivr.net/npm/@observablehq/stdlib@5.8.7/+esm');
mosaic_core = await import('https://cdn.jsdelivr.net/npm/@uwdata/mosaic-core@0.9.0/+esm');
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.