Public
Edited
Aug 15, 2024
1 fork
getTableMemoryUsage(inputTable)
getTableMemoryUsage(outputTable)
tableSize = 10000
function repeatString(val, numRows) {
return arrow.vectorFromArray(
Array.from({ length: numRows }).fill(val),
new arrow.Dictionary(new arrow.Utf8, new arrow.Uint8),
)
}
inputTable = arrow.makeTable({
foo: repeatString("someRepeatedStringHere00000000", tableSize)
})
db = {
return tableSize && DuckDBClient.of({});
}
conn = await db._db.connect()
insertionComplete = {
const buffer = arrow.tableToIPC(inputTable, 'stream');
conn.insertArrowFromIPCStream(buffer, {
name: "arrowTable",
schema: "main"
});
return true;
}
outputTable = tableSize && insertionComplete && conn.query('SELECT * FROM arrowTable')
inputTable.getChild("foo")
inputTable.getChild("foo").get(0)
outputTable.getChild("foo")
outputTable.getChild("foo").get(0)
inputTable.batches.length === 1
outputTable.batches.length === 1
outputTable.batches[0].data.children[0]
inputTable.batches[0].data.children[0].values
outputTable.batches[0].data.children[0].values
inputTable.batches[0].data.children[0].dictionary.data[0].values
outputTable.batches[0].data.children[0].values
outputTable.batches[0].data.children[0].valueOffsets
getTableMemoryUsage = (t) => {
const typedArrs = [];
t.batches.forEach(b => {
if(b.data.children[0].dictionary === undefined) {
typedArrs.push(b.data.children[0].values);
typedArrs.push(b.data.children[0].valueOffsets);
} else {
typedArrs.push(b.data.children[0].values);
typedArrs.push(b.data.children[0].dictionary.data[0].values);
}
})
return typedArrs.map(a => a.length * a.BYTES_PER_ELEMENT).reduce((a, h) => a + h, 0);
}
// Reference: https://github.com/observablehq/feedback/issues/623#issuecomment-1999937586
arrow = await import('https://cdn.observableusercontent.com/npm/apache-arrow@11.0.0/+esm')
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
