Public
Edited
Mar 21, 2024
mapContainer = html`<div style="height:900px"></div>`
layer = new deck.SolidPolygonLayer({
// This is an Observable hack - changing the id will force the layer to refresh when the cell reevaluates
// id: `layer-${Date.now()}`,
data:data,
// wireframe: true,
// Skip normalization for binary data
_normalize: false,
// Counter-clockwise winding order
_windingOrder: "CCW",
getFillColor: [0, 0, 255, 100],
getColor: [255, 0, 0],
// extruded: true
})
// deckglMap.setProps({ layers: [layer, pathLayer] })
deckglMap.setProps({ layers: [pathLayer] })
deckglMap = {
// This is an Observable hack: clear previously generated content
mapContainer.innerHTML = "";
return new deck.DeckGL({
// The HTML container to render into
container: mapContainer,
views: [new deck.OrthographicView({id: 'ortho'})],
initialViewState: {target: [20000, 20000, 0], zoom: 0},
controller: true
});
}
geometry_column_1 = arrow_tables[0].getChild('GEOMETRY')
geometry_column_2 = arrow_tables[1].getChild('GEOMETRY')
// arrowTable = concatenate_arrow_tables(tile_cell_tables)
arrowTable = arrow_tables[0]
geometryColumn = arrowTable.getChild('GEOMETRY')
geometryColumn.data[0].valueOffsets
// {
// let concatenatedOffsets = new Int32Array();
// let cumulativeOffset = 0; // This will track the adjustment needed for each subsequent offsets array
// // Assuming geometryColumn.data is an iterable array of your data pieces
// geometryColumn.data.forEach((dataPiece, index) => {
// let offsets = dataPiece.valueOffsets; // The current offsets array
// if (index === 0) {
// // For the first data piece, just copy the offsets
// concatenatedOffsets = new Int32Array(offsets);
// } else {
// // Adjust and concatenate for subsequent data pieces
// let adjustedOffsets = new Int32Array(offsets.length);
// for (let i = 0; i < offsets.length; i++) {
// adjustedOffsets[i] = offsets[i] + cumulativeOffset;
// }
// // Concatenate the adjusted offsets with what we have so far
// concatenatedOffsets = new Int32Array([...concatenatedOffsets, ...adjustedOffsets]);
// }
// // Update the cumulative offset for the next iteration
// // Subtract by offsets[0] if it's not guaranteed to be 0 or if it's a relative offset
// cumulativeOffset += offsets[offsets.length - 1];
// });
// console.log(concatenatedOffsets);
// return concatenatedOffsets
// }
arrowTable.getChild('GEOMETRY')
geometryColumn.data[0]
geometryColumn.getChildAt(0)
geometryColumn.getChildAt(0).getChildAt(0).getChildAt(0)
geometryColumn
get_data = (arrowTable) => {
// var geometryColumn = arrowTable.getChildAt(0)
var geometryColumn = arrowTable.getChild('GEOMETRY')
// here is where we will have to work with multiple arrays
////////////////////////////////////////////////////////////////
console.log('checking geometry columns from 0 and 1')
console.log(geometryColumn.data[0])
console.log(geometryColumn.data[1])
var polygonIndices = geometryColumn.data[0].valueOffsets
var ringIndices = geometryColumn.getChildAt(0).data[0].valueOffsets
var flatCoordinateVector = geometryColumn.getChildAt(0).getChildAt(0).getChildAt(0)
var flatCoordinateArray = flatCoordinateVector.data[0].values
const resolvedIndices = new Int32Array(polygonIndices.length);
for (let i = 0; i < resolvedIndices.length; ++i) {
// Perform the lookup into the ringIndices array using the polygonIndices array
resolvedIndices[i] = ringIndices[polygonIndices[i]]
}
var data = {
// Number of geometries
length: arrowTable.numRows,
// Indices into coordinateArray where each polygon starts
startIndices: resolvedIndices,
// Flat coordinates array
attributes: {
getPolygon: { value: flatCoordinateArray, size: 2 }
}
}
return data
}
// data = get_data(arrowTable)
// data = concatenateData([data_0, data_1 /*, more data objects if needed*/]);
data = concatenate_polygon_data(datas);
concatenate_polygon_data = (dataObjects) => {
// Initialize concatenated data structure
let concatenatedData = {
length: 0,
startIndices: new Int32Array(),
attributes: {
getPolygon: {
value: new Float64Array(),
size: 2 // Assuming 'size' remains constant
}
}
};
// Iterate over each data object to combine them
dataObjects.forEach((data, index) => {
concatenatedData.length += data.length;
// Handle startIndices
// let lastValue = concatenatedData.attributes.getPolygon.value.length
let lastValue = concatenatedData.attributes.getPolygon.value.length/2;
let adjustedStartIndices = data.startIndices;
if (index > 0) {
// Adjust startIndices (except for the first data object)
adjustedStartIndices = new Int32Array(data.startIndices.length);
for (let i = 0; i < data.startIndices.length; i++) {
adjustedStartIndices[i] = data.startIndices[i] + lastValue;
}
}
// Combine startIndices and coordinate values
concatenatedData.startIndices = new Int32Array([...concatenatedData.startIndices, ...adjustedStartIndices.slice(index > 0 ? 1 : 0)]);
concatenatedData.attributes.getPolygon.value = new Float64Array([...concatenatedData.attributes.getPolygon.value, ...data.attributes.getPolygon.value]);
});
return concatenatedData;
}
data_0 = get_data(arrow_tables[0])
data_1 = get_data(arrow_tables[1])
datas = arrow_tables.map(x => get_data(x))
// data = get_data(arrowTables)
pathLayer = new deck.PathLayer({
id: `path-layer-${Date.now()}`,
// data: polygonPaths,
data: polygonPathsConcat,
pickable: true,
widthScale: 1,
widthMinPixels: 0.5,
getPath: d => d,
getColor: d => [255, 0, 0, 255], // Red outline
widthUnits: 'pixels',
})
// polygonPaths = extractPolygonPaths(data)
extractPolygonPaths = (data) => {
const paths = [];
const { startIndices, attributes } = data;
const coordinates = attributes.getPolygon.value;
const numPolygons = startIndices.length - 1;
for (let i = 0; i < numPolygons; ++i) {
const startIndex = startIndices[i] * 2; // Multiply by 2 because each coordinate is a pair of values (x, y)
const endIndex = startIndices[i + 1] * 2; // The next start index marks the end of the current polygon
const path = [];
for (let j = startIndex; j < endIndex; j += 2) {
const x = coordinates[j];
const y = coordinates[j + 1];
path.push([x, y]);
}
paths.push(path);
}
return paths;
}
url_1 = 'https://raw.githubusercontent.com/cornhundred/test_human_ffpe/main/cell_segmentation_v2/cell_tile_20_20.parquet'
url_2 = 'https://raw.githubusercontent.com/cornhundred/test_human_ffpe/main/cell_segmentation_v2/cell_tile_21_20.parquet'
url_3 = 'https://raw.githubusercontent.com/cornhundred/test_human_ffpe/main/cell_segmentation_v2/cell_tile_22_20.parquet'
url_4 = 'https://raw.githubusercontent.com/cornhundred/test_human_ffpe/main/cell_segmentation_v2/cell_tile_20_21.parquet'
url_5 = 'https://raw.githubusercontent.com/cornhundred/test_human_ffpe/main/cell_segmentation_v2/cell_tile_21_21.parquet'
url_6 = 'https://raw.githubusercontent.com/cornhundred/test_human_ffpe/main/cell_segmentation_v2/cell_tile_22_21.parquet'
urls = [url_1, url_2, url_3, url_4, url_5, url_6]
arrow_tables = await fetch_all_tables(urls)
polygonPathsConcat = extractPolygonPaths(data)
fetch_all_tables = async (urls) => {
try {
const promises = urls.map(url => get_arrow_table(url));
const tables = await Promise.all(promises);
return tables;
} catch (error) {
console.error("Error fetching tables:", error);
return [];
}
}
concatenate_arrow_tables = (tables) => {
if (tables.length === 0) return null; // No tables to concatenate
let baseTable = tables[0]; // Use the first table as the base
for (let i = 1; i < tables.length; i++) { // Start from the second table
baseTable = baseTable.concat(tables[i]); // Concatenate each table to the base table
}
return baseTable; // Return the concatenated result
};
get_arrow_table = async (url) => {
try {
const response = await fetch(url);
const arrayBuffer = await response.arrayBuffer();
const arr = new Uint8Array(arrayBuffer);
const arrowIPC = pq.readParquet(arr);
const arrowTable = arrow.tableFromIPC(arrowIPC);
return arrowTable
} catch (error) {
console.error("Error loading data:", error);
return [];
}
}
// Load the parquet-wasm library
// readParquet = {
pq = {
const parquetModule = await import(
"https://unpkg.com/parquet-wasm@0.4.0-beta.5/esm/arrow2.js"
);
// Need to await the default export first to initialize the WebAssembly code
await parquetModule.default();
return parquetModule;
}
arrow = require("apache-arrow")
mapboxgl = require("mapbox-gl@1.6.0/dist/mapbox-gl.js")
deck = require.alias({ h3: {} })("deck.gl@8.8.6/dist.min.js")
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
