2c:null 2d:["$","title",null,{"children":"ImageBox3 + Phikon for on-the-fly embeddings | Praful Bhawsar | Observable"}] 32:T411,viewof tileParams = Object.assign( Inputs.form({ tileX: Inputs.range([0, imageInfo.width], { step: 1, value: Math.floor((imageInfo.width - 256) / 2), label: html`Tile X:` }), tileY: Inputs.range([0, imageInfo.height], { step: 1, value: Math.floor((imageInfo.height - 256) / 2), label: html`Tile Y:` }), tileWidth: Inputs.range([0, 4096], { step: 1, value: 256, label: html`Region Width:` }), tileHeight: Inputs.range([0, 4096], { step: 1, value: 256, label: html`Region Height:` }), tileSize: Inputs.radio([224], { value: 224, format: (x) => `${x}x${x}`, label: html`Tile Resolution:` }) }), { oninput: (event) => event.isTrusted && event.stopImmediatePropagation(), onchange: (event) => event.currentTarget.dispatchEvent(new Event("input")) } )33:T7b0,function imageTransforms( imageUrl, mean = [0.485, 0.456, 0.406], std = [0.229, 0.224, 0.225], canvasDim = 224 ) { return new Promise((resolve, reject) => { const tempImg = new Image(); tempImg.src = imageUrl; tempImg.crossOrigin = "anonymous"; tempImg.onload = () => { // Resize + Center cropping const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d'); canvas.width = canvasDim; canvas.height = canvasDim; const shortestEdge = Math.min(tempImg.width, tempImg.height); const scale = canvasDim / shortestEdge; const newWidth = tempImg.width * scale; const newHeight = tempImg.height * scale; const offsetX = (newWidth - canvasDim) / 2; const offsetY = (newHeight - canvasDim) / 2; ctx.drawImage(tempImg, -offsetX, -offsetY, newWidth, newHeight); const imageData = ctx.getImageData(0, 0, canvasDim, canvasDim); const { data, width, height } = imageData; const inputData = new Float32Array(width * height * 3); let j = 0; for (let i = 0; i < data.length; i += 4) { // Standard norm let r = data[i] / 255.0; let g = data[i + 1] / 255.0; let b = data[i + 2] / 255.0; // Renorm using model preprocessor_config inputData[j] = (r - mean[0]) / std[0]; inputData[j + 1] = (g - mean[1]) / std[1]; inputData[j + 2] = (b - mean[2]) / std[2]; j += 3; } const tensor = new onnxRuntime.Tensor('float32', inputData, [1, 3, canvasDim, canvasDim]); resolve(tensor); }; tempImg.onerror = (err) => reject(`Failed to load image: ${err}`); tempImg.src = imageUrl; }); }34:[] 30:{"id":"296abd89c923472b","slug":"imagebox3-plus-phikon","title":"ImageBox3 + Phikon for on-the-fly embeddings","thumbnail":"3e84f1ee973a556d8da2978f78ee38087dbf5239b485c1f96777958a0ce86569","access_level":"public","create_time":"$D2024-08-20T19:41:57.826Z","update_time":"$D2025-09-04T15:18:20.113Z","view_time":null,"first_public_time":"$D2024-08-20T19:41:57.826Z","has_unpublished_edits":null,"fork":{"id":"c8d16ab004b9e040","slug":"web-ai-digital-pathology","title":"ImageBox3 + CTransPath + Weakly Supervised Attention Model for on-device Cancer Type Classification","owner":{"id":"b0b4412d0985303e","login":"prafulb","name":"Praful Bhawsar"}},"stars":0,"starred":false,"owner":{"type":"team","id":"b0b4412d0985303e","login":"prafulb","name":"Praful Bhawsar","avatar_url":"https://avatars-next.observableusercontent.com/765f5b661080e697f1ac35466b8e2cef535a03897b0dc7163c9b13484530ac2a","tier":"starter_2024"},"version":869,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"49bdf3232a315bf5","login":"prafulb","name":"Praful Bhawsar","avatar_url":"https://avatars-next.observableusercontent.com/765f5b661080e697f1ac35466b8e2cef535a03897b0dc7163c9b13484530ac2a","tier":null}],"members":[{"type":"individual","id":"49bdf3232a315bf5","login":"prafulb","name":"Praful Bhawsar","avatar_url":"https://avatars-next.observableusercontent.com/765f5b661080e697f1ac35466b8e2cef535a03897b0dc7163c9b13484530ac2a","tier":null,"role":"editor"},{"type":"team","id":"b0b4412d0985303e","login":"prafulb","name":"Praful Bhawsar","avatar_url":"https://avatars-next.observableusercontent.com/765f5b661080e697f1ac35466b8e2cef535a03897b0dc7163c9b13484530ac2a","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"ImageBox3 + Phikon for on-the-fly embeddings","cells":[{"id":30,"value":"# ImageBox3 + Phikon for on-the-fly embeddings","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":265,"value":"viewof imageURL = Inputs.text({\n label: html`URL to WSI: `,\n value:\n \"https://storage.googleapis.com/imagebox_test/openslide-testdata/Aperio/CMU-1.svs\",\n width: 700,\n submit: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":297,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"html` $\"No$ `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":683,"value":"embeddings = {\n const sessionInput = {};\n sessionInput[session.handler.inputNames[0]] = imageTensor;\n console.time(\"embedding\");\n const x = await session.run(sessionInput);\n console.log(x);\n // const clsToken = last_hidden_state.cpuData.slice(\n // 0,\n // last_hidden_state.dims[last_hidden_state.dims.length - 1]\n // );\n console.timeEnd(\"embedding\");\n return x;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":376,"value":"html`
`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":342,"value":"imageInfo = {\n const info = await imagebox3Instance.getInfo(imageURL);\n // viewof tileParams.tileWidth.value = info * tileSizeForClassifier;\n // viewof tileParams.tileHeight.value = info * tileSizeForClassifier;\n return info;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":532,"value":"imagebox3Instance = {\n const instance = new Imagebox3(imageSource);\n await instance.init();\n return instance;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":306,"value":"tileURL = {\n console.time(\"tile\");\n const tileURL = await getImageTile(imageSource, tileParams);\n console.timeEnd(\"tile\");\n return tileURL;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":566,"value":"mutable imageSource = {\n let imageSrc = undefined;\n if (imageURL) {\n imageSrc = imageURL;\n }\n // if (imageLocal && localStorage.imageLocal !== imageLocal.name) {\n // viewof imageURL.value = \"\";\n // imageSrc = await imageLocal.blob();\n // localStorage.imageLocal = await imageLocal.name;\n // }\n return imageSrc;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":647,"value":"session = {\n // onnxRuntime.env.logLevel = \"verbose\";\n // onnxRuntime.env.debug = true;\n return onnxRuntime.InferenceSession.create(\n \"https://huggingface.co/prafulb/phikonv2-onnx/resolve/main/model.onnx\",\n {\n executionProviders: [\"webgpu\"]\n }\n );\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":37,"value":"getImageTile = async (imageURL, tileParams) => {\n const tileURL = URL.createObjectURL(\n await imagebox3Instance.getTile(...Object.values(tileParams))\n );\n return tileURL;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":656,"value":"imageTensor = await imageTransforms(tileURL)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":649,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"Imagebox3 = (\n await import(\"https://episphere.github.io/imagebox3/imagebox3.mjs\")\n).default","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":643,"value":"onnxRuntime = import(\n \"https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.min.mjs\"\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":624,"value":"localStorage.clear()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W34","stdlib":"1"}} 31:"$30:body"