2c:null 2d:["$","title",null,{"children":"ImageBox3 + CTransPath + Weakly Supervised Attention Model for on-device Cancer Type Classification | Praful Bhawsar | Observable"}] 32:T493,embeddings = { const embeddingsForSampledTiles = []; for (let i in tileURLs) { const canvas = new OffscreenCanvas( imageSizeForEncoder, imageSizeForEncoder ); const ctx = canvas.getContext("2d"); const imageTensor = await new Promise((resolve) => { const tempImg = tilePreviews.children[i]; tempImg.src = tileURLs[i]; tempImg.onload = () => { ctx.drawImage(tempImg, 0, 0); const pixelArray = Array.from( ctx .getImageData(0, 0, imageSizeForEncoder, imageSizeForEncoder) .data.filter((v, i) => i % 4 !== 3) ); resolve( new onnxRuntime.Tensor( "float32", imageTransforms(pixelArray), [1, 3, 224, 224] ) ); }; }); const { embedding: { cpuData } } = await encoderSession.run({ image: imageTensor }); embeddingsForSampledTiles.push(cpuData); } const embeddingsTensor = new onnxRuntime.Tensor( "float32", embeddingsForSampledTiles.map((e) => [...e]).flat(), [tileParams.numTiles, embeddingsForSampledTiles[0].length] ); return embeddingsTensor; }33:T619,getImageTiles = async (imageURL, tileParams) => { let activeTileRequests = 0; const getImageTile = async (imageURL) => { activeTileRequests += 1; const tileX = Math.floor( tileXSamplingRange[0] + (tileXSamplingRange[1] - tileXSamplingRange[0] - tileWidth) * Math.random() ); const tileY = Math.floor( tileYSamplingRange[0] + (tileYSamplingRange[1] - tileYSamplingRange[0] - tileHeight) * Math.random() ); const tileResolution = imageSizeForEncoder; console.log( "Requesting tile with parameters:", tileX, tileY, tileWidth, tileHeight ); const blob = await imagebox3Instance.getTile( tileX, tileY, tileWidth, tileHeight, tileResolution ); return URL.createObjectURL(blob); }; const tileURLs = await new Promise((resolve) => { const allRetrievedURLs = []; for (let i = 0; i < maxConcurrentRequests; i++) { const tileGetter = () => { getImageTile(imageURL).then((url) => { if (allRetrievedURLs.length >= tileParams.numTiles) { return; } allRetrievedURLs.push(url); activeTileRequests -= 1; if ( activeTileRequests < maxConcurrentRequests && allRetrievedURLs.length < tileParams.numTiles ) { tileGetter(); return; } else { resolve(allRetrievedURLs); } }); }; tileGetter(); } }); return tileURLs; }34:T50a,classesMap = { return { ACC: "Adrenocortical carcinoma", BLCA: "Bladder Urothelial Carcinoma", BRCA: "Breast invasive carcinoma", CESC: "Cervical squamous cell carcinoma and endocervical adenocarcinoma", CHOL: "Cholangiocarcinoma", COAD: "Colon adenocarcinoma", DLBC: "Lymphoid Neoplasm Diffuse Large B-cell Lymphoma", ESCA: "Esophageal carcinoma", GBM: "Glioblastoma multiforme", HNSC: "Head and Neck squamous cell carcinoma", KICH: "Kidney Chromophobe", KIRC: "Kidney renal clear cell carcinoma", KIRP: "Kidney renal papillary cell carcinoma", LGG: "Brain Lower Grade Glioma", LIHC: "Liver hepatocellular carcinoma", LUAD: "Lung adenocarcinoma", LUSC: "Lung squamous cell carcinoma", MESO: "Mesothelioma", OV: "Ovarian serous cystadenocarcinoma", PAAD: "Pancreatic adenocarcinoma", PCPG: "Pheochromocytoma and Paraganglioma", PRAD: "Prostate adenocarcinoma", READ: "Rectum adenocarcinoma", SARC: "Sarcoma", SKCM: "Skin Cutaneous Melanoma", STAD: "Stomach adenocarcinoma", TGCT: "Testicular Germ Cell Tumors", THCA: "Thyroid carcinoma", THYM: "Thymoma", UCEC: "Uterine Corpus Endometrial Carcinoma", UCS: "Uterine Carcinosarcoma", UVM: "Uveal Melanoma" }; }35:[] 30:{"id":"c8d16ab004b9e040","slug":"web-ai-digital-pathology","title":"ImageBox3 + CTransPath + Weakly Supervised Attention Model for on-device Cancer Type Classification","thumbnail":"8edd761cc5562d730d4f789bcd80c1dbf31b56f44ad973a4ca6465e949c1f97a","access_level":"public","create_time":"$D2024-06-12T18:06:01.643Z","update_time":"$D2024-10-10T21:23:14.099Z","view_time":null,"first_public_time":"$D2024-06-12T18:06:01.643Z","has_unpublished_edits":null,"fork":{"id":"dd6c23633d26c17c","slug":"imagebox3-in-observable","title":"ImageBox3 in Observable","owner":{"id":"b0b4412d0985303e","login":"prafulb","name":"Praful Bhawsar"}},"stars":1,"starred":false,"owner":{"type":"team","id":"b0b4412d0985303e","login":"prafulb","name":"Praful Bhawsar","avatar_url":"https://avatars-next.observableusercontent.com/765f5b661080e697f1ac35466b8e2cef535a03897b0dc7163c9b13484530ac2a","tier":"starter_2024"},"version":1044,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"49bdf3232a315bf5","login":"prafulb","name":"Praful Bhawsar","avatar_url":"https://avatars-next.observableusercontent.com/765f5b661080e697f1ac35466b8e2cef535a03897b0dc7163c9b13484530ac2a","tier":null}],"members":[{"type":"individual","id":"49bdf3232a315bf5","login":"prafulb","name":"Praful Bhawsar","avatar_url":"https://avatars-next.observableusercontent.com/765f5b661080e697f1ac35466b8e2cef535a03897b0dc7163c9b13484530ac2a","tier":null,"role":"editor"},{"type":"team","id":"b0b4412d0985303e","login":"prafulb","name":"Praful Bhawsar","avatar_url":"https://avatars-next.observableusercontent.com/765f5b661080e697f1ac35466b8e2cef535a03897b0dc7163c9b13484530ac2a","tier":"starter_2024","role":"viewer"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"ImageBox3 + CTransPath + Weakly Supervised Attention Model for on-device Cancer Type Classification","cells":[{"id":30,"value":"# ImageBox3 + CTransPath + Weakly Supervised Attention Model for on-device Cancer Type Classification","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":265,"value":"viewof imageURL = Inputs.text({\n label: html`URL to WSI: `,\n value:\n \"https://storage.googleapis.com/imagebox_test/openslide-testdata/Aperio/CMU-1.svs\",\n width: 700,\n submit: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":812,"value":"OR","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":810,"value":"viewof imageLocal = Inputs.file({\n label: html`Choose Local File: `,\n accept: \".svs,.tiff,.tif,.ndpi\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":297,"value":"viewof tileParams = Object.assign(\n Inputs.form({\n numTiles: Inputs.range([1, 100], {\n step: 1,\n value: 10,\n label: html`No. of patches to sample:`\n })\n }),\n {\n oninput: (event) => event.isTrusted && event.stopImmediatePropagation(),\n onchange: (event) => event.currentTarget.dispatchEvent(new Event(\"input\"))\n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":889,"value":"### Sampled Patches:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":855,"value":"tilePreviews = {\n const parentDiv = document.createElement(\"div\");\n parentDiv.style.display = \"grid\";\n parentDiv.style.gridTemplateColumns = \"repeat(5, 224px)\";\n parentDiv.style.gridColumnGap = \"0.5rem\";\n parentDiv.style.gridRowGap = \"0.5rem\";\n\n Array(tileParams.numTiles)\n .fill(undefined)\n .forEach(() => {\n const img = document.createElement(\"img\");\n img.style.width = `${imageSizeForEncoder}px`;\n img.style.height = `${imageSizeForEncoder}px`;\n img.style.border = \"1px solid gray\";\n parentDiv.appendChild(img);\n });\n return parentDiv;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":995,"value":"The cancer type predicted for these sampled tiles is: **${predictedClass}**","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":376,"value":"html`
`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":566,"value":"mutable imageSource = {\n let imageSrc = undefined;\n if (imageURL && imageSrc !== imageURL) {\n viewof imageLocal.value = \"\";\n imageSrc = imageURL;\n }\n if (imageLocal) {\n viewof imageURL.value = \"\";\n imageSrc = await imageLocal.blob();\n }\n return imageSrc;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":342,"value":"imageInfo = {\n const info = await imagebox3Instance.getInfo(imageURL);\n // viewof tileParams.tileWidth.value = info * tileSizeForClassifier;\n // viewof tileParams.tileHeight.value = info * tileSizeForClassifier;\n return info;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":306,"value":"tileURLs = await getImageTiles(imageSource, tileParams)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":895,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":925,"value":"predictedClass = {\n const { logits } = classifierOutput\n const highestLogitIndex = logits.cpuData.reduce(\n (maxIndex, current, index) => {\n if (current > logits.cpuData[maxIndex]) {\n maxIndex = index;\n }\n return maxIndex;\n },\n 0\n );\n\n return Object.values(classesMap)[highestLogitIndex];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1035,"value":"classifierOutput = await classifierSession.run({ input: embeddings })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":838,"value":"tileWidth = Math.floor(imageInfo.pixelsPerMicron * 128)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":843,"value":"tileHeight = tileWidth","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":37,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"Imagebox3 = (\n await import(\"https://episphere.github.io/imagebox3/imagebox3.mjs\")\n).default","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":532,"value":"imagebox3Instance = {\n const instance = new Imagebox3(imageSource);\n await instance.init();\n return instance;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":643,"value":"onnxRuntime = import(\n \"https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/esm/ort.min.js\"\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":647,"value":"encoderSession = {\n onnxRuntime.env.wasm.wasmPaths =\n \"https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/\";\n return onnxRuntime.InferenceSession.create(\n \"https://huggingface.co/kaczmarj/CTransPath/resolve/main/model.onnx\"\n );\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":920,"value":"classifierSession = {\n onnxRuntime.env.wasm.wasmPaths =\n \"https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/\";\n return onnxRuntime.InferenceSession.create(\n \"https://huggingface.co/kaczmarj/pancancer-tissue-classifier.tcga/resolve/main/model.onnx\"\n );\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":649,"value":"function imageTransforms(\n imageTensor,\n mean = [0.485, 0.456, 0.406],\n std = [0.229, 0.224, 0.225]\n) {\n const maxPixelValue = imageTensor.reduce((max, curr) =>\n max <= curr ? curr : max\n );\n const minPixelValue = imageTensor.reduce((min, curr) =>\n min >= curr ? curr : min\n );\n const minMaxNormalizedTensor = imageTensor.map(\n (v) => (v - minPixelValue) / (maxPixelValue - minPixelValue)\n );\n\n const normalizeImage = (image, mean, std) => {\n const normalizedImage = image.map(\n (value, index) => (value - mean[index % 3]) / std[index % 3]\n );\n return normalizedImage;\n };\n\n const normalizedImage = normalizeImage(minMaxNormalizedTensor, mean, std);\n return normalizedImage;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":690,"value":"tileSizeForEncoder = 128 * 10**-6","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":825,"value":"imageSizeForEncoder = 224","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":818,"value":"maxConcurrentRequests = imageSource === imageURL ? 5 : 20","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":872,"value":"tileXSamplingRange = [27000, 33500]\n// tileXSamplingRange = [30000, 70000]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":874,"value":"tileYSamplingRange = [2000, 15500]\n// tileYSamplingRange = [20000, 80000]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":941,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1"}} 31:"$30:body"