2c:null 2d:["$","title",null,{"children":"PDF Crush | Israel Gonzalez-Brooks | Observable"}] 32:T781,sentenceWiseNormalizedWeights = { const openClassPOS = Object.create(null); openClassPOS.ADJ = true; openClassPOS.ADV = true; openClassPOS.INTJ = true; openClassPOS.NOUN = true; openClassPOS.PROPN = true; openClassPOS.VERB = true; openClassPOS.NUM = true; openClassPOS.SYM = true; const NGram = 4; const posGroupWeightTable = Object.create(null); text.sentences().each((s) => { const pos = s .tokens() .out(wp.its.pos) .filter((p) => p !== "SPACE" && p !== "PUNCT"); if (pos.length < 4) return; for (let k = 0; k + NGram - 1 < pos.length; k += 1) { const pos4Gram = pos.slice(k, k + NGram); const initInfoContent = pos4Gram.reduce( (pv, cv) => pv - (openClassPOS[cv] ? 0 : 1), 0 ); const posGroup = pos4Gram.join("_"); posGroupWeightTable[posGroup] = posGroupWeightTable[posGroup] || Object.create(null); posGroupWeightTable[posGroup].group = posGroup; posGroupWeightTable[posGroup].sentences = posGroupWeightTable[posGroup].sentences || []; posGroupWeightTable[posGroup].sentences.push(s.index()); posGroupWeightTable[posGroup].weight = posGroupWeightTable[posGroup].weight === undefined ? initInfoContent + 1 : posGroupWeightTable[posGroup].weight + 1; posGroupWeightTable[posGroup].iv = initInfoContent; } }); const posGroupWeights = Object.keys(posGroupWeightTable) .map((e) => posGroupWeightTable[e]) .filter((e) => e.weight > 0); let sentenceWiseWeights = new Array(text.sentences().length()); sentenceWiseWeights.fill(0); posGroupWeights.forEach((pgw) => { pgw.sentences.forEach((e) => { sentenceWiseWeights[e] += pgw.weight; }); }); sentenceWiseWeights = sentenceWiseWeights.map((e) => e / 2); const max = Math.max(...sentenceWiseWeights); return sentenceWiseWeights.map((e) => e / max); }33:[{"id":"b17b7d86a3b365a7459ce13917d93889a307f5688a5938de43e7d846a4798f731d9bae2e8e2c8bab668db5b24b2fe3b06987d869793e1a5c2edf54a1e2241dfe","href":"https://static.observableusercontent.com/files/b17b7d86a3b365a7459ce13917d93889a307f5688a5938de43e7d846a4798f731d9bae2e8e2c8bab668db5b24b2fe3b06987d869793e1a5c2edf54a1e2241dfe","name":"Generalist Agent.pdf","type":"application/pdf","lastModified":1669941839276.092,"size":7359418}] 30:{"id":"a67f15787098349e","slug":null,"title":"PDF Crush","thumbnail":"5ef034d4fbec05cb27032fd7c464ae3866de63cd9e23f320af8703072bf61d45","access_level":"public","create_time":"$D2022-12-02T00:43:59.276Z","update_time":"$D2023-02-10T19:34:22.541Z","view_time":null,"first_public_time":"$D2022-12-02T00:43:59.276Z","has_unpublished_edits":null,"fork":true,"stars":0,"starred":false,"owner":{"type":"team","id":"b6cc08c484e885c2","login":"iz","name":"Israel Gonzalez-Brooks","avatar_url":"https://avatars-next.observableusercontent.com/b18cd83a73bc56f94cb9d184956c2a4fb3e685f9b74d25627832cf49d44a16c9","tier":"starter_2024"},"version":4650,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"3edb53fdfab76653","login":"iz","name":"Israel Gonzalez-Brooks","avatar_url":"https://avatars-next.observableusercontent.com/b18cd83a73bc56f94cb9d184956c2a4fb3e685f9b74d25627832cf49d44a16c9","tier":null}],"members":[{"type":"individual","id":"3edb53fdfab76653","login":"iz","name":"Israel Gonzalez-Brooks","avatar_url":"https://avatars-next.observableusercontent.com/b18cd83a73bc56f94cb9d184956c2a4fb3e685f9b74d25627832cf49d44a16c9","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"title":"PDF Crush","cells":[{"id":1109,"value":"# PDF Crush\n## v0.4 mecha","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":4279,"value":"viewof arxivUrl = Inputs.text({ label: \"arXiv link\", submit: true })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1428,"value":"viewof pageSelect = Inputs.select(\n [...new Array(pdf.numPages)].map((i, idx) => idx + 1),\n { label: \"View Page\", value: 1 }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4431,"value":"viewof renderCanvas = Inputs.toggle({ label: \"Render page as image\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1238,"value":"md`***\n${pdf.numPages} pages\\n\\n*${pdfData.meta.info.Producer}*\\n\n***`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4643,"value":"scrollAnchor = html`

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1483,"value":"view = {\n let sentences = text\n .sentences()\n .each((s, k) => s.markup(beginTag(k, s), endTag()));\n return html`

${viewof backButton}

${\n /*page.textContent.items\n .map(({ str, hasEOL, fontName }) => {\n if (!str.length) return \"

\";\n\n let out = str;\n let outLastIdx = out.length - 1;\n let lastOutChar = out[outLastIdx];\n\n if (lastOutChar === \"-\") out = [...out].slice(0, outLastIdx).join(\"\");\n\n // sentences.map((i) =>\n // out.includes(i[0]) ? out.replace(i[0], `${i[0]}`) : null\n // );\n\n if (\"!.?!\".includes(lastOutChar)) out = out + \"

\";\n\n return `${`${texDecode(out)}`}`;\n })\n .join(\" \")*/\n\n html`

${text\n .out(wp.its.markedUpText)\n .replaceAll(\"\\n\", \"
\")}

`\n }

${viewof forwardButton}

`;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4569,"value":"text.sentences().out(wp.its.markedUpText)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4521,"value":"plainText = page.textContent.items\n .map(({ str, hasEOL, fontName }) => {\n if (!str.length) return \"\\n\\n\";\n\n let out = str.trim();\n let outLastIdx = out.length - 1;\n let lastOutChar = out[outLastIdx];\n let endsInAbbrev = out[outLastIdx - 2]; // i.e. or e.g.\n\n if (lastOutChar === \"-\") out = [...out].slice(0, outLastIdx).join(\"\");\n\n if (\"!.?!\".includes(lastOutChar) && !endsInAbbrev) out = out + \"\\n\";\n\n return `${`${texDecode(out)}`}`;\n })\n .join(\" \")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3619,"value":"canvas = {\n const canvas = htl.html`

`;\n if (renderCanvas) renderPdf(canvas, pdf, pageNumber);\n return canvas;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":606,"value":"***\n### Notebook Guts 🦾","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1462,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4359,"value":"mutable pageNumber = pageSelect","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2878,"value":"texDecode = (str) => {\n let encoded = encodeURIComponent(\n String.fromCharCode(\n ...[...lib.removeNullCharacters(str)].map((s, idx) => str.charCodeAt(idx))\n )\n );\n\n if (!encoded.includes(\"%ED\")) return str;\n\n return decodeURIComponent(encoded.replaceAll(\"%ED\", \"%F0%9D\"));\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4207,"value":"page = {\n const data = await pdf.getPage(pageNumber);\n return {\n data,\n textContent: await data.getTextContent()\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4194,"value":"pdfData = {\n const outline = await pdf.getOutline();\n const TOC = outline.map((p) => [\n p.title,\n p.dest,\n p.items.map((q) => [q.title, q.dest]),\n `***\n### ${p.title}\n${p.items.map((i) => ` - ##### ${i.title}\\n`).join(\"\\n\")}`\n ]);\n\n return {\n meta: await pdf.getMetadata(),\n outline,\n TOC,\n subsections: TOC.map((section) => section).flatMap((i) => i[2])\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1054,"value":"pdf = {\n const gato = FileAttachment(\"Generalist Agent.pdf\");\n // const buffer = gato.arrayBuffer();\n const docNum = arxivUrl\n ? arxivUrl\n .split(\"/\")\n .filter((i) => 57 - i.charCodeAt() > 0)[0]\n .split(\".\")\n .filter((i) => 57 - i.charCodeAt() > 0)\n .join(\".\")\n : \"\";\n const data = docNum\n ? fetch(`https://arxiv.org/pdf/${docNum}.pdf`).then((r) => r.arrayBuffer())\n : \"\";\n return pdfReader(data || gato.arrayBuffer());\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":639,"value":"lib = {\n const pdfjs = await require(\"pdfjs-dist@2.9.359\");\n\n const parser = await import(\n \"https://deno.land/x/pdf_parser@v1.1.2/main.compress.js\"\n );\n\n pdfjs.GlobalWorkerOptions.workerSrc = `https://unpkg.com/pdfjs-dist@${pdfjs.version}/build/pdf.worker.js`;\n return { ...pdfjs, ...parser };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4162,"value":"renderPdf = (canvas, pdf, pageNum) => {\n pdf.getPage(pageNum).then((page) => {\n const canvasContext = canvas.getContext(\"2d\");\n const viewport = page.getViewport({ scale: 1 });\n\n canvas.height = viewport.height;\n canvas.width = viewport.width;\n\n page.render({ canvasContext, viewport }).then(() => {\n const img = document.createElement(\"img\");\n img.src = canvas.toDataURL();\n img.onload = () => img;\n });\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4128,"value":"pdfReader = async (arrayBuffer) => lib.loadpdfInClient(await arrayBuffer, lib)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4375,"value":"viewof backButton = Inputs.button(\"\", {\n reduce: () => {\n viewof pageSelect.value = mutable pageNumber =\n pageNumber > 1 ? pageNumber - 1 : 1;\n scrollAnchor.scrollIntoView();\n viewof pageSelect.dispatchEvent(new Event(\"change\"));\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4376,"value":"viewof forwardButton = Inputs.button(\"\", {\n reduce: () => {\n viewof pageSelect.value = mutable pageNumber =\n pageNumber < pdf.numPages ? pageNumber + 1 : pdf.numPages;\n scrollAnchor.scrollIntoView();\n viewof pageSelect.dispatchEvent(new Event(\"change\"));\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4450,"value":"wp = {\n let wp = (await import(\"https://cdn.skypack.dev/wink-nlp\")).default;\n let model = (await import(\"https://cdn.skypack.dev/wink-eng-lite-web-model\"))\n .default;\n return wp(model);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4453,"value":"text = wp.readDoc(plainText)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4584,"value":"weightedSentences = [[], [], [], []]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4462,"value":"beginTag = function (k, s) {\n const a =\n sentenceWiseNormalizedWeights[k] >\n d3.quantile(sentenceWiseNormalizedWeights, (100 - 20) / 100)\n ? sentenceWiseNormalizedWeights[k]\n : 0;\n // if (a) return ``\n if (a) {\n if (a > 0.9) {\n weightedSentences[0].push(s.out());\n return ``;\n } else if (a > 0.75) {\n weightedSentences[1].push(s.out());\n return ``;\n } else if (a > 0.5) {\n weightedSentences[2].push(s.out());\n return ``;\n } else {\n weightedSentences[3].push(s.out());\n return ``;\n }\n }\n if (focus) return ``;\n return ``;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4464,"value":"endTag = () => ``","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4467,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4477,"value":"","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":4593,"value":"***\n## Sections","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":4595,"value":"texts = {\n let results = [];\n for (let i = 1; i <= pdf.numPages; i++) {\n results.push(pdf.getPage(i));\n }\n return Promise.allSettled(results).then((r) => r.map((i) => i.value));\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4597,"value":"textContents = Promise.allSettled(texts.map((i) => i.getTextContent())).then(\n (r) => r.map((j) => j.value)\n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4599,"value":"outText = textContents\n .map((i) => i.items.map((j) => [j.str, j.hasEOL]))\n //.slice(0, 5)\n .map((k) => k.map((l) => `${l[0]}${l[1] ? \"\\n\" : \"\"}`).join(\"\"))\n//.join(\"\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4649,"value":"pdf.getMetadata()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4601,"value":"outl = {\n let outline = await pdf.getOutline();\n // Might want to run this next func recursively, eg to drill down as far as necessary?\n return outline.map(({ dest, items, title }) => {\n return {\n dest,\n title,\n items: items.map(({ dest, title, items }) => {\n return {\n dest,\n title,\n items: items.map(({ dest, items }) => {\n return { dest, title, items };\n })\n };\n })\n };\n });\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4603,"value":"destinations = outl\n .flatMap((i) => [\n [i.dest, i.title],\n i.items.flatMap((j) => [\n [j.dest, j.title],\n j.items.flatMap((k) => [[k.dest, k.title], k.items])\n ])\n ])\n .filter((l) => l.length)\n .flatMap((m) => (typeof m[0] === \"object\" ? m : [m]))\n .filter((l) => l.length)\n .flatMap((m) => (typeof m[0] === \"object\" ? m : [m]))\n .filter((l) => l.length)\n\n//.map((k) => (typeof k[0] === \"object\" ? k.flatMap((l) => l) : k))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4605,"value":"destinationDetails = Promise.allSettled(\n destinations.map(async (i) => [await pdf.getDestination(i[0]), i[1]])\n).then((r) => r.map((s) => s.value))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4607,"value":"pageIndexes = Promise.allSettled(\n destinationDetails.map(async (i) => [await pdf.getPageIndex(i[0][0]), i[1]])\n).then((r) => r.map((s) => s.value))\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4611,"value":"pageRanges = pageIndexes\n .map((i, idx, arr) => [\n [arr[idx][0], arr?.[idx + 1]?.[0] ?? null],\n arr[idx][1]\n ])\n .filter((t, tdx, arr) => t[1] !== arr?.[tdx - 1]?.[1]) // This filter removes duplicates, make sure to check if weirdness happens","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4613,"value":"textByRange = pageRanges.map((i, idx, arr) => {\n // return i;\n let pageJoin = outText\n .slice(i[0][0], i[0][1] ? i[0][1] + 1 : i[0][0] + pdf.numPages)\n .join(\"\");\n let slicer = [\n pageJoin.indexOf(i[1]),\n arr?.[idx + 1] ? pageJoin.indexOf(arr[idx + 1][1]) : pageJoin.length\n ];\n return pageJoin.slice(...slicer);\n // return [\n // i[1],\n // i[1].split(\" \").slice(1),\n // i[1].split(\" \").slice(1).join(\" \"),\n // pageJoin.indexOf(i[1].split(\" \").slice(1).join(\" \")),\n // arr?.[idx + 1]\n // ? pageJoin.indexOf(arr[idx + 1][1].split(\" \").slice(1).join(\" \"))\n // : pageJoin.length,\n // arr?.[idx + 1],\n // pageJoin.length\n // ];\n return pageJoin.slice(\n pageJoin.indexOf(i[1].split(\" \").slice(1).join(\" \")),\n arr?.[idx + 1]\n ? pageJoin.indexOf(arr[idx + 1][1].split(\" \").slice(1).join(\" \"))\n : pageJoin.length\n );\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4615,"value":"md`${textByRange[23]}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}} 31:"$30:body"