2b:null 2c:["$","title",null,{"children":"Playground: Convert PDF to text, keeping each page separate | Tanner Hobson | Observable"}] 31:T9e3,viewof SHOULD_KEEP = { const textContents = TEXT_CONTENTS; let it = textContents; it = MAP(it, ({ items }) => items); it = FLATTEN(it); it = GROUP(it, KEY); // it = Object.entries(it); it = FILTER(it, ([_key, { length }]) => length >= (textContents.length/2)|0); const dedup = []; const badKeys = new Set(); for (const item of it) { const [key, _textContents] = item; dedup.push(item); badKeys.add(key); } dedup.sort(([_aKey, { length: a }], [_bKey, { length: b }]) => b - a); // return dedup; return Object.defineProperties(VIEW(), { value: { get: () => SHOULD_KEEP, }, }); function SHOULD_KEEP(textContentItem) { const key = KEY(textContentItem); if (badKeys.has(key)) { return false; } return true; } function VIEW() { return htl.html`

Found ${dedup.length} categories (${dedup.map(([_transform, { length }]) => length).reduce((prev, length) => prev + length, 0)} items) to remove

${Plot.line(dedup, { x: (_d, i) => i, y: ([_key, { length }]) => length }).plot()} ${dedup.map(([transform, textContents]) => htl.html.fragment`

Offending transform: ${transform} (${textContents.length} items)

${JSON.stringify(str)} `/* htl.html.fragment */)} `/* htl.html.fragment */)} `/* htl.html */; } function KEY({ transform: [a, b, c, d, e, f] }) { return `${a.toFixed(2)},${b.toFixed(2)},${c.toFixed(2)},${d.toFixed(2)},${e.toFixed(2)},${f.toFixed(2)}`; } function *MAP(iter, func=IDENTITY) { for (const item of iter) { yield func(item); } } function *FILTER(iter, condFunc=IDENTITY) { for (const item of iter) { if (condFunc(item)) { yield item; } } } function *FLATTEN(iter) { for (const item of iter) { if (Symbol.iterator in item) { yield *FLATTEN(item); } else { yield item; } } } function GROUP(iter, keyFunc=IDENTITY) { const groups = new Map(); for (const item of iter) { const key = keyFunc(item); if (!groups.has(key)) { groups.set(key, [item]); } else { groups.get(key).push(item); } } return groups; } function IDENTITY(x) { return x; } }32:[{"id":"8936e6460d79ad49a91c22e14af62822e156d223d5db6a29066261c185dc7049744c00fc171175b147f5516636fd4352f4b235f5d2c8c9a7f9f36499946d6afa","href":"https://static.observableusercontent.com/files/8936e6460d79ad49a91c22e14af62822e156d223d5db6a29066261c185dc7049744c00fc171175b147f5516636fd4352f4b235f5d2c8c9a7f9f36499946d6afa","name":"09462351-tvcg-preprint.pdf","type":"application/pdf","lastModified":1686702489003.291,"size":12313077},{"id":"5aa7e331fca100cea75cba25c7162710ee3ce31518126ed90f62a9f40e89f4e7bdc65ed1967fffee310d2ec682c61778485a1c60fd53702c50c15a7c3c6754b1","href":"https://static.observableusercontent.com/files/5aa7e331fca100cea75cba25c7162710ee3ce31518126ed90f62a9f40e89f4e7bdc65ed1967fffee310d2ec682c61778485a1c60fd53702c50c15a7c3c6754b1","name":"pdf.js","type":"application/javascript","lastModified":1686702489003.291,"size":515603},{"id":"eff45ac5935d35b0f1c47eddfc440037f9a524380a29c11274bdc267f17df37ffb572b9a74cf1dc7ef1176d2a447e788320477188824a889cdf17266149bf18e","href":"https://static.observableusercontent.com/files/eff45ac5935d35b0f1c47eddfc440037f9a524380a29c11274bdc267f17df37ffb572b9a74cf1dc7ef1176d2a447e788320477188824a889cdf17266149bf18e","name":"pdf.worker.js","type":"application/javascript","lastModified":1686702489003.291,"size":1966101}] 2f:{"id":"59c7320840549500","slug":null,"title":"Playground: Convert PDF to text, keeping each page separate","thumbnail":"0188df37245cf4ebfcf03a022ec67ac0faabaa5848f4cf6031f546bd6f961569","access_level":"public","create_time":"$D2023-06-14T00:28:09.003Z","update_time":"$D2023-06-14T00:28:36.557Z","view_time":null,"first_public_time":"$D2023-06-14T00:28:09.003Z","fork":{"id":"aaba0997c3b6a615","slug":"tool-extract-text-from-a-pdf-automatic","title":"Tool: Extract Text from a PDF (Automatic)","owner":{"id":"7343f790779e6735","login":"player1537","name":"Tanner Hobson"}},"stars":1,"starred":false,"owner":{"type":"team","id":"7343f790779e6735","login":"player1537","name":"Tanner Hobson","avatar_url":"https://avatars-next.observableusercontent.com/16170a1a32e364e1ff6054c6045de6814a032d01331aba01288f82afceeaaec2","tier":"starter_2024"},"version":244,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"e0a627f6b6f3beb0","login":"player1537","name":"Tanner Hobson","avatar_url":"https://avatars-next.observableusercontent.com/16170a1a32e364e1ff6054c6045de6814a032d01331aba01288f82afceeaaec2","tier":null}],"members":[{"type":"individual","id":"e0a627f6b6f3beb0","login":"player1537","name":"Tanner Hobson","avatar_url":"https://avatars-next.observableusercontent.com/16170a1a32e364e1ff6054c6045de6814a032d01331aba01288f82afceeaaec2","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"title":"Playground: Convert PDF to text, keeping each page separate","cells":[{"id":242,"value":"# Playground: Convert PDF to text, keeping each page separate","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":6,"value":"viewof FILE = Inputs.file()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"PDF = {\n let file = FILE;\n \n if (file === null) {\n file = await FileAttachment('09462351-tvcg-preprint.pdf');\n }\n\n return await pdfjsLib.getDocument(await file.arrayBuffer()).promise;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":171,"value":"TITLE = {\n const name = FILE?.name?.replace('.pdf', '');\n if (name !== undefined) {\n return name;\n }\n \n try {\n const metadata = await PDF.getMetadata();\n return metadata.info.Title;\n } catch (e) {\n return 'ExtractedText';\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":106,"value":"PAGES = {\n const pdf = PDF;\n \n const pages = [];\n for (let i=1, n=pdf.numPages; i<=n; ++i) {\n const page = await pdf.getPage(i);\n pages.push(page);\n }\n\n return pages;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":109,"value":"TEXT_CONTENTS = {\n const pages = PAGES;\n \n const textContents = [];\n for (let i=0, n=pages.length; i