2d:null 2e:["$","title",null,{"children":"Tool: Extract Text from a PDF (Automatic) | Tanner Hobson | Observable"}] 33:T81a,# Tool: Extract Text from a PDF (Automatic) I wanted to create a notebook for extracting text from a PDF file, especially a PDF file that is a 2-column academic paper. --- ## Demo Select a file to process. _This file will not be uploaded and will only be processed locally in the browser._ ${viewof FILE} The following text elements have been determined to be repeated multiple times in the PDF. They will be removed. ${viewof SHOULD_KEEP} The resulting text: ${viewof TEXT} ${DOM.download(() => new Blob([TEXT], { type: 'text/plain' }), `${TITLE}.txt`)} --- ## Explanation This notebook uses PDF.js to extract the text contents from the PDF. The `items` in a `TextContent` object have a few properties: `${ Object .keys(TEXT_CONTENTS[0].items[0]) .join(', ') }`. We are most interested in the `str`, `hasEOL`, and `transform` properties. - `str` is the actual text for the current item. - `hasEOL` tells us if there is more text on the same _logical_ line (because there are multiple columns, it's important to clarify that this is one line within one column, and not one line across two columns). - `transform` is a 6 element array, corresponding with the arguments to `CanvasRenderingContext2D.setTransform()`. So, it is: `scale x`, `skew y`, `skew x`, `scale y`, `translate x`, `translate y`. If we simply use the `str` and `hasEOL` properties, then we can get all the text from the document, but academic papers usually have some common header and footer text. Instead, we can determine which items of text have been duplicated with the assumption that repeated elements will be on half or more of the total pages of the PDF. To identify a text item, we can use the `transform` property. With a tiny bit of rounding, this property serves as a pretty good indication that the text is an automatic, repeated item. Finally, the items can be filtered by the comparing the `transform` property with the `transform` properties of items that are duplicated, combined according to the `hasEOL` property, and then combined across each of the pages.34:T9e3,viewof SHOULD_KEEP = { const textContents = TEXT_CONTENTS; let it = textContents; it = MAP(it, ({ items }) => items); it = FLATTEN(it); it = GROUP(it, KEY); // it = Object.entries(it); it = FILTER(it, ([_key, { length }]) => length >= (textContents.length/2)|0); const dedup = []; const badKeys = new Set(); for (const item of it) { const [key, _textContents] = item; dedup.push(item); badKeys.add(key); } dedup.sort(([_aKey, { length: a }], [_bKey, { length: b }]) => b - a); // return dedup; return Object.defineProperties(VIEW(), { value: { get: () => SHOULD_KEEP, }, }); function SHOULD_KEEP(textContentItem) { const key = KEY(textContentItem); if (badKeys.has(key)) { return false; } return true; } function VIEW() { return htl.html`

Found ${dedup.length} categories (${dedup.map(([_transform, { length }]) => length).reduce((prev, length) => prev + length, 0)} items) to remove

${Plot.line(dedup, { x: (_d, i) => i, y: ([_key, { length }]) => length }).plot()} ${dedup.map(([transform, textContents]) => htl.html.fragment`

Offending transform: ${transform} (${textContents.length} items)

${JSON.stringify(str)} `/* htl.html.fragment */)} `/* htl.html.fragment */)} `/* htl.html */; } function KEY({ transform: [a, b, c, d, e, f] }) { return `${a.toFixed(2)},${b.toFixed(2)},${c.toFixed(2)},${d.toFixed(2)},${e.toFixed(2)},${f.toFixed(2)}`; } function *MAP(iter, func=IDENTITY) { for (const item of iter) { yield func(item); } } function *FILTER(iter, condFunc=IDENTITY) { for (const item of iter) { if (condFunc(item)) { yield item; } } } function *FLATTEN(iter) { for (const item of iter) { if (Symbol.iterator in item) { yield *FLATTEN(item); } else { yield item; } } } function GROUP(iter, keyFunc=IDENTITY) { const groups = new Map(); for (const item of iter) { const key = keyFunc(item); if (!groups.has(key)) { groups.set(key, [item]); } else { groups.get(key).push(item); } } return groups; } function IDENTITY(x) { return x; } }35:[{"id":"8936e6460d79ad49a91c22e14af62822e156d223d5db6a29066261c185dc7049744c00fc171175b147f5516636fd4352f4b235f5d2c8c9a7f9f36499946d6afa","href":"https://static.observableusercontent.com/files/8936e6460d79ad49a91c22e14af62822e156d223d5db6a29066261c185dc7049744c00fc171175b147f5516636fd4352f4b235f5d2c8c9a7f9f36499946d6afa","name":"09462351-tvcg-preprint.pdf","type":"application/pdf","lastModified":1686234735526.013,"size":12313077},{"id":"5aa7e331fca100cea75cba25c7162710ee3ce31518126ed90f62a9f40e89f4e7bdc65ed1967fffee310d2ec682c61778485a1c60fd53702c50c15a7c3c6754b1","href":"https://static.observableusercontent.com/files/5aa7e331fca100cea75cba25c7162710ee3ce31518126ed90f62a9f40e89f4e7bdc65ed1967fffee310d2ec682c61778485a1c60fd53702c50c15a7c3c6754b1","name":"pdf.js","type":"application/javascript","lastModified":1686234735526.013,"size":515603},{"id":"eff45ac5935d35b0f1c47eddfc440037f9a524380a29c11274bdc267f17df37ffb572b9a74cf1dc7ef1176d2a447e788320477188824a889cdf17266149bf18e","href":"https://static.observableusercontent.com/files/eff45ac5935d35b0f1c47eddfc440037f9a524380a29c11274bdc267f17df37ffb572b9a74cf1dc7ef1176d2a447e788320477188824a889cdf17266149bf18e","name":"pdf.worker.js","type":"application/javascript","lastModified":1686234735526.013,"size":1966101}] 31:{"id":"aaba0997c3b6a615","slug":"tool-extract-text-from-a-pdf-automatic","title":"Tool: Extract Text from a PDF (Automatic)","thumbnail":"e740163959ce45c5e080c21273bee1433d9d0cd746c8d8cee44f833e57b79139","access_level":"public","create_time":"$D2023-06-08T14:32:15.526Z","update_time":"$D2023-09-28T12:34:37.582Z","view_time":null,"first_public_time":"$D2023-06-08T14:32:15.526Z","fork":{"id":"abdf47bbc11ee18b","slug":"extract-text-from-a-pdf","title":"Tool: Extract Text from a PDF (Manual)","owner":{"id":"7343f790779e6735","login":"player1537","name":"Tanner Hobson"}},"stars":1,"starred":false,"owner":{"type":"team","id":"7343f790779e6735","login":"player1537","name":"Tanner Hobson","avatar_url":"https://avatars-next.observableusercontent.com/16170a1a32e364e1ff6054c6045de6814a032d01331aba01288f82afceeaaec2","tier":"starter_2024"},"version":231,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"e0a627f6b6f3beb0","login":"player1537","name":"Tanner Hobson","avatar_url":"https://avatars-next.observableusercontent.com/16170a1a32e364e1ff6054c6045de6814a032d01331aba01288f82afceeaaec2","tier":null}],"members":[{"type":"individual","id":"e0a627f6b6f3beb0","login":"player1537","name":"Tanner Hobson","avatar_url":"https://avatars-next.observableusercontent.com/16170a1a32e364e1ff6054c6045de6814a032d01331aba01288f82afceeaaec2","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":{"id":"cded6594ed063fa4","slug":"usable-tools","title":"Usable Tools","owner":{"id":"7343f790779e6735","login":"player1537","name":"Tanner Hobson"}},"type":"notebook","body":{"title":"Tool: Extract Text from a PDF (Automatic)","cells":[{"id":0,"value":"$33","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":6,"value":"viewof FILE = Inputs.file()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"PDF = {\n let file = FILE;\n \n if (file === null) {\n file = await FileAttachment('09462351-tvcg-preprint.pdf');\n }\n\n return await pdfjsLib.getDocument(await file.arrayBuffer()).promise;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":171,"value":"TITLE = {\n return FILE?.name?.replace('.pdf', '');\n try {\n const metadata = await PDF.getMetadata();\n return metadata.info.Title;\n } catch (e) {\n return 'ExtractedText';\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":106,"value":"PAGES = {\n const pdf = PDF;\n \n const pages = [];\n for (let i=1, n=pdf.numPages; i<=n; ++i) {\n const page = await pdf.getPage(i);\n pages.push(page);\n }\n\n return pages;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":109,"value":"TEXT_CONTENTS = {\n const pages = PAGES;\n \n const textContents = [];\n for (let i=0, n=pages.length; i