45:["$","title",null,{"children":"Visualizing PDFs with Sankey Diagrams | PDF | Observable"}] 4d:T655,# Visualizing PDFs with Sankey Diagrams [Sankey diagrams](https://en.wikipedia.org/wiki/Sankey_diagram) date back to 1898 and the era of steam engines when they were first used to explain energy efficiency. They are a type of flow diagram where the width of arrows represent a proportion of a total and allow visualization and understanding of contributions to various subtotals. They are still in use today for visualizing energy flows, as shown on these ObservableHQ [Sankey Diagram](https://observablehq.com/@d3/sankey) and [Flow-O-Matic](https://observablehq.com/@mbostock/flow-o-matic) pages. As part of my SafeDocs research work, I recently discovered that Sankey diagrams are also extremely valuable in visualizing a file’s physical and logical layout. The Sankey diagram **nodes** (the vertical bars) represent bytes in a file, while the **flows** show how bytes map to a file formats' different logical/semantic constructs. This work seems highly applicable to both PDF and ICC formats studied under SafeDocs, and is far more expressive and accessible than color-coded hex dumps. **_To my knowledge, this is the first application of Sankey diagrams to digital file forensics!_** This is just the start! I have [lots more ideas](https://observablehq.com/@pdf/visualizing-pdfs-with-sankey-diagrams#cell-903) limited only by my limited D3 capabilities... ## Conventional PDF example The CSV data for the conventional PDF shown here lower down this page and can be updated. The following controls will also alter the overall appearance (the two most appropriate alignments appear to be "left" and justify"):4e:T850,chart = { const svg = d3.select(DOM.svg(width, height)) .style("background", "#fff") .style("width", "100%") .style("height", "auto"); const {nodes, links} = sankey({ nodes: data.nodes.map(d => Object.assign({}, d)), links: data.links.map(d => Object.assign({}, d)) }); svg.append("g") .selectAll("rect") .data(nodes) .join("rect") .attr("x", d => d.x0 + 1) .attr("y", d => d.y0) .attr("height", d => d.y1 - d.y0) .attr("width", d => d.x1 - d.x0 - 2) .attr("fill", d => { let c; for (const link of d.sourceLinks) { if (c === undefined) c = link.color; else if (c !== link.color) c = null; } if (c === undefined) { for (const link of d.targetLinks) { if (c === undefined) c = link.color; else if (c !== link.color) c = null; } } return (d3.color(c) || d3.color(defcolor)).darker(0.5); }) .append("title") .text(d => `${d.name}\n${d.value.toLocaleString()}`); const link = svg.append("g") .attr("fill", "none") .selectAll("g") .data(links) .join("g") .attr("stroke", d => d3.color(d.color) || defcolor) .style("mix-blend-mode", "multiply"); link.append("path") .attr("d", d3.sankeyLinkHorizontal()) .attr("stroke-width", d => Math.max(1, d.width)); link.append("title") .text(d => `${d.source.name} → ${d.target.name}\n${d.value.toLocaleString()}`); svg.append("g") .style("font", "10px sans-serif") .selectAll("text") .data(nodes) .join("text") .attr("x", d => d.x0 < width / 2 ? d.x1 + 6 : d.x0 - 6) .attr("y", d => (d.y1 + d.y0) / 2) .attr("dy", "0.35em") .attr("text-anchor", d => d.x0 < width / 2 ? "start" : "end") .text(d => d.name) .append("tspan") .attr("fill-opacity", 0.7) .text(d => ` ${d.value.toLocaleString()}`); return svg.node(); }4f:Tc8b,viewof source = { const textarea = html`

Preamble,Cavity,16,red
"PDF file",Header,16,
"PDF file","Object 1",20,
"PDF file","Object 2",22,
"PDF file","Object 3",42,
"PDF file",Cavity,6,red
"PDF file","Object 4",15,
"PDF file","Object 6",3494,
"PDF file","Object 7",441,
"PDF file","Object 5",30,
"PDF file","Object 9",30,
"PDF file","Object 8",404,
"PDF file","Object 10",104,
"PDF file","Object 11",40,
"PDF file","Object 16",4050,
"PDF file","Object 15",4041,
"PDF file","Object 14",30,
"PDF file","Object 13",10,
"PDF file","Object 12",5,
"PDF file",xref,248,lightgreen
"PDF file",trailer,28,lightblue
"PDF file",EOF,6,blue
"Incremental Update 1","Object 4",27,
"Incremental Update 1","Object 17",485,
"Incremental Update 1","Object 18",72,
"Incremental Update 1","Object 19",46,
"Incremental Update 1","Object 22",120,
"Incremental Update 1","Object 23",4305,
"Incremental Update 1",xref,128,lightgreen
"Incremental Update 1",trailer,31,lightblue
"Incremental Update 1",EOF,6,blue
Gap,Cavity,12,red
"Incremental Update 2","Object 4",22,
"Incremental Update 2","Object 24",59,
"Incremental Update 2","Object 20",283,
"Incremental Update 2","Object 21",42,
"Incremental Update 2","Object 5",10,
"Incremental Update 2","Object 9",5,
"Incremental Update 2",xref,128,lightgreen
"Incremental Update 2",trailer,32,lightblue
"Incremental Update 2",EOF,6,blue
Postamble,Cavity,12,red
"Object 1",Document,20,
"Object 2",Document,22,
"Object 3",Document,42,
"Object 4","Page 1",27,Linen
"Object 5","Page 1",30,Linen
"Object 6","Page 1",3494,Linen
"Object 7","Page 1",441,Linen
"Object 8","Page 1",404,Linen
"Object 9",Document,30,
"Object 10",Document,104,
"Object 11","Page 2",40,LavenderBlush
"Object 12","Page 2",5,LavenderBlush
"Object 13","Page 2",10,LavenderBlush
"Object 14","Page 2",30,LavenderBlush
"Object 15","Page 2",4041,LavenderBlush
"Object 16","Page 2",4050,LavenderBlush
"Object 17","Page 1",485,Linen
"Object 18","Page 1",72,Linen
"Object 19","Page 1",46,Linen
"Object 20","Page 1",283,Linen
"Object 21","Page 1",42,Linen
"Object 22","Page 2",120,LavenderBlush
"Object 23","Page 2",4305,LavenderBlush
"Object 24","Page 2",59,LavenderBlush
"Page 1",Content,426,PaleGoldenRod
"Page 1",Fonts,1331,LightBlue
"Page 1",Colors,639,LightCyan
"Page 1",Images,1171,MistyRose
"Page 1",Annotations,0,Wheat
"Page 1",Tagged PDF,1065,PeachPuff
"Page 1",Other,692,OldLace
"Page 2",Content,1772,PaleGoldenRod
"Page 2",Fonts,2912,LightBlue
"Page 2",Colors,1772,LightCyan
"Page 2",Images,0,MistyRose
"Page 2",Annotations,772,Wheat
"Page 2",Tagged PDF,1899,PeachPuff
"Page 2",Other,3533,OldLace
`;
  textarea.style.display = "block";
  textarea.style.boxSizing = "border-box";
  textarea.style.width = "calc(100% + 28px)";
  textarea.style.font = "var(--monospace-font, var(--mono_fonts))";
  textarea.style.minHeight = "60px";
  textarea.style.border = "none";
  textarea.style.padding = "4px 10px";
  textarea.style.margin = "0 -14px";
  textarea.style.background = "rgb(247,247,249)";
  textarea.style.tabSize = 2;
  textarea.oninput = () => {
    textarea.style.height = "initial";
    textarea.style.height = `${textarea.scrollHeight}px`;
  };
  yield textarea;
  textarea.oninput();
}50:Tce4,## Understanding PDF Sankey diagrams

In applying Sankey diagrams to a file format such as PDF, the proportions or values of each flow are the number of bytes of a file that contribute to a different logical or semantic visualizations. The figure above visualizes a small conventional PDF (i.e. no cross-reference streams and no object streams) with 2 incremental updates, comprising 2 pages. Viewing from left to right:

* The left side represents the **physical file layout**, with the original PDF and two incremental updates as well as any “junk” bytes that may exist before the PDF header (`%PDF-x.y`) or after the last `%%EOF` end-of-PDF file marker. For SafeDocs we have also been investigating what we refer to as “cavities” within PDF files that can enable malicious behavior or polyglots. The values are the number of bytes contributing to each of the physical file sections;
    * the flows visualize the number of raw bytes in the input file
    * the flows visualize which objects each incremental update modifies
    * cavities (blind spots), file "magic"/scaffolding bytes are explicitly labelled

* The next vertical representation shows how the bytes in the physical file map to **PDF objects**. Note that when an incremental update updates an object, the Sankey diagram clearly visualizes this relationship. Again, the values are the number of bytes contributing to each PDF object. In the case of objects that are redefined by an incremental update, we have chosen to display the largest object across all object generations.
    * the ordering of the objects in the physical file is retained (since PDF object numbering is arbitrary - they are merely identifiers for random access purposes) 
    * currently this shows compressed object size (i.e. matching the physical file)

* The next vertical representation shows **how objects contribute to pages**. For objects that are not directly related to pages (e.g. Document Catalog, Outlines, metadata, intermediate page tree nodes, etc) these are shown on the right-most representation.
    * the flows clearly show which objects contribute to which pages, including objects added or modified by incremental updates

* Finally, the right most representation categorizes the objects into groupings to provide an overall visualization of bytes in the file that contribute to the different logical or semantic parts of a PDF file.
    * this was chosen simply because it is a common feature in some PDF viewers and many PDF users want to know what "makes up their PDF file". In Adobe Acrobat Pro DC, see the "Audit space usage..." feature under `File` | `Save as Other...` | `Optimized PDF...`.
    * See [below](https://observablehq.com/@pdf/visualizing-pdfs-with-sankey-diagrams#cell-903) for other ideas

The three flows across these four nodes provide a nice simple, yet understandable, visualization of both the magnitude and locality of bytes in the file and how the PDF is composed. See below for further discussion on how more complex PDFs ([cross reference streams and object streams](https://observablehq.com/@pdf/visualizing-pdfs-with-sankey-diagrams#cell-858)) would be handled, and for [future applications and ideas for improvement](https://observablehq.com/@pdf/visualizing-pdfs-with-sankey-diagrams#cell-903).51:T9b8,These visualizations are created by the [d3.sankey JavaScript library](https://github.com/d3/d3-sankey) which can accept CSV data defining each **flow**. Each line of CSV data defines a `source`, `target` as a string (double quote if necessary), and a numeric `value` followed by an optional HTML color name. The `source` represents the left side of a flow, the `target` the right side, and the `value` represents the size (width) of the flow.

For the conventional PDF file shown above, because we have 3 “sets” of flows, we have 3 "sets" of CSV rows in the CSV data:

1. The physical file layout, ordered from start of the physical file to end of the physical file, where the value is the length of each part. The `source`s are _Preamble_, _PDF file_, _Incremental Update X_, _Postamble_, and _Gap X_. The `target` is _header_, an object number, _xref_, _trailer_, _Cavity_, etc. Whether cavities (gaps) are explicitly listed, or the level of granularity of parts in the physical file is up to you, but the sum of all values in this set should be done such that they equal the length of the physical file to ensure correctness. Thus every byte in the physical file is then clearly accounted for.

2. The object layout of all PDF body sections ordered by the starting offset in the physical file, where the `source` is the list of targets from the 1st set above, and the `target` is either _Document_ or _Page X_. The `value` for each object is the size (length in bytes) of the corresponding object. Objects that are modified by an incremental update can have different sizes - you may choose to select the last generation object or the largest object across all generations, depending on what you want the visualization to represent.
The sum of values in this set will obviously be less than that in the first set, and for objects for which there is only a single generation the value will be the same in set 1 and set 2.

3. The third set of rows in the CSV file are for the categories of objects and associated pages. The `source` is always _Page X_ and the `target`  are groupings relevant for you (e.g. _Content_, _Font_, _Colors_, _Images_, _Annotations_, etc. in our case). The `value` is simply the sum of the relevant object sizes from the second set.

The choice of colors is completely arbitrary and we use it to aid understanding of the visualization and to accentuate what is important for our analysis. You can edit this CSV data and see the changes reflected above.52:T4c1,## Future Ideas and Application

Brainstormed ideas with Galois for the future (_and especially for someone with more skills in D3!_):

* **interactivity** - click on an object and highlight where it is in the PDF file (original PDF and each incremental update where it is altered) as well as how it contributes to pages and object groupings. Especially useful as many PDF objects are very small so the flows are very fine and difficult to see.

* **polyglots** - could show an alternate file format to the _left_ of the physical file to show how the same set of bytes contributes to a secondary format.

* **visual contributions** - not only show file bytes but also the "visual contribution" objects have to a page. For example this might be the area the object covers or the number of pixels marked on a page. So a physically large image that is dramatically scaled down would show a low visual contribution (or vice-versa)

* **sub-formats** - expand with additional vertical nodes to show decompression or contained (nested) formats that have sub-structure. This is probably better left to tools like [PolyFile](https://github.com/trailofbits/polyfile)/[PolyTracker](https://github.com/trailofbits/polytracker).53:[{"id":"118cdc6266aad98eab76298112f953d92cf36a79e9aa8686b4a994703d35e00067b5c038c216a7f6a98e1d01963f50aa2e152f7c8a6062662019431b058b64e5","href":"https://static.observableusercontent.com/files/118cdc6266aad98eab76298112f953d92cf36a79e9aa8686b4a994703d35e00067b5c038c216a7f6a98e1d01963f50aa2e152f7c8a6062662019431b058b64e5","name":"Canary-dot-org-PDF.csv","type":"text/csv","lastModified":1684370651069.093,"size":2444},{"id":"9408cea7cb4be3c835c69e4458b013af775b5aa2d26a4913fa859bec1ade8875f7534037257f4aaf095bb6d3e16b11f8bcb91740c9908772c6d50ce2b2ae8dfe","href":"https://static.observableusercontent.com/files/9408cea7cb4be3c835c69e4458b013af775b5aa2d26a4913fa859bec1ade8875f7534037257f4aaf095bb6d3e16b11f8bcb91740c9908772c6d50ce2b2ae8dfe","name":"ASUS-Monitor-Invoice.csv","type":"text/csv","lastModified":1684375498665.795,"size":2914},{"id":"86d0c10c48fc90b0ee89652b8a588fb5179373d0296ee584679dbc70e18c2cf56b81d4f8d5405c1819bb48175a65c90ec6e5ec0607ea6da5f5a769ace97727fe","href":"https://static.observableusercontent.com/files/86d0c10c48fc90b0ee89652b8a588fb5179373d0296ee584679dbc70e18c2cf56b81d4f8d5405c1819bb48175a65c90ec6e5ec0607ea6da5f5a769ace97727fe","name":"MaliciousJavaScript.csv","type":"text/csv","lastModified":1684375498689.743,"size":4167}]
49:{"id":"4b496c81bd3392f1","slug":"visualizing-pdfs-with-sankey-diagrams","title":"Visualizing PDFs with Sankey Diagrams","thumbnail":"4eccc619f8713ec31d6fa531f137fe2160a454653c2177bd6a089f00c9af4c93","access_level":"public","create_time":"$D2023-03-16T08:11:34.033Z","update_time":"$D2026-10-08T07:54:41.780Z","view_time":null,"first_public_time":"$D2023-04-20T06:03:35.659Z","has_unpublished_edits":null,"fork":{"id":"99b2e903e7df3b0a","slug":"sankey-component","title":"Sankey diagram component","owner":{"id":"863951e3ebe4c0ae","login":"d3","name":"D3"}},"stars":13,"starred":false,"owner":{"type":"team","id":"fe714748c3ece01b","login":"pdf","name":"PDF","avatar_url":"https://avatars-next.observableusercontent.com/80187a4e9bdaa461147a1ea3823af3f82fefc2b6c0c286dcdf0482677481dba2","tier":"starter_2024","bio":"","followed_by_you":false,"follows_you":false},"version":1048,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"af6dc9ad4498c7e3","login":"petervwyatt","name":"Peter Wyatt","avatar_url":"https://avatars-next.observableusercontent.com/1be04ea37c4ef7bf259202a628b1d97b944eca2f9eec46a717ce6458d8fe1fa6","tier":null,"bio":"CTO of PDF Association\nPI of DARPA SafeDocs\nPL of ISO 32000","followed_by_you":false,"follows_you":false}],"members":[{"type":"team","id":"fe714748c3ece01b","login":"pdf","name":"PDF","avatar_url":"https://avatars-next.observableusercontent.com/80187a4e9bdaa461147a1ea3823af3f82fefc2b6c0c286dcdf0482677481dba2","tier":"starter_2024","role":"editor"},{"type":"individual","id":"af6dc9ad4498c7e3","login":"petervwyatt","name":"Peter Wyatt","avatar_url":"https://avatars-next.observableusercontent.com/1be04ea37c4ef7bf259202a628b1d97b944eca2f9eec46a717ce6458d8fe1fa6","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Visualizing PDFs with Sankey Diagrams","cells":[{"id":0,"value":"$4d","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":373,"value":"viewof align = Inputs.select([\"left\", \"right\", \"center\", \"justify\"], {\n  value: \"left\",\n  label: \"Alignment =\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":720,"value":"viewof height = {\n  const form = html`<form class=\"observablehq--inspect\">Height  = <input name=i type=number min=0 value=900 step=20 style=\"padding:2px;margin:-2px 0;width:80px;\"> <span style=\"font-family:serif;\">(acts like a zoom)</span></form>`;\n  (form.oninput = () => form.value = form.i.valueAsNumber)();\n  return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":723,"value":"viewof width = {\n  const form = html`<form class=\"observablehq--inspect\">Width   = <input name=i type=number min=0 value=720 step=20 style=\"padding:2px;margin:-2px 0;width:80px;\"></form>`;\n  (form.oninput = () => form.value = form.i.valueAsNumber)();\n  return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":878,"value":"viewof nodeWidth = {\n  const form = html`<form class=\"observablehq--inspect\">Node width   = <input name=i type=number min=0 value=15 step=1 style=\"padding:2px;margin:-2px 0;width:80px;\"></form>`;\n  (form.oninput = () => form.value = form.i.valueAsNumber)();\n  return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":726,"value":"viewof padding = {\n  const form = html`<form class=\"observablehq--inspect\">Node padding = <input name=i type=number min=0 value=10 step=1 style=\"padding:2px;margin:-2px 0;width:80px;\"> <span style=\"font-family:serif;\">(vertical spacing between nodes)</span></form>`;\n  (form.oninput = () => form.value = form.i.valueAsNumber)();\n  return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":732,"value":"viewof defcolor = {\n  const form = html`<form class=\"observablehq--inspect\">Default Color = <input name=i type=color value=\"#dddddd\" style=\"padding:2px;margin:-2px 0;\"> <span style=\"font-family:serif;\">(for CSV data that does not explicitly specify a color)</span></form>`;\n  (form.oninput = () => form.value = form.i.value)();\n  return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":746,"value":"$4e","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":947,"value":"You can explore the data here (in JSON):","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":945,"value":"data","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":886,"value":"html`\n${DOM.download(await rasterize(chart), \"pdf-sankey\", \"Download as PNG\")}\n${DOM.download(await serialize(chart), \"pdf-sankey\", \"Download as SVG\")}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":841,"value":"## CSV data\nYou can change the above Sankey diagram and JSON data object by live editing the CSV data below:\n* no header row is needed in this CSV editor\n* labels with quotes need to be quoted\n* do not add additional spaces between columns\n* colors are manually selected to look \"pretty\"","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":704,"value":"$4f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":822,"value":"$50","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":858,"value":"## Cross-reference streams and object streams","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":831,"value":"For PDF 1.5 (and later) files that contain cross-reference streams and object streams streams, extra sets of flows need to be inserted just to the right of the physical file layout node. These will visualize:\n* the mapping of traditional objects in the body of the PDF file to cross-reference streams and object streams streams;\n* the mapping of cross-reference stream data to objects (whether as traditional body objects or in object streams), where the values are the offsets within the streams;\n* the mapping of PDF objects inside object streams, where the values are the offsets within the streams.\n\nIn the case of hybrid-reference PDF files with both cross-reference streams and traditional cross-reference tables, an object may have multiple definitions from both types of cross-reference data.\n\nPS. In the same way, compressed data (some PDF streams) could be uncompressed and represented using another set of vertical nodes.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":844,"value":"## CSV data and the d3.sankey JavaScript library","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":825,"value":"$51","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":903,"value":"$52","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":860,"value":"## Tooling (_Hacks_)\n\nBy far the biggest challenge in creating Sankey diagram visualizations is finding tools that extract the required information, such as byte offsets as most PDF utilities and SDKs hide this information!\n\nThe initial exploratory process was very manual - various Linux CLI commands, some sed/text editing to create the raw data that was loaded into a spreadsheet. Then from within Excel the CSV data was created using `=SUM(...)`, `=VLOOKUP(...)`, etc. (e.g. the size of the pages and groupings are simply the sum of the appropriate objects).\n\nNow that the basics have been shown to work and be useful as a digital file forensic visualization, I developed some very primitive Python scripts that uses `grep` with byte offsets and QPDF (to decompress streams). There is also the start of a script for visualization ICC profiles. The Python script are in GitHub at https://github.com/pdf-association/safedocs/tree/main/SankeyDiagrams. **WARNING:** do not use this script on large or malformed PDFs!","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":846,"value":"## Mechanics of making things work ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":749,"value":"data = {\n  const links = d3.csvParseRows(source, ([source, target, value, linkColor = defcolor]) => (source && target ? {source, target, value: !value || isNaN(value = +value) ? 1 : value, color: linkColor} : null));\n  const nodeByName = new Map;\n  for (const link of links) {\n    if (!nodeByName.has(link.source)) nodeByName.set(link.source, {name: link.source});\n    if (!nodeByName.has(link.target)) nodeByName.set(link.target, {name: link.target});\n  }\n  return {nodes: Array.from(nodeByName.values()), links};\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":281,"value":"sankey = d3.sankey()\n    .nodeId(d => d.name)\n    .nodeAlign(d3[`sankey${align[0].toUpperCase()}${align.slice(1)}`])\n    .nodeSort(null) // keep order as per CSV data!!\n    .nodeWidth(nodeWidth)\n    .nodePadding(padding)\n    .extent([[0, 5], [width, height - 5]]);","pinned":false,"hidden":true,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":888,"value":"import {rasterize, serialize} from \"@mbostock/saving-svg\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":741,"value":"d3 = require(\"d3@7\", \"d3-sankey@0.12\")","pinned":false,"hidden":true,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":921,"value":"---\nCopyright © 2023, PDF Association.\n\nThis material is based upon work supported by the Defense Advanced Research Projects Agency (DARPA) under Contract No. HR001119C0079. Any opinions, findings and conclusions or recommendations expressed in this material are those of the author(s) and do not necessarily reflect the views of the Defense Advanced Research Projects Agency (DARPA). Approved for public release.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"}],"files":"$W53","stdlib":"1"}}
44:null
</article></body></html>