2a:null 2b:["$","title",null,{"children":"Difference between two Parquet files | Hugging Face | Observable"}] 30:T693,// generated with // de synthetic -s 1 -e 2 '{"a": "int", "b": "str", "c": ["int"]}' // see https://github.com/huggingface/dataset-dedupe-estimator defaultExamples = [ { label: "appended 5% of 1M rows at the end", layout: s1c3e2SnappyAppendedCdcLayout, previousLayout: s1c3e2SnappyOriginalCdcLayout }, { label: "inserted 5% of 1M rows at random places", layout: s1c3e2SnappyInsertedCdcLayout, previousLayout: s1c3e2SnappyOriginalCdcLayout }, { label: "deleted 5% of 1M rows at random places", layout: s1c3e2SnappyDeletedCdcLayout, previousLayout: s1c3e2SnappyOriginalCdcLayout }, { label: "updated 5% of 1M rows at random places - example 1", layout: s1c3e2SnappyUpdatedCdcLayout, previousLayout: s1c3e2SnappyOriginalCdcLayout }, { label: "updated 5% of 1M rows at random places - example 2", layout: s1c3e2SnappyUpdated_aCdcLayout, previousLayout: s1c3e2SnappyOriginalCdcLayout }, { label: "updated 5% of 1M rows at random places - example 3", layout: s1c3e2SnappyUpdated_bCdcLayout, previousLayout: s1c3e2SnappyOriginalCdcLayout }, { label: "updated 5% of 1M rows at random places - example 4", layout: s1c3e2SnappyUpdated_cCdcLayout, previousLayout: s1c3e2SnappyOriginalCdcLayout }, { label: "datasets.parquet", layout: layoutAfter, previousLayout: layoutBefore, beforeUrl: "https://huggingface.co/datasets/cfahlgren1/hub-stats/resolve/088ad7617e35e809d8058cc8839a0401a9590e14/datasets.parquet", afterUrl: "https://huggingface.co/datasets/cfahlgren1/hub-stats/resolve/088ad7617e35e809d8058cc8839a0401a9590e14/datasets.parquet" } ]31:T417,function createColumn({ index, column, previousColumn }) { let hasDiff = false; const createdDeleted = getCreatedDeleted(column, previousColumn); hasDiff ||= createdDeleted; const node = html`

`; const columnName = createColumnName({ index, column, previousColumn }); hasDiff ||= columnName.hasDiff; if (!ignoreElement.column_title) { node.appendChild(columnName.node); } const pagesMetadata = createMetadata({ value: column?.pages.length, previousValue: previousColumn?.pages.length, label: "num_pages" }); hasDiff ||= pagesMetadata.hasDiff; if ( (pagesMetadata.hasDiff || expand.columnMetadata) && !ignoreElement.all_metadata ) { pagesMetadata.node && node.appendChild(pagesMetadata.node); } const pages = createPages({ pages: column?.pages, previousPages: previousColumn?.pages }); hasDiff ||= pages.hasDiff; if (pages.hasDiff || expand.pages) { node.appendChild(pages.node); } return { node, hasDiff }; }32:T439,function createPages({ pages, previousPages }) { let hasDiff = false; const createdDeleted = getCreatedDeleted(pages, previousPages); hasDiff ||= createdDeleted; const additionalClasses = mode === "viz1" ? "box" : ""; const node = html`

`; const numArticles = Math.max(pages?.length ?? 0, previousPages?.length ?? 0); for (const index of d3.range(numArticles)) { const page = mode === "viz0" ? createPageViz0({ index, page: pages?.[index], previousPage: previousPages?.[index] }) : mode === "viz1" ? createPageViz1({ index, page: pages?.[index], previousPage: previousPages?.[index] }) : createTextPage({ index, page: pages?.[index], previousPage: previousPages?.[index] }); hasDiff ||= page.hasDiff; if (page.hasDiff || expand.page) { node.appendChild(page.node); } } return { node, hasDiff }; }33:T53e,function createPageViz0({ index, page, previousPage }) { let hasDiff = false; const createdDeleted = getCreatedDeleted(page, previousPage); hasDiff ||= createdDeleted; // height is given by the previous file (the reference) // width of the diff node is given by the difference, if any // exception: if previousPage is undefined: the current page is the reference for the height, and it's recovered by the diff node const previousValue = getSize(previousPage); const currentValue = getSize(page); const value = previousValue ?? currentValue ?? 0; const height = scale(value); const style = `height: ${height}px;`; const node = html`

`; const diff = (currentValue ?? 0) - (previousValue ?? 0); if (diff !== 0 && value !== 0) { const width = Math.abs(100 * diff) / value; const attachment = previousValue === undefined ? "left: 0" : diff > 0 ? "left: 100%" : "right: 0"; const diffStyle = `height: ${height}px; ${attachment}; top: 0; width: ${width}%;`; const moreLessClassName = getMoreLessClassName(page, previousPage, getSize); const diffNode = html`

`; node.appendChild(diffNode); } return { node, hasDiff }; }34:Te76,35:[{"id":"9eacbdae67a990efce6410e1db6b77aae7f2f664542c67ad74648b7657196cd6b82d3691f10f87aaf1bd11f7370c0cdd224a1f488adb952cdf2b2d0702a5a8a4","href":"https://static.observableusercontent.com/files/9eacbdae67a990efce6410e1db6b77aae7f2f664542c67ad74648b7657196cd6b82d3691f10f87aaf1bd11f7370c0cdd224a1f488adb952cdf2b2d0702a5a8a4","name":"s1c3e2-snappy-updated_a-cdc-layout.json","type":"application/json","lastModified":1747317658523.055,"size":29343},{"id":"c40e025a965f536c3119b2b27818d89d70c4de626a94bceded6068e614db1a6658fd2a92e8d048f38da467d2ea9caad242944188a2b8d4cf132ea2d1f3fa5d11","href":"https://static.observableusercontent.com/files/c40e025a965f536c3119b2b27818d89d70c4de626a94bceded6068e614db1a6658fd2a92e8d048f38da467d2ea9caad242944188a2b8d4cf132ea2d1f3fa5d11","name":"s1c3e2-snappy-deleted-cdc-layout.json","type":"application/json","lastModified":1747317658542.396,"size":29343},{"id":"5639baeaffa30a66472ab3203489814705b9a54df651ded3bfeb52960710982a1b6f1d028b8980997a183ae48359b27a747e67441171a7a9a68c9b1cf84b1ab1","href":"https://static.observableusercontent.com/files/5639baeaffa30a66472ab3203489814705b9a54df651ded3bfeb52960710982a1b6f1d028b8980997a183ae48359b27a747e67441171a7a9a68c9b1cf84b1ab1","name":"s1c3e2-snappy-updated-cdc-layout.json","type":"application/json","lastModified":1747317658509.072,"size":29343},{"id":"c68b6ddec22a966f86bbb632417881c8bc07c35e2da72627ac310fa5904f76c503db59ad62f612f5d67052ca7cc532e5200db7283c0e7a40a830135e5fd79a2e","href":"https://static.observableusercontent.com/files/c68b6ddec22a966f86bbb632417881c8bc07c35e2da72627ac310fa5904f76c503db59ad62f612f5d67052ca7cc532e5200db7283c0e7a40a830135e5fd79a2e","name":"s1c3e2-snappy-updated_c-cdc-layout.json","type":"application/json","lastModified":1747317658541.165,"size":29343},{"id":"7196ec27804c92f3d1e3d1e5f6a8caf392d343f4fdcb7257bc707b3c382dce8697869630a1966ae86a8f7e831b0d3490d200dadcc59f2e7311c16a8512e87ab1","href":"https://static.observableusercontent.com/files/7196ec27804c92f3d1e3d1e5f6a8caf392d343f4fdcb7257bc707b3c382dce8697869630a1966ae86a8f7e831b0d3490d200dadcc59f2e7311c16a8512e87ab1","name":"s1c3e2-snappy-updated_b-cdc-layout.json","type":"application/json","lastModified":1747317658543.14,"size":29343},{"id":"7e057158007016f88b68a30fdba689585d8c18e4235763d0719454e3cb7a456b593de6d5c941d78d1bef613139b7517e699cc4b4db8bad2bbb934af3ff4e8b7b","href":"https://static.observableusercontent.com/files/7e057158007016f88b68a30fdba689585d8c18e4235763d0719454e3cb7a456b593de6d5c941d78d1bef613139b7517e699cc4b4db8bad2bbb934af3ff4e8b7b","name":"s1c3e2-snappy-inserted-cdc-layout.json","type":"application/json","lastModified":1747317658551.009,"size":30954},{"id":"ac5c705857e1ce9a904db83aed6271f3bd79a990c4dbf03fd7585702b24eaa28f6097bf325f6abd30ad270b6b296c4001239859ffa9f0ccc70af133ffffea1e9","href":"https://static.observableusercontent.com/files/ac5c705857e1ce9a904db83aed6271f3bd79a990c4dbf03fd7585702b24eaa28f6097bf325f6abd30ad270b6b296c4001239859ffa9f0ccc70af133ffffea1e9","name":"s1c3e2-snappy-appended-cdc-layout.json","type":"application/json","lastModified":1747317658564.453,"size":31652},{"id":"1ac63d2df560800745d4cbbadf9147140b81fe8fc3a9599c4ac5ec90d76220d2364ec828ead739c93913d7c5c0670ff8e4ab1da9bef86b4e08bc630558c6f804","href":"https://static.observableusercontent.com/files/1ac63d2df560800745d4cbbadf9147140b81fe8fc3a9599c4ac5ec90d76220d2364ec828ead739c93913d7c5c0670ff8e4ab1da9bef86b4e08bc630558c6f804","name":"s1c3e2-snappy-original-cdc-layout.json","type":"application/json","lastModified":1747317658569.085,"size":29343},{"id":"87174443df681d216804e33e6f46c0b1f8e90b9b4a1fd391fd20d0a293f6fbaddbd09b90c18dcd78fac78135ad11f8e9801e60dbeeb2011e72741eb65f541a08","href":"https://static.observableusercontent.com/files/87174443df681d216804e33e6f46c0b1f8e90b9b4a1fd391fd20d0a293f6fbaddbd09b90c18dcd78fac78135ad11f8e9801e60dbeeb2011e72741eb65f541a08","name":"datasets-after-e7a05c9-layout.json","type":"application/json","lastModified":1747237390663.932,"size":138608},{"id":"fa3a6e7352b4a73ff845c049bc033ffc6f1f78ba5a18cf90ba0421a044fe529cb336ed74ccc25af681f88f7a1c7007e40bf45b293ff1fd61d5a4301f9f207e40","href":"https://static.observableusercontent.com/files/fa3a6e7352b4a73ff845c049bc033ffc6f1f78ba5a18cf90ba0421a044fe529cb336ed74ccc25af681f88f7a1c7007e40bf45b293ff1fd61d5a4301f9f207e40","name":"datasets-before-088ad76-layout.json","type":"application/json","lastModified":1747237390690.771,"size":137950}] 2e:{"id":"632fcb34887728f5","slug":"difference-between-two-parquet-files","title":"Difference between two Parquet files","thumbnail":"7e7651a9ef62de93fb9f40434ef8874f6e8c9f300d37fc655d20da09291b1981","access_level":"public","create_time":"$D2025-05-14T15:38:57.063Z","update_time":"$D2025-05-16T16:45:03.542Z","view_time":null,"first_public_time":"$D2025-05-15T13:38:37.271Z","fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"08f80506ae76fdf3","login":"huggingface","name":"Hugging Face","avatar_url":"https://avatars-next.observableusercontent.com/798cf3379d3840e4759a43470c74a7225fe5d81c88a08e763c446c93eff8797c","tier":"pro"},"version":1440,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"bf1872cca880d345","login":"severo","name":"Sylvain Lesage","avatar_url":"https://avatars-next.observableusercontent.com/7713195c83e5bfd8090ffe70c871c23e743c711e9e712675c99a8960ca8c7944","tier":null}],"members":[{"type":"individual","id":"bf1872cca880d345","login":"severo","name":"Sylvain Lesage","avatar_url":"https://avatars-next.observableusercontent.com/7713195c83e5bfd8090ffe70c871c23e743c711e9e712675c99a8960ca8c7944","tier":null,"role":"editor"},{"type":"team","id":"08f80506ae76fdf3","login":"huggingface","name":"Hugging Face","avatar_url":"https://avatars-next.observableusercontent.com/798cf3379d3840e4759a43470c74a7225fe5d81c88a08e763c446c93eff8797c","tier":"pro","role":"viewer"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"title":"Difference between two Parquet files","cells":[{"id":0,"value":"# Difference between two Parquet files\n\nLet's use the data from parquet-layout to show an overview of the parts that differ between two Parquet files.\n\nMore details about parquet-layout at https://observablehq.com/@huggingface/hello-parquet-layout-rust and https://observablehq.com/@huggingface/parquet-layout-vs-hyparquet-vs-duckdb.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1384,"value":"_some parameters - to tweak finely_","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1179,"value":"viewof onlyDiffs = Inputs.toggle({ label: \"Show only diffs\", value: false })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":391,"value":"viewof expandView = Inputs.checkbox(values, { label: \"Expand\", value: values })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":616,"value":"viewof ignoreMetadataView = Inputs.checkbox(metadataLabels, {\n label: \"Ignore metadata\",\n value: [\"offset\"]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1303,"value":"viewof ignoreElementView = Inputs.checkbox(elementLabels, {\n label: \"Ignore\",\n value: [\"all_metadata\"]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1404,"value":"## Select an example","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":648,"value":"viewof example = Inputs.select(examples, {\n label: \"Select an example\",\n format: (x) => x.label\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1392,"value":"or create layout files locally with\n\n parquet-layout file.parquet > file.json\n\nand upload them here:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1399,"value":"viewof previousFile = Inputs.file({\n label: \"Previous\",\n accept: \".json\",\n required: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1394,"value":"viewof currentFile = Inputs.file({\n label: \"Current\",\n accept: \".json\",\n required: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1386,"value":"**Try changing these values 👇 **","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1326,"value":"viewof compact = Inputs.toggle({ label: \"Compact mode\", value: false })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1343,"value":"viewof vertical = Inputs.toggle({ label: \"Vertical layout\", value: true })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":948,"value":"viewof mode = Inputs.radio([\"text\", \"viz0\", \"viz1\"], {\n label: \"Visualization mode\",\n value: \"viz1\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1031,"value":"createLayout()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":669,"value":"---\n\n_data_","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":651,"value":"$30","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1433,"value":"examples = {\n yield defaultExamples;\n const [layout, previousLayout] = await Promise.all([\n uploadedCurrent,\n uploadedPrevious\n ]);\n yield [\n {\n label: \"uploaded\",\n layout,\n previousLayout\n },\n ...defaultExamples\n ];\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":705,"value":"s1c3e2SnappyOriginalCdcLayout = FileAttachment(\"s1c3e2-snappy-original-cdc-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":699,"value":"s1c3e2SnappyAppendedCdcLayout = FileAttachment(\"s1c3e2-snappy-appended-cdc-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":706,"value":"s1c3e2SnappyInsertedCdcLayout = FileAttachment(\"s1c3e2-snappy-inserted-cdc-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":707,"value":"s1c3e2SnappyDeletedCdcLayout = FileAttachment(\"s1c3e2-snappy-deleted-cdc-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":704,"value":"s1c3e2SnappyUpdatedCdcLayout = FileAttachment(\"s1c3e2-snappy-updated-cdc-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":701,"value":"s1c3e2SnappyUpdated_cCdcLayout = FileAttachment(\"s1c3e2-snappy-updated_c-cdc-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":702,"value":"s1c3e2SnappyUpdated_bCdcLayout = FileAttachment(\"s1c3e2-snappy-updated_b-cdc-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":703,"value":"s1c3e2SnappyUpdated_aCdcLayout = FileAttachment(\"s1c3e2-snappy-updated_a-cdc-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":21,"value":"In the last example, we compare two revisions of a Parquet file in the dataset [cfahlfren/hub-stats](https://huggingface.co/datasets/cfahlgren1/hub-stats) ([before](https://huggingface.co/datasets/cfahlgren1/hub-stats/blob/088ad7617e35e809d8058cc8839a0401a9590e14/datasets.parquet) / [after](https://huggingface.co/datasets/cfahlgren1/hub-stats/blob/e7a05c99cbe80eba18c140b5f65b566ec284ab34/datasets.parquet)).","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":11,"value":"layoutBefore = FileAttachment(\"datasets-before-088ad76-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":10,"value":"layoutAfter = FileAttachment(\"datasets-after-e7a05c9-layout.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1409,"value":"uploadedPrevious = previousFile.json()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1412,"value":"uploadedCurrent = currentFile.json()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":95,"value":"---\n\n_code_","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":404,"value":"values = [\n \"rowGroups\",\n \"rowGroup\",\n \"rowGroupMetadata\",\n \"columns\",\n \"column\",\n \"columnMetadata\",\n \"pages\",\n \"page\",\n \"pageMetadata\"\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":402,"value":"expand = Object.fromEntries(\n values.map((value) => [value, expandView.includes(value) && !onlyDiffs])\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":620,"value":"pageFields = [\n \"compression\",\n \"encoding\",\n \"page_type\",\n \"offset\",\n \"compressed_bytes\",\n \"uncompressed_bytes\",\n \"header_bytes\",\n \"num_values\"\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":623,"value":"metadataLabels = [...pageFields, \"num_pages\", \"num_columns\", \"row_count\"]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":627,"value":"ignoreMetadata = Object.fromEntries(\n metadataLabels.map((value) => [value, ignoreMetadataView.includes(value)])\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1308,"value":"elementLabels = [\"all_metadata\", \"column_title\", \"row_group_title\"]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1314,"value":"ignoreElement = Object.fromEntries(\n elementLabels.map((value) => [value, ignoreElementView.includes(value)])\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":45,"value":"function createLayout() {\n const { label, layout, previousLayout } = example;\n let hasDiff = false;\n const createdDeleted = getCreatedDeleted(layout, previousLayout);\n hasDiff ||= createdDeleted;\n const node = html`

`;\n\n node.appendChild(html`

${label}

`);\n\n const rowGroups = createRowGroups({\n rowGroups: layout?.row_groups,\n previousRowGroups: previousLayout?.row_groups\n });\n hasDiff ||= rowGroups.hasDiff;\n if (rowGroups.hasDiff || expand.rowGroups) {\n node.appendChild(rowGroups.node);\n }\n\n return node;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":516,"value":"function createRowGroups({ rowGroups, previousRowGroups }) {\n let hasDiff = false;\n const createdDeleted = getCreatedDeleted(rowGroups, previousRowGroups);\n hasDiff ||= createdDeleted;\n const node = html`

`;\n\n const numArticles = Math.max(\n rowGroups?.length ?? 0,\n previousRowGroups?.length ?? 0\n );\n for (const index of d3.range(numArticles)) {\n const rowGroup = createRowGroup({\n index,\n rowGroup: rowGroups?.[index],\n previousRowGroup: previousRowGroups?.[index]\n });\n hasDiff ||= rowGroup.hasDiff;\n if (rowGroup.hasDiff || expand.rowGroup) {\n node.appendChild(rowGroup.node);\n }\n }\n return { node, hasDiff };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":143,"value":"function createRowGroup({ index, rowGroup, previousRowGroup }) {\n let hasDiff = false;\n const createdDeleted = getCreatedDeleted(rowGroup, previousRowGroup);\n hasDiff ||= createdDeleted;\n const node = html`

`;\n\n if (!ignoreElement.row_group_title) {\n node.appendChild(html`

Row group ${index}

`);\n }\n\n const metadata = createRowGroupMetadata({ rowGroup, previousRowGroup });\n hasDiff ||= metadata.hasDiff;\n if (\n (metadata.hasDiff || expand.rowGroupMetadata) &&\n !ignoreElement.all_metadata\n ) {\n node.appendChild(metadata.node);\n }\n\n const columns = createColumns({\n columns: rowGroup?.columns,\n previousColumns: previousRowGroup?.columns\n });\n hasDiff ||= columns.hasDiff;\n if (columns.hasDiff || expand.columns) {\n node.appendChild(columns.node);\n }\n\n return { node, hasDiff };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":252,"value":"function createColumns({ columns, previousColumns }) {\n let hasDiff = false;\n const createdDeleted = getCreatedDeleted(columns, previousColumns);\n hasDiff ||= createdDeleted;\n const node = html`

`;\n\n const numArticles = Math.max(\n columns?.length ?? 0,\n previousColumns?.length ?? 0\n );\n for (const index of d3.range(numArticles)) {\n const col = createColumn({\n index,\n column: columns?.[index],\n previousColumn: previousColumns?.[index]\n });\n hasDiff ||= col.hasDiff;\n if (col.hasDiff || expand.column) {\n node.appendChild(col.node);\n }\n }\n return { node, hasDiff };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":292,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":307,"value":"function createColumnName({\n index,\n column, // can be undefined\n previousColumn // can be undefined\n}) {\n let hasDiff = false;\n const node = html`

`;\n\n // node.appendChild(html`Column ${index}`);\n if (previousColumn?.path && previousColumn.path !== column?.path) {\n node.appendChild(\n html`${column?.path ?? \"\"}`\n );\n node.appendChild(\n html`${previousColumn.path}`\n );\n hasDiff = true;\n } else {\n node.appendChild(html`${column?.path ?? \"\"}`);\n }\n\n return { node, hasDiff };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":493,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":496,"value":"function createTextPage({ index, page, previousPage }) {\n let hasDiff = false;\n const createdDeleted = getCreatedDeleted(page, previousPage);\n hasDiff ||= createdDeleted;\n const node = html`

`;\n\n node.appendChild(html`

Page ${index}

`);\n\n for (const label of pageFields) {\n const metadata = createMetadata({\n value: page?.[label],\n previousValue: previousPage?.[label],\n label\n });\n hasDiff ||= metadata.hasDiff;\n if (metadata.hasDiff || expand.pageMetadata) {\n metadata.node && node.appendChild(metadata.node);\n }\n }\n\n return { node, hasDiff };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1037,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1120,"value":"function createPageViz1({ index, page, previousPage }) {\n const hasDiff = !isPageIdentical(page, previousPage);\n const hasDiffClass = hasDiff ? \"diff\" : \"\";\n\n const previousValue = getSize(previousPage);\n const currentValue = getSize(page);\n const previousHeight = scale(previousValue ?? 0);\n const currentHeight = scale(currentValue ?? 0);\n const maxHeight = Math.max(previousHeight, currentHeight);\n const style = `height: ${maxHeight}px;`;\n\n const node = html`

`;\n\n if (previousValue) {\n node.appendChild(\n html`

`\n );\n }\n if (currentValue) {\n node.appendChild(\n html`

`\n );\n }\n\n return { node, hasDiff };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":156,"value":"function createRowGroupMetadata({\n rowGroup,\n previousRowGroup // can be undefined\n}) {\n let hasDiff = false;\n const node = html`

`;\n\n const rowsMetadata = createMetadata({\n value: rowGroup?.row_count,\n previousValue: previousRowGroup?.row_count,\n label: \"row_count\"\n });\n rowsMetadata.node && node.appendChild(rowsMetadata.node);\n hasDiff ||= rowsMetadata.hasDiff;\n\n const columnsMetadata = createMetadata({\n value: rowGroup?.columns.length,\n previousValue: previousRowGroup?.columns.length,\n label: \"num_columns\"\n });\n columnsMetadata.node && node.appendChild(columnsMetadata.node);\n hasDiff ||= columnsMetadata.hasDiff;\n\n return { node, hasDiff };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":180,"value":"function createMetadata({ value, previousValue, label }) {\n let hasDiff = false;\n\n if (ignoreMetadata[label]) {\n return { node: undefined, hasDiff: false };\n }\n\n const node = html`

`;\n\n const labelSpan = html`${label}`;\n node.appendChild(labelSpan);\n\n if (value !== undefined && value === previousValue) {\n node.appendChild(html`${formatValue({ value })}`);\n } else {\n if (previousValue !== undefined) {\n node.appendChild(\n html`${formatValue({\n value: previousValue\n })}`\n );\n hasDiff = true;\n }\n if (value !== undefined) {\n node.appendChild(\n html`${formatValue({ value })}`\n );\n hasDiff = true;\n }\n }\n\n return { node, hasDiff };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":588,"value":"function formatValue({ value, signDisplay }) {\n return value.toLocaleString(\"en-US\", { signDisplay });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":210,"value":"function formatWithUnit({ value, unit, signDisplay }) {\n const str = `${value.toLocaleString(\"en-US\", { signDisplay })} ${unit}`;\n if (Math.abs(value) > 1) {\n return str + \"s\";\n }\n return str;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":471,"value":"function getCreatedDeleted(value, previous) {\n if (value === undefined) {\n if (previous === undefined) {\n return \"\";\n } else {\n return \"deleted\";\n }\n } else {\n if (previous === undefined) {\n return \"created\";\n } else {\n return \"\";\n }\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":990,"value":"function getMoreLessClassName(value, previous, accessor) {\n if (value !== undefined) {\n if (previous !== undefined) {\n const diff = accessor(value) - accessor(previous);\n if (diff > 0) {\n return \"more\";\n } else if (diff < 0) {\n return \"less\";\n }\n return \"same\";\n } else {\n return \"more\";\n }\n } else {\n if (previous !== undefined) {\n return \"less\";\n } else {\n return \"\";\n }\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1070,"value":"function getSize(page) {\n if (page === undefined) {\n return undefined;\n }\n return page.header_bytes + page.compressed_bytes;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1000,"value":"scale = {\n const lastPage = example.layout?.row_groups\n .at(-1)\n ?.columns.at(-1)\n ?.pages.at(-1);\n const lastPreviousPage = example.previousLayout?.row_groups\n .at(-1)\n ?.columns.at(-1)\n ?.pages.at(-1);\n const maxBytesPositionAfterData = Math.max(\n lastPage?.offset + getSize(lastPage) ?? 0,\n lastPreviousPage?.offset + getSize(lastPreviousPage) ?? 0\n );\n return d3\n .scaleLinear()\n .domain([0, maxBytesPositionAfterData])\n .range([0, 500]);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1130,"value":"function isPageIdentical(page, previousPage) {\n // it's a heuristic. If parquet-layout provided the checksum (which is optional), we could improve the detection\n return (\n page?.compressed_bytes === previousPage?.compressed_bytes &&\n page?.uncompressed_bytes === previousPage?.uncompressed_bytes &&\n page?.num_values === previousPage?.num_values\n );\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":48,"value":"$34","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined"}],"files":"$W35","stdlib":"1","titling":"auto"}} 2f:"$2e:body"