2a:null 2b:["$","title",null,{"children":"Datasets API: size of the splits 💔 | Hugging Face | Observable"}] 30:[] 2e:{"id":"0490fcae6f646391","slug":"datasets-api-size-of-the-splits","title":"Datasets API: size of the splits 💔","thumbnail":"a4a4f4c66ba29b11ca685d2b174879d465b66a96d67d84b1d3da067092e6e837","access_level":"public","create_time":"$D2021-08-19T13:16:38.816Z","update_time":"$D2024-04-05T13:30:12.572Z","view_time":null,"first_public_time":"$D2022-09-12T13:47:08.340Z","fork":false,"stars":6,"starred":false,"owner":{"type":"team","id":"08f80506ae76fdf3","login":"huggingface","name":"Hugging Face","avatar_url":"https://avatars-next.observableusercontent.com/798cf3379d3840e4759a43470c74a7225fe5d81c88a08e763c446c93eff8797c","tier":"pro"},"version":950,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"bf1872cca880d345","login":"severo","name":"Sylvain Lesage","avatar_url":"https://avatars-next.observableusercontent.com/7713195c83e5bfd8090ffe70c871c23e743c711e9e712675c99a8960ca8c7944","tier":null}],"members":[{"type":"team","id":"08f80506ae76fdf3","login":"huggingface","name":"Hugging Face","avatar_url":"https://avatars-next.observableusercontent.com/798cf3379d3840e4759a43470c74a7225fe5d81c88a08e763c446c93eff8797c","tier":"pro","role":"editor"},{"type":"individual","id":"bf1872cca880d345","login":"severo","name":"Sylvain Lesage","avatar_url":"https://avatars-next.observableusercontent.com/7713195c83e5bfd8090ffe70c871c23e743c711e9e712675c99a8960ca8c7944","tier":null,"role":"editor"}],"comments":[{"id":"543788c67ec8b148","reply_to_id":null,"content":"datasetId is always glue. Is it normal ? Shouldn't be updated with the selection above ? Thus the link at the bottom of the treemap is always glue. \n\nBy the way, the use of Inputs.table with multiple:false is a very nice trick.","create_time":"$D2022-09-12T13:59:15.553Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"c1c217aebd614b45","login":"pac02","name":"PAC","avatar_url":"https://avatars-next.observableusercontent.com/2e849389f24e4a039178c801e4df9065e03c23e4077b0ff6bfc8e2763dd06bf9","tier":null}},{"id":"0962d4fbcbe9b091","reply_to_id":"543788c67ec8b148","content":"Yes it would be good to update the URL automatically. For now, we have to click on Bookmark to update it.","create_time":"$D2022-09-12T13:59:59.879Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"bf1872cca880d345","login":"severo","name":"Sylvain Lesage","avatar_url":"https://avatars-next.observableusercontent.com/7713195c83e5bfd8090ffe70c871c23e743c711e9e712675c99a8960ca8c7944","tier":null}},{"id":"2e975d3bbb1ab4b4","reply_to_id":"543788c67ec8b148","content":"Thanks for your answer.\n\nVery nice notebook.","create_time":"$D2022-09-12T14:52:46.722Z","update_time":null,"resolved":true,"user":{"type":"individual","id":"c1c217aebd614b45","login":"pac02","name":"PAC","avatar_url":"https://avatars-next.observableusercontent.com/2e849389f24e4a039178c801e4df9065e03c23e4077b0ff6bfc8e2763dd06bf9","tier":null}},{"id":"dd85bd6ffa7c2038","reply_to_id":null,"content":"It looks like this route may no longer exist?","create_time":"$D2024-05-01T15:49:49.641Z","update_time":null,"resolved":false,"user":{"type":"individual","id":"07362516b5994994","login":"mootari","name":"Fabian Iwand","avatar_url":"https://avatars-next.observableusercontent.com/0d7defa821f38094c03bad23b9b360a5364e6e97e21fc238c39ddc48db7994ad","tier":null}},{"id":"7809fb782efa3a43","reply_to_id":"dd85bd6ffa7c2038","content":"indeed, we deprecated it! I'll have to rewrite this notebook a bit","create_time":"$D2024-05-02T11:58:42.818Z","update_time":null,"resolved":false,"user":{"type":"individual","id":"bf1872cca880d345","login":"severo","name":"Sylvain Lesage","avatar_url":"https://avatars-next.observableusercontent.com/7713195c83e5bfd8090ffe70c871c23e743c711e9e712675c99a8960ca8c7944","tier":null}}],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":1,"collection":{"id":"efa944eb2bf1c5e6","slug":"datasets","title":"Datasets","owner":{"id":"08f80506ae76fdf3","login":"huggingface","name":"Hugging Face"}},"type":"notebook","body":{"title":"Datasets API: size of the splits 💔","cells":[{"id":0,"value":"md`# Datasets API: size of the splits 💔\n\n> The [🤗 dataset viewer API](https://huggingface.co/docs/dataset-viewer) gives access to the contents, metadata and basic statistics of the Hugging Face Hub datasets via a REST API.\n\nHere we use it to visualize the *size of the splits* of a dataset published on the [Hub](https://huggingface.co/datasets).\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":164,"value":"html`

\n ${Treemap(data, {\n //path: (d) => d.name.replace(/\\./g, \"/\"), // e.g., \"flare/animate/Easing\"\n value: (d) => d[field], // size of each node (file); null for internal nodes (folders)\n group: (d) => d.name.split(\".\")[1], // e.g., \"animate\" in \"flare.animate.Easing\"; for color\n label: (d, n) => fields[field].getLabel(d, n),\n title: (d, n) => fields[field].getLabel(d, n), // text to show on hover\n link: (d, n) => {\n const [da, c, s] = d.name.split(\".\");\n return (\n `https://huggingface.co/datasets/${da}/viewer/${c}` + (s ? `/${s}` : \"\")\n );\n },\n tile, // e.g., d3.treemapBinary; set by input above\n width: (2 / 3) * width,\n height: width / 3,\n colors\n })}\n

Dataset: ${datasetId}

\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":178,"value":"viewof groupBy = Inputs.radio([\"config\", \"split\"], {\n label: \"Group by\",\n value: \"config\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":681,"value":"viewof field = Inputs.radio([\"num_bytes_memory\", \"num_rows\"], {\n label: \"Unit\",\n value: \"num_rows\",\n format: (d) => fields[d].label\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":886,"value":"bookmark.create(\n { datasetId: dataset ? dataset.id : null },\n `👉 bookmark ${datasetId}`\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":88,"value":"md`---\n\n### Select another dataset\n\nSearch for a dataset (ie ${dsLink(\"glue\")}, ${dsLink(\n \"allenai/nllb\"\n)} or ${dsLink(\"oscar\")}) and **click on a row** in the table.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":72,"value":"viewof search = Inputs.search(datasets, { query: datasetId })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":631,"value":"viewof dataset = Inputs.table(search, {\n multiple: false,\n value: search.find((d) => d.id === datasetId)\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":814,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":102,"value":"md`### First rows of the split\n\nThe [🤗 dataset viewer API](https://huggingface.co/docs/dataset-viewer) also provides the 100 first rows of every split.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":81,"value":"viewof config = Inputs.select(configs, { label: \"Config (subset)\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":127,"value":"viewof split = Inputs.select(splits, { label: \"Split\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":32,"value":"rowsResponse.rows\n ? Inputs.table(rowsResponse.rows.map((d) => d.row))\n : md`This split cannot be streamed (see [issue](https://github.com/huggingface/datasets-preview-backend/issues/9)). The reported error is:\n\n

\n${rowsResponse.cause}\n${rowsResponse.cause_message}\n\n

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":130,"value":"md`---\n\n### Settings\n\nEnter a HF token to get access to the [gated datasets](https://huggingface.co/docs/hub/datasets-gated).`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":667,"value":"viewof token = Inputs.text({\n label: md`HF Token ([settings](https://huggingface.co/settings/tokens))`\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":554,"value":"viewof tile = Inputs.select(\n new Map([\n [\"binary\", d3.treemapBinary],\n [\"squarify\", d3.treemapSquarify],\n [\"slice-dice\", d3.treemapSliceDice],\n [\"slice\", d3.treemapSlice],\n [\"dice\", d3.treemapDice]\n ]),\n { label: \"Tiling method\", value: d3.treemapBinary }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":10,"value":"md`The dataset viewer API is ${\n (await fetch(endpoint + \"/splits?dataset=glue\").then((r) => r.status === 200))\n ? \"on ✅\"\n : \"off ❌\"\n}\n\n---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":685,"value":"---\n\n#### Variables","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":235,"value":"configSizes = getData(dataset.id, splitItemsByConfig, false).children.sort(\n (a, b) => d3.descending(a[field], b[field])\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":155,"value":"data = getData(dataset.id, splitItemsByConfig, groupBy)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":378,"value":"rowsResponse = fetchRows({ dataset: dataset.id, config, split })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":937,"value":"size = await fetchSize({ dataset: dataset.id })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":505,"value":"splitItems = size.size\n .splits /*await fetchSplits({ dataset: dataset.id }).then((d) => d.splits)*/","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":530,"value":"splitItemsByConfig = d3.group(splitItems, (d) => d.config)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":501,"value":"configs = [...new Set(splitItems.map(({ config }) => config))]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":120,"value":"splits = [...new Set(splitItemsByConfig.get(config).map(({ split }) => split))]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":516,"value":"splitItem = splitItemsByConfig\n .get(config)\n .filter((d) => d.split === split)\n .pop()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":46,"value":"datasets = await fetch(endpoint + \"/valid\").then((d) =>\n d\n .json()\n .then((d) =>\n [...d.preview, ...d.viewer].map((dataset) => ({ id: dataset }))\n )\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":670,"value":"options = ({\n headers: {\n authorization: `Bearer ${token}`\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":400,"value":"datasetId = {\n while (true) {\n yield bookmark.read().datasetId || defaultDatasetId;\n await new Promise((resolve) => {\n window.onhashchange = resolve;\n });\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":436,"value":"selectedDataset = datasets.find(d => d.id === datasetId)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":727,"value":"---\n\n#### Constants","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":746,"value":"fields = ({\n num_rows: {\n label: \"rows\",\n axisLabel: \"number of rows\",\n getLabel: (d, n) => {\n const [da, c, s] = d.name.split(\".\");\n return [c, \"\", s, n.value.toLocaleString(\"en\") + \" rows\"].join(\"\\n\");\n }\n },\n num_bytes_memory: {\n label: \"bytes\",\n axisLabel: \"size in MB (in the memory)\",\n getLabel: (d, n) => {\n const [da, c, s] = d.name.split(\".\");\n return [\n c,\n \"\",\n s,\n n.value.toLocaleString(\"en\", {\n style: \"unit\",\n unit: \"byte\",\n notation: \"compact\"\n })\n ].join(\"\\n\");\n }\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":732,"value":"defaultDatasetId = \"glue\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":616,"value":"endpoint = \"https://datasets-server.huggingface.co\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":694,"value":"---\n\n#### Code","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":226,"value":"colors = d3.schemeSet2","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":191,"value":"function getData(dataset, splitItemsByConfig, groupBy) {\n function getConfigNumBytes(splitItems) {\n return d3.sum(splitItems, (d) => d.num_bytes_memory);\n }\n function getConfigNumRows(splitItems) {\n return d3.sum(splitItems, (d) => d.num_rows);\n }\n return {\n name: dataset,\n children: [...splitItemsByConfig].map(([config, splitItems]) => ({\n name: dataset + \".\" + config,\n ...(groupBy === \"split\"\n ? {\n children: splitItems.map(\n ({ split, num_bytes_memory, num_rows }) => ({\n name: dataset + \".\" + config + \".\" + split,\n num_bytes_memory: num_bytes_memory,\n num_rows: num_rows\n })\n )\n }\n : {\n num_bytes_memory: getConfigNumBytes(splitItems),\n num_rows: getConfigNumRows(splitItems)\n })\n }))\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":934,"value":"async function fetchSize({ dataset }) {\n const r = await fetch(endpoint + \"/size?dataset=\" + dataset, options);\n if (r.status !== 200) {\n throw new Error(await r.text());\n }\n const json = await r.json();\n return json;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":278,"value":"async function fetchSplits({ dataset }) {\n const r = await fetch(endpoint + \"/splits?dataset=\" + dataset, options);\n if (r.status !== 200) {\n throw new Error(await r.text());\n }\n const json = await r.json();\n return json;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":273,"value":"async function fetchRows({ dataset, config, split } = {}) {\n const r = await fetch(\n endpoint +\n \"/first-rows?dataset=\" +\n dataset +\n \"&config=\" +\n config +\n \"&split=\" +\n split,\n options\n );\n if (r.status !== 200) {\n throw new Error(await r.text());\n }\n const json = await r.json();\n return json;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":848,"value":"dsLink = (datasetId) =>\n `[${datasetId}](${document.baseURI.split(\"#\")[0]}#${encodeURIComponent(\n JSON.stringify({ datasetId })\n )})`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":690,"value":"---\n\n#### Imports","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":216,"value":"import { swatches as Swatches } from \"@d3/color-legend\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":396,"value":"import { bookmark } from \"@severo/bookmark\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":162,"value":"import {Treemap} from \"@d3/treemap\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W30","stdlib":"1","titling":"auto"}} 2f:"$2e:body"