31:[] 2f:{"title":"Datasets API: size of the splits 💔","cells":[{"id":0,"value":"md`# Datasets API: size of the splits 💔\n\n> The [🤗 dataset viewer API](https://huggingface.co/docs/dataset-viewer) gives access to the contents, metadata and basic statistics of the Hugging Face Hub datasets via a REST API.\n\nHere we use it to visualize the *size of the splits* of a dataset published on the [Hub](https://huggingface.co/datasets).\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":164,"value":"html`

\n ${Treemap(data, {\n //path: (d) => d.name.replace(/\\./g, \"/\"), // e.g., \"flare/animate/Easing\"\n value: (d) => d[field], // size of each node (file); null for internal nodes (folders)\n group: (d) => d.name.split(\".\")[1], // e.g., \"animate\" in \"flare.animate.Easing\"; for color\n label: (d, n) => fields[field].getLabel(d, n),\n title: (d, n) => fields[field].getLabel(d, n), // text to show on hover\n link: (d, n) => {\n const [da, c, s] = d.name.split(\".\");\n return (\n `https://huggingface.co/datasets/${da}/viewer/${c}` + (s ? `/${s}` : \"\")\n );\n },\n tile, // e.g., d3.treemapBinary; set by input above\n width: (2 / 3) * width,\n height: width / 3,\n colors\n })}\n

Dataset: ${datasetId}

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":178,"value":"viewof groupBy = Inputs.radio([\"config\", \"split\"], {\n label: \"Group by\",\n value: \"config\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":681,"value":"viewof field = Inputs.radio([\"num_bytes_memory\", \"num_rows\"], {\n label: \"Unit\",\n value: \"num_rows\",\n format: (d) => fields[d].label\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":886,"value":"bookmark.create(\n { datasetId: dataset ? dataset.id : null },\n `👉 bookmark ${datasetId}`\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":88,"value":"md`---\n\n### Select another dataset\n\nSearch for a dataset (ie ${dsLink(\"glue\")}, ${dsLink(\n \"allenai/nllb\"\n)} or ${dsLink(\"oscar\")}) and **click on a row** in the table.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":72,"value":"viewof search = Inputs.search(datasets, { query: datasetId })","pinned":false,"hidden":false,"mode":"ojs"},{"id":631,"value":"viewof dataset = Inputs.table(search, {\n multiple: false,\n value: search.find((d) => d.id === datasetId)\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":814,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":102,"value":"md`### First rows of the split\n\nThe [🤗 dataset viewer API](https://huggingface.co/docs/dataset-viewer) also provides the 100 first rows of every split.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":81,"value":"viewof config = Inputs.select(configs, { label: \"Config (subset)\" })","pinned":false,"hidden":false,"mode":"ojs"},{"id":127,"value":"viewof split = Inputs.select(splits, { label: \"Split\" })","pinned":false,"hidden":false,"mode":"ojs"},{"id":32,"value":"rowsResponse.rows\n ? Inputs.table(rowsResponse.rows.map((d) => d.row))\n : md`This split cannot be streamed (see [issue](https://github.com/huggingface/datasets-preview-backend/issues/9)). The reported error is:\n\n

\n${rowsResponse.cause}\n${rowsResponse.cause_message}\n\n

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":130,"value":"md`---\n\n### Settings\n\nEnter a HF token to get access to the [gated datasets](https://huggingface.co/docs/hub/datasets-gated).`","pinned":false,"hidden":false,"mode":"ojs"},{"id":667,"value":"viewof token = Inputs.text({\n label: md`HF Token ([settings](https://huggingface.co/settings/tokens))`\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":554,"value":"viewof tile = Inputs.select(\n new Map([\n [\"binary\", d3.treemapBinary],\n [\"squarify\", d3.treemapSquarify],\n [\"slice-dice\", d3.treemapSliceDice],\n [\"slice\", d3.treemapSlice],\n [\"dice\", d3.treemapDice]\n ]),\n { label: \"Tiling method\", value: d3.treemapBinary }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"md`The dataset viewer API is ${\n (await fetch(endpoint + \"/splits?dataset=glue\").then((r) => r.status === 200))\n ? \"on ✅\"\n : \"off ❌\"\n}\n\n---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":685,"value":"---\n\n#### Variables","pinned":false,"hidden":false,"mode":"md"},{"id":235,"value":"configSizes = getData(dataset.id, splitItemsByConfig, false).children.sort(\n (a, b) => d3.descending(a[field], b[field])\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":155,"value":"data = getData(dataset.id, splitItemsByConfig, groupBy)","pinned":false,"hidden":false,"mode":"ojs"},{"id":378,"value":"rowsResponse = fetchRows({ dataset: dataset.id, config, split })","pinned":false,"hidden":false,"mode":"ojs"},{"id":937,"value":"size = await fetchSize({ dataset: dataset.id })","pinned":false,"hidden":false,"mode":"ojs"},{"id":505,"value":"splitItems = size.size\n .splits /*await fetchSplits({ dataset: dataset.id }).then((d) => d.splits)*/","pinned":false,"hidden":false,"mode":"ojs"},{"id":530,"value":"splitItemsByConfig = d3.group(splitItems, (d) => d.config)","pinned":false,"hidden":false,"mode":"ojs"},{"id":501,"value":"configs = [...new Set(splitItems.map(({ config }) => config))]","pinned":false,"hidden":false,"mode":"ojs"},{"id":120,"value":"splits = [...new Set(splitItemsByConfig.get(config).map(({ split }) => split))]","pinned":false,"hidden":false,"mode":"ojs"},{"id":516,"value":"splitItem = splitItemsByConfig\n .get(config)\n .filter((d) => d.split === split)\n .pop()","pinned":false,"hidden":false,"mode":"ojs"},{"id":46,"value":"datasets = await fetch(endpoint + \"/valid\").then((d) =>\n d\n .json()\n .then((d) =>\n [...d.preview, ...d.viewer].map((dataset) => ({ id: dataset }))\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":670,"value":"options = ({\n headers: {\n authorization: `Bearer ${token}`\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":400,"value":"datasetId = {\n while (true) {\n yield bookmark.read().datasetId || defaultDatasetId;\n await new Promise((resolve) => {\n window.onhashchange = resolve;\n });\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":436,"value":"selectedDataset = datasets.find(d => d.id === datasetId)","pinned":false,"hidden":false,"mode":"ojs"},{"id":727,"value":"---\n\n#### Constants","pinned":false,"hidden":false,"mode":"md"},{"id":746,"value":"fields = ({\n num_rows: {\n label: \"rows\",\n axisLabel: \"number of rows\",\n getLabel: (d, n) => {\n const [da, c, s] = d.name.split(\".\");\n return [c, \"\", s, n.value.toLocaleString(\"en\") + \" rows\"].join(\"\\n\");\n }\n },\n num_bytes_memory: {\n label: \"bytes\",\n axisLabel: \"size in MB (in the memory)\",\n getLabel: (d, n) => {\n const [da, c, s] = d.name.split(\".\");\n return [\n c,\n \"\",\n s,\n n.value.toLocaleString(\"en\", {\n style: \"unit\",\n unit: \"byte\",\n notation: \"compact\"\n })\n ].join(\"\\n\");\n }\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":732,"value":"defaultDatasetId = \"glue\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":616,"value":"endpoint = \"https://datasets-server.huggingface.co\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":694,"value":"---\n\n#### Code","pinned":false,"hidden":false,"mode":"md"},{"id":226,"value":"colors = d3.schemeSet2","pinned":false,"hidden":false,"mode":"ojs"},{"id":191,"value":"function getData(dataset, splitItemsByConfig, groupBy) {\n function getConfigNumBytes(splitItems) {\n return d3.sum(splitItems, (d) => d.num_bytes_memory);\n }\n function getConfigNumRows(splitItems) {\n return d3.sum(splitItems, (d) => d.num_rows);\n }\n return {\n name: dataset,\n children: [...splitItemsByConfig].map(([config, splitItems]) => ({\n name: dataset + \".\" + config,\n ...(groupBy === \"split\"\n ? {\n children: splitItems.map(\n ({ split, num_bytes_memory, num_rows }) => ({\n name: dataset + \".\" + config + \".\" + split,\n num_bytes_memory: num_bytes_memory,\n num_rows: num_rows\n })\n )\n }\n : {\n num_bytes_memory: getConfigNumBytes(splitItems),\n num_rows: getConfigNumRows(splitItems)\n })\n }))\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":934,"value":"async function fetchSize({ dataset }) {\n const r = await fetch(endpoint + \"/size?dataset=\" + dataset, options);\n if (r.status !== 200) {\n throw new Error(await r.text());\n }\n const json = await r.json();\n return json;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":278,"value":"async function fetchSplits({ dataset }) {\n const r = await fetch(endpoint + \"/splits?dataset=\" + dataset, options);\n if (r.status !== 200) {\n throw new Error(await r.text());\n }\n const json = await r.json();\n return json;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":273,"value":"async function fetchRows({ dataset, config, split } = {}) {\n const r = await fetch(\n endpoint +\n \"/first-rows?dataset=\" +\n dataset +\n \"&config=\" +\n config +\n \"&split=\" +\n split,\n options\n );\n if (r.status !== 200) {\n throw new Error(await r.text());\n }\n const json = await r.json();\n return json;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":848,"value":"dsLink = (datasetId) =>\n `[${datasetId}](${document.baseURI.split(\"#\")[0]}#${encodeURIComponent(\n JSON.stringify({ datasetId })\n )})`","pinned":false,"hidden":false,"mode":"ojs"},{"id":690,"value":"---\n\n#### Imports","pinned":false,"hidden":false,"mode":"md"},{"id":216,"value":"import { swatches as Swatches } from \"@d3/color-legend\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":396,"value":"import { bookmark } from \"@severo/bookmark\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":162,"value":"import {Treemap} from \"@d3/treemap\"","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}