31:T434,# 'Parquet Content-Defined Chunking' blog post replication The blog post [Parquet Content-Defined Chunking](https://huggingface.co/blog/parquet-cdc) describes the new PyArrow CDC Parquet encoding, which enables deduplication for similar Parquet files. The example files are stored in https://huggingface.co/datasets/severo/pq_reproduction. > **Warning:** the charts slightly differ from the blog post. > > In the blog post, the author Krisztian Szucs has access to the files locally and compares them at a [custom "chunk"](https://github.com/huggingface/dataset-dedupe-estimator/blob/main/src/show.rs) level. In this notebook, we reproduce the examples without downloading the files. Instead, we use metadata provided by the [Xet API](https://observablehq.com/@severo/hello-xet-api). > > The metadata are very lightweight, which lets us immediately show the diff between two files. But, as a drawback, the granularity to compare two files is Xet chunk-level, which is slightly different from the custom chunks used by the author. That explains the difference in the charts.32:T7ac,function display(data) { function showInfo({ file, fileInfo, chunks }) { return html`

${file.path.split("/").at(-1)} (download)

${filesize(fileInfo.size)}. ${chunks.length.toLocaleString( "en-US" )} chunks.

`; } const wrapper = html`

`; for (const { useDataPageIndex, data: dpiData } of data) { const dpiCharts = html`

`; for (const { useCDC, data: cdcData } of dpiData) { const cdcCharts = html`

`; for (const { compression, data: compressionData } of cdcData) { const chart = diffView === "Additions" ? squareAdditionsDiff(compressionData) : diffView === "Additions/Removals" ? squareDiff(compressionData) : diffView === "Two rows" ? twoRowsDiff(compressionData) : html`not implemented`; const section = html`

${ compression === "none" ? "No compression" : "Snappy compression" }

`; section.append(chart); section.append( html`

${showInfo(compressionData.a)} ${showInfo(compressionData.b)} ` ); cdcCharts.append(section); } const cdcSection = html`

${ useCDC === "yes" ? "CDC Parquet" : "Vanilla Parquet" }

`; cdcSection.append(cdcCharts); dpiCharts.append(cdcSection); } const dpiSection = html`

${ useDataPageIndex === "yes" ? "Parquet with data page index" : "Parquet without data page index" }

`; dpiSection.append(dpiCharts); wrapper.append(dpiSection); } return wrapper; }33:T5e4,examples = [ { label: "1. Re-uploading Exact Copies of the Table", url: "https://huggingface.co/blog/parquet-cdc#1-re-uploading-exact-copies-of-the-table", a: "orca.parquet", b: "orca-copy.parquet" }, { label: "2a. Adding Columns to the Table", url: "https://huggingface.co/blog/parquet-cdc#2-adding-and-removing-columns-from-the-table", a: "orca.parquet", b: "orca-added-columns.parquet" }, { label: "2b. Removing Columns from the Table", url: "https://huggingface.co/blog/parquet-cdc#2-adding-and-removing-columns-from-the-table", a: "orca.parquet", b: "orca-removed-columns.parquet" }, { label: "3. Changing Column Types in the Table", url: "https://huggingface.co/blog/parquet-cdc#3-changing-column-types-in-the-table", a: "orca-without-text.parquet", b: "orca-casted-column.parquet" }, { label: "4. Appending New Rows and Concatenating Tables", url: "https://huggingface.co/blog/parquet-cdc#4-appending-new-rows-and-concatenating-tables", a: "orca.parquet", b: "orca-appended-rows.parquet" }, { label: "5a. Inserting Rows in the Table", url: "https://huggingface.co/blog/parquet-cdc#5-inserting--deleting-rows-in-the-table", a: "orca.parquet", b: "orca-inserted-rows.parquet" }, { label: "5b. Deleting Rows in the Table", url: "https://huggingface.co/blog/parquet-cdc#5-inserting--deleting-rows-in-the-table", a: "orca.parquet", b: "orca-deleted-rows.parquet" } ]34:[] 2f:{"title":"'Parquet Content-Defined Chunking' blog post replication","cells":[{"id":0,"value":"$31","pinned":false,"hidden":false,"mode":"md"},{"id":41,"value":"viewof example = Inputs.select(examples, {\n label: \"Example\",\n format: (d) => d.label,\n width: \"auto\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":90,"value":"## ${example.label}\n\nRead section [${example.label}](${example.url}) in the blog post.","pinned":false,"hidden":false,"mode":"md"},{"id":383,"value":"display(data)","pinned":false,"hidden":false,"mode":"ojs"},{"id":424,"value":"---\n\n_configure the chart_\n","pinned":false,"hidden":false,"mode":"md"},{"id":257,"value":"viewof compressions = Inputs.radio([\"none\", \"snappy\", \"show both\"], {\n label: \"Compression\",\n value: \"show both\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":262,"value":"viewof cdcs = Inputs.radio([\"no\", \"yes\", \"show both\"], {\n label: \"Use Content-Defined Chunking (CDC)\",\n value: \"show both\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":306,"value":"viewof dataIndexes = Inputs.radio([\"no\", \"yes\", \"show both\"], {\n label: \"Use data page index\",\n value: \"show both\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":428,"value":"viewof diffView = Inputs.radio(\n [\"Additions\", \"Additions/Removals\", \"Two rows\"],\n {\n label: \"Diff view\",\n value: \"Additions\"\n }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"---\n\n_code_\n","pinned":false,"hidden":false,"mode":"md"},{"id":381,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":354,"value":"async function prepareData({ example, compressions, cdcs, dataIndexes }) {\n const data = [];\n for (const useDataPageIndex of dataIndexes === \"show both\"\n ? [\"no\", \"yes\"]\n : [dataIndexes]) {\n const _data = [];\n for (const useCDC of cdcs === \"show both\" ? [\"no\", \"yes\"] : [cdcs]) {\n const __data = [];\n for (const compression of compressions === \"show both\"\n ? [\"none\", \"snappy\"]\n : [compressions]) {\n __data.push({\n compression,\n data: await prepareIndividualData({\n example,\n compression,\n useCDC: useCDC === \"yes\",\n useDataPageIndex: useDataPageIndex === \"yes\"\n })\n });\n }\n _data.push({\n useCDC,\n data: __data\n });\n }\n data.push({\n useDataPageIndex,\n data: _data\n });\n }\n\n return data;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":341,"value":"async function prepareIndividualData({\n example,\n compression,\n useCDC,\n useDataPageIndex\n}) {\n const pathPrefix = `${useDataPageIndex ? \"index\" : \"no_index\"}/${\n useCDC ? \"cdc\" : \"no_cdc\"\n }/${compression}`;\n const fileA = {\n repo: { type: \"dataset\", name: datasetName },\n path: pathPrefix + \"/\" + example.a,\n revision: \"main\"\n };\n const fileB = {\n repo: { type: \"dataset\", name: datasetName },\n path: pathPrefix + \"/\" + example.b,\n revision: \"main\"\n };\n\n return {\n example,\n compression,\n useCDC,\n useDataPageIndex,\n ...(await xetDiff({ fileA, fileB }))\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":38,"value":"function filesize(size) {\n return (size / Math.pow(2, 20)).toLocaleString(\"en-US\", {\n unit: \"megabyte\",\n style: \"unit\",\n maximumFractionDigits: 1\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"hf = import(\"https://cdn.jsdelivr.net/npm/@huggingface/hub@2.4.1/+esm\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":631,"value":"import {\n xetDiff,\n squareAdditionsDiff,\n squareDiff,\n twoRowsDiff\n} from \"@severo/xet-diff\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":6,"value":"---\n\n_data_","pinned":false,"hidden":false,"mode":"md"},{"id":54,"value":"datasetName = \"severo/pq_reproduction\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":22,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":359,"value":"data = prepareData({ example, compressions, cdcs, dataIndexes })","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W34","stdlib":"1","titling":"auto"}