30:T492,!data ? md`> ⚠️ No data page index found. **Try another Parquet file**.` : Plot.plot({ caption: `Structure of the Parquet file "${url .split("/") .at(-1)}" (columns, row groups, data pages)`, grid: false, width, height: numRowGroups * 200, facet: { label: null, marginLeft: 60 }, color: { legend: true, swatchSize: 10, tickFormat: (d) => (d === "normal" ? "data page" : d), range: ["#58b368"], domain: ["normal"] }, fx: { label: "columns", axis: "top", grid: false, domain: columnNames, // ordered along column index tickSize: 0 }, inset: 4, insetBottom: 8, fy: { axis: "left" }, y: { label: null, tickFormat: (d) => "", tickSize: 0 }, marks: [ Plot.frame(), Plot.waffleY(data, { fx: (d) => d.columnName, fy: (d) => `row group ${d.rowGroupIndex}`, y: (d) => -1, fill: (d) => "normal", gap: 3, stroke: "black", strokeWidth: 0.5 }) ] })31:T5ca,function createParquetChart({ metadata, offsetIndexes }) { const chart = html`

`; let header; for (const [rowGroupIndex, rowGroup] of metadata.row_groups.entries()) { if (!header) { const headerDiv = html`

`; for (const [columnIndex, column] of rowGroup.columns.entries()) { const headerColumnDiv = html`

`; headerColumnDiv.append(column.meta_data.path_in_schema[0]); headerDiv.append(headerColumnDiv); } chart.append(headerDiv); } const rowGroupDiv = html`

`; for (const [columnIndex, column] of rowGroup.columns.entries()) { const columnDiv = html`

`; const { page_locations } = offsetIndexes[rowGroupIndex][columnIndex]; for (const pageLocation of page_locations) { const pageDiv = html`

`; columnDiv.append(pageDiv); } rowGroupDiv.append(columnDiv); } chart.append(rowGroupDiv); } return chart; }32:[] 2e:{"title":"Parquet structure","cells":[{"id":0,"value":"# Parquet structure\n\nA chart representing the elements of a Parquet file: metadata, columns, row groups, data pages. Not to scale in bytes (one square = one data page).\n\nIt requires the Parquet file to have a data page index (see `write_page_index` in [`pyarrow.parquet.write_table`](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.write_table.html)).","pinned":false,"hidden":false,"mode":"md"},{"id":99,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":377,"value":"columnNames.length > 10\n ? md`> ⚠️ The Parquet file contains many columns (${columnNames.length}). The chart is designed to 10 columns or less.`\n : md``","pinned":false,"hidden":false,"mode":"ojs"},{"id":303,"value":"Paste a Parquet file URL, or upload a file.","pinned":false,"hidden":false,"mode":"md"},{"id":292,"value":"viewof url = fileInput({ initialUrl })","pinned":false,"hidden":false,"mode":"ojs"},{"id":414,"value":"---\n\n_statistics_","pinned":false,"hidden":false,"mode":"md"},{"id":397,"value":"Plot.plot({\n caption: \"Distribution of data page sizes (bytes)\",\n y: { grid: true },\n x: { label: \"Data page bytes\" },\n height: 150,\n marks: [\n Plot.rectY(\n dataPageSizes,\n Plot.binX({ y: \"count\" }, { x: (d) => d, thresholds: 50 })\n ),\n Plot.ruleY([0])\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":255,"value":"---\n\n_data_","pinned":false,"hidden":false,"mode":"md"},{"id":5,"value":"initialUrl = \"https://huggingface.co/datasets/severo/pq_reproduction/resolve/main/index/no_cdc/none/orca.parquet\"\n\n// \"https://raw.githubusercontent.com/apache/parquet-testing/master/data/data_index_bloom_encoding_stats.parquet\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":11,"value":"arrayBuffer = fetch(url).then((d) => d.arrayBuffer())","pinned":false,"hidden":false,"mode":"ojs"},{"id":13,"value":"metadata = hyparquet.parquetMetadata(arrayBuffer)","pinned":false,"hidden":false,"mode":"ojs"},{"id":15,"value":"offsetIndexes = {\n try {\n return metadata.row_groups.map((rowGroup) =>\n rowGroup.columns.map((column) => {\n if (\n column.offset_index_offset === undefined ||\n column.offset_index_length === undefined\n )\n throw new Error(\"No data page index\");\n const offsetIndexOffset = Number(column.offset_index_offset);\n const offsetIndexLength = Number(column.offset_index_length);\n const offsetIndexArrayBuffer = arrayBuffer.slice(\n offsetIndexOffset,\n offsetIndexOffset + offsetIndexLength\n );\n const offsetIndexReader = {\n view: new DataView(offsetIndexArrayBuffer),\n offset: 0\n };\n return hyparquet.readOffsetIndex(offsetIndexReader);\n })\n );\n } catch {\n return undefined;\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":184,"value":"columnNames = metadata.row_groups[0].columns.map(\n ({ meta_data }) => meta_data.path_in_schema[0]\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":214,"value":"numRowGroups = metadata.row_groups.length","pinned":false,"hidden":false,"mode":"ojs"},{"id":95,"value":"data = offsetIndexes ? prepareData({ url }) : undefined","pinned":false,"hidden":false,"mode":"ojs"},{"id":390,"value":"dataPageSizes = offsetIndexes.flatMap((group) =>\n group.flatMap(({ page_locations }) =>\n page_locations.flatMap(({ compressed_page_size }) => compressed_page_size)\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":252,"value":"---\n\n_code_","pinned":false,"hidden":false,"mode":"md"},{"id":289,"value":"import {fileInput} from \"@mbostock/file-input\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"hyparquet = import(\"https://cdn.jsdelivr.net/npm/hyparquet/+esm\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":89,"value":"async function prepareData({ url }) {\n const response = await fetch(url);\n const arrayBuffer = await response.arrayBuffer();\n const data = [];\n for (const [rowGroupIndex, rowGroup] of metadata.row_groups.entries()) {\n for (const [columnIndex, column] of rowGroup.columns.entries()) {\n const columnName = column.meta_data.path_in_schema[0];\n const { page_locations } = offsetIndexes[rowGroupIndex][columnIndex];\n for (const [pageIndex, pageLocation] of page_locations.entries()) {\n data.push({ rowGroupIndex, columnName, columnIndex, pageIndex });\n }\n }\n }\n return data;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":218,"value":"---\n\n_trash bin_","pinned":false,"hidden":false,"mode":"md"},{"id":48,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":64,"value":"createParquetChart({ metadata, offsetIndexes })","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}