30:T4f5,htl.html`

${result_file.name}

${file_type}

${ file_type.startsWith("image") ? display_image(loaded_file) : file_type.startsWith("font") ? await display_font(loaded_file) : file_type === "application/java-vm" ? await display_java(loaded_file) : file_type === "application/json" ? await display_json(loaded_file) : htl.html`

          ${await result_file.text()}

` }

`31:T411,Pretty simple, right? The first tricky thing is that in the zip format is read from the end of the file. This is a historic artifact for when zip files were written on slow floppy disks. ^{[[Wikipedia]](https://en.wikipedia.org/wiki/ZIP_%28file_format%29#Structure)}. The basis for a zip file is finding the end of central directory record, and from there read all the central directory headers. From the above image it may seem like the local file header entries are neatly in order, but, again for historic reasons, there is no assumption for the order or existance of local file header. A way that our zip might actually be structure is with our local file headers in a random order and with even an unremoved entry still hanging around, with junk data in between that should be ignored by the zip reader.32:T5c8,### Central Directory File Header offset | length | description ------------| ---------- | ------------- 0 | 4 | Central directory file header signature = 0x02014b50 4 | 2 | Version made by 6 | 2 | Version needed to extract (minimum) 8 | 2 | General purpose bit flag 10 | 2 | Compression method 12 | 2 | File last modification time 14 | 2 | File last modification date 16 | 4 | CRC-32 of uncompressed data 20 | 4 | Compressed size (or 0xffffffff for ZIP64) 24 | 4 | Uncompressed size (or 0xffffffff for ZIP64) 28 | 2 | File name length (n) 30 | 2 | Extra field length (m) 32 | 2 | File comment length (k) 34 | 2 | Disk number where file starts (or 0xffff for ZIP64) 36 | 2 | Internal file attributes 38 | 4 | External file attributes 42 | 4 | Relative offset of local file header (or 0xffffffff for ZIP64). This is the number of bytes between the start of the first disk on which the file occurs, and the start of the local file header. This allows software reading the central directory to locate the position of the file inside the ZIP file. 46 | n | File name 46+n | m | Extra field 46+n+m | k | File comment33:Td40,central_directory_protocol = { let base_protocol = combine([ { protocol: exact(string.encode("PK\x01\x02")) }, { name: "version_made_by", protocol: bytes.uint16le }, { name: "version_required", protocol: bytes.uint16le }, // { name: "raw_flags", protocol: bytes.uint16le }, { name: "flags_1", protocol: flags([ "encrypted", "compression_1", "compression_2", "crc32_at_the_end", null, "is_compressed_patched_data", "strong_encryption", null ]) }, { name: "flags_2", protocol: flags([null, null, null, "use_utf8", null, null, null, null]) }, { name: "compression", protocol: switch_protocol( bytes.uint16le, new Map([ [0, "uncompressed"], [8, "deflate"] ]) ) }, { name: "modification_time", protocol: bytes.uint16le }, { name: "modification_date", protocol: bytes.uint16le }, { name: "crc_32", protocol: bytes.uint32le }, { name: "compressed_size", protocol: bytes.uint32le }, { name: "uncompressed_size", protocol: bytes.uint32le }, { name: "name_length", protocol: bytes.uint16le }, { name: "extra_field_length", protocol: bytes.uint16le }, { name: "comment_length", protocol: bytes.uint16le }, { name: "start_disk_number", protocol: bytes.uint16le }, { name: "internal_attributes", protocol: bytes.uint16le }, { name: "external_attributes", protocol: bytes.uint32le }, { name: "offset", protocol: bytes.uint32le } ]); return just_repeat({ decode: (buffer) => { /// First we parse the data that has a set length, /// that contains the information about the length of the following attributes let [ { name_length, compression, flags_1, flags_2, extra_field_length, comment_length, modification_time, modification_date, ...data }, base_offset ] = base_protocol.decode(buffer); /// let [{ file_name, extra_field, comment }, rest_offset] = combine([ { name: "file_name", protocol: with_exact_length(name_length, string) }, { name: "extra_field", protocol: with_exact_length( extra_field_length, just_repeat( combine([ { name: "header", protocol: bytes.uint16le }, { name: "value", protocol: with_length(bytes.uint16le, uint8array) } ]) ) ) }, { name: "comment", protocol: with_exact_length(comment_length, string) } ]).decode(buffer.subarray(base_offset)); return [ { ...data, compression: compression, last_modified: parseDosDateTime(modification_date, modification_time), file_name_length: name_length, file_name: file_name, extra_field_length: extra_field_length, extra_field: extra_field, comment_length: comment_length, comment: comment, flags: new Set([...flags_1, ...flags_2]) }, base_offset + rest_offset ]; } }); }34:T5b0,extra_fields_central = ({ extended_timestamp: { header: 0x5455, protocol: { decode: (buffer) => { let [flags, initial_offset] = bytes.uint8.decode(buffer); let MUTABLE_buffer = buffer.slice(initial_offset); let result = { modtime: null, actime: null, crtime: null }; if (flags & 0b00000001) { console.log("1"); let [modtime, modtime_offset] = bytes.uint32le.decode(MUTABLE_buffer); MUTABLE_buffer = MUTABLE_buffer.slice(modtime_offset); result.modtime = new Date(modtime * 1000); } if (flags & 0b00000010) { console.log("2"); let [actime, actime_offset] = bytes.uint32le.decode(MUTABLE_buffer); MUTABLE_buffer = MUTABLE_buffer.slice(actime_offset); result.actime = new Date(actime * 1000); } if (flags & 0b00000100) { console.log("4"); let [crtime, crtime_offset] = bytes.uint32le.decode(MUTABLE_buffer); MUTABLE_buffer = MUTABLE_buffer.slice(crtime_offset); result.crtime = new Date(crtime * 1000); } return [result, buffer.length]; } } }, unix: { header: 0x7875, protocol: combine([ { name: "version", protocol: exact(bytes.uint8.encode(1)) }, { name: "uid", protocol: with_length(bytes.uint8, dynamic_byte_le) }, { name: "guid", protocol: with_length(bytes.uint8, dynamic_byte_le) } ]) } })35:T5b1,extra_fields_local = ({ extended_timestamp: { header: 0x5455, protocol: { decode: (buffer) => { let [applied_flags, initial_offset] = flags([ "modtime", "actime", "crtime" ]).decode(buffer); let MUTABLE_buffer = buffer.slice(initial_offset); let result = { modtime: null, actime: null, crtime: null }; if (applied_flags.has("modtime")) { let [modtime, modtime_offset] = bytes.uint32le.decode(MUTABLE_buffer); MUTABLE_buffer = MUTABLE_buffer.slice(modtime_offset); result.modtime = new Date(modtime * 1000); } if (applied_flags.has("actime")) { let [actime, actime_offset] = bytes.uint32le.decode(MUTABLE_buffer); MUTABLE_buffer = MUTABLE_buffer.slice(actime_offset); result.actime = new Date(actime * 1000); } if (applied_flags.has("crtime")) { let [crtime, crtime_offset] = bytes.uint32le.decode(MUTABLE_buffer); MUTABLE_buffer = MUTABLE_buffer.slice(crtime_offset); result.crtime = new Date(crtime * 1000); } return [result, buffer.length]; } } }, unix: { header: 0x7875, protocol: combine([ { protocol: exact(bytes.uint8.encode(1)) }, { name: "uid", protocol: with_length(bytes.uint8, dynamic_byte_le) }, { name: "guid", protocol: with_length(bytes.uint8, dynamic_byte_le) } ]) } })36:T62a,local_file_header = { let [{ name_length, extra_field_length, ...data }, offset] = local_file_header_protocol.decode( await blob_to_bytes(zip_blob.slice(selected_file.offset).slice(0, 30)) ); let more_bytes = await blob_to_bytes( zip_blob.slice( selected_file.offset + offset, selected_file.offset + offset + name_length + extra_field_length ) ); let [file_name, offset_2] = string.decode(more_bytes.slice(0, name_length)); let [extra_field, offset_end] = just_repeat( combine([ { name: "header", protocol: bytes.uint16le }, { name: "value", protocol: with_length(bytes.uint16le, uint8array) } ]) ).decode(more_bytes.slice(offset_2).slice(0, extra_field_length)); let extra_fields = extra_field.map((field) => { try { let matching = Object.entries(extra_fields_local).find( ([key, value]) => value.header === field.header ); if (matching == null) { return { type: "unknown", header: field.header, header_hex: field.header.toString(16), value: field.value }; } else { return { type: matching[0], result: matching[1].protocol.decode(field.value)[0] }; } } catch (error) { return { type: "error", id: field.header, value: field.value, error: error }; } }); return [ { ...data, file_name: file_name, extra_fields: extra_fields }, selected_file.offset + offset + offset_2 + offset_end ]; }37:T51f,result_file = new LazyFile( { byteLength: selected_file.uncompressed_size, stream(start, end) { // ... read the file contents from somewhere and return a ReadableStream let data = zip_blob .slice( local_file_header[1], local_file_header[1] + selected_file.compressed_size ) .stream(); if (selected_file.compression === "deflate") { if (STRICT_CRC) { return concat_streams([ ReadableStreamFrom([gzip_header_protocol.encode({})]), data, ReadableStreamFrom([ gzip_trailer_protocol.encode({ crc32: selected_file.crc_32, uncompressed_size: selected_file.uncompressed_size }) ]) ]) .pipeThrough(new DecompressionStream("gzip")) .pipeThrough(new ByteSliceStream(start, end)); } else { return data .pipeThrough(new DecompressionStream("deflate-raw")) .pipeThrough(new ByteSliceStream(start, end)); } } else { return data.pipeThrough(new ByteSliceStream(start, end)); } } }, selected_file.file_name, { lastModified: selected_file.last_modified } // { type: Mime.lookup(selected_file.file_name) ?? null } )38:T40f,with_length = (length_protocol, payload_protocol) => { return { encode: (value) => { let payload = payload_protocol.encode(value); let length_bits = length_protocol.encode(payload.length); return concat([length_bits, payload]); // return encode_with_varint_length(protocol.encode(value)); }, decode: (buffer) => { try { let [length, offset] = length_protocol.decode(buffer); let [value, value_offset] = payload_protocol.decode( buffer.subarray(offset, offset + length) ); if (length !== value_offset) { console.log(`length_protocol.decode:`, length_protocol.decode); console.log(`buffer:`, buffer.subarray(0, offset)); console.log(`length:`, length); // prettier-ignore throw new Error(`Decoded too little (expected ${length}, but only decoded ${value_offset})`); } return [value, offset + length]; } catch (e) { // e.stack = stack; throw e; } } }; }39:T4c1,bytes = ({ uint8: { encode: (value) => { return new Uint8Array([value]); }, decode: (buffer) => { return [buffer[0], 1]; } }, uint16: { encode: (value) => { let buffer = new ArrayBuffer(2); new DataView(buffer).setUint16(0, value); return new Uint8Array(buffer); }, decode: (buffer) => { return [to_dataview(buffer).getUint16(0), 2]; } }, uint16le: { encode: (value) => { let buffer = new ArrayBuffer(2); new DataView(buffer).setUint16(0, value, true); return new Uint8Array(buffer); }, decode: (buffer) => { return [to_dataview(buffer).getUint16(0, true), 2]; } }, uint32le: { encode: (value) => { let buffer = new ArrayBuffer(4); new DataView(buffer).setUint32(0, value, true); return new Uint8Array(buffer); }, decode: (buffer) => { return [to_dataview(buffer).getUint32(0, true), 4]; } }, uint32: { encode: (value) => { let buffer = new ArrayBuffer(4); new DataView(buffer).setUint32(0, value); return new Uint8Array(buffer); }, decode: (buffer) => { return [to_dataview(buffer).getUint32(0), 4]; } } })3a:T4b8,hexview = (bytes) => { let style = { // lineHeight: 0, margin: "0px", padding: "0px", fontFamily: "monospace", width: `${16 * 2 + 15}ch` }; if (bytes.length < 16 * 5) { let thing = htl.html`

${_.chunk(bytes, 16).map( (chunk) => htl.html`

${hex(chunk)}

` )}

`; thing.value = bytes; return thing; } else { let thing = htl.html`

${_.chunk(bytes.slice(0, 32), 16).map( (chunk) => htl.html`

${hex(chunk)}

` )} ... ${humanreadable_bytes(bytes.length - 64)} more ... ${_.chunk(bytes.slice(-32), 16).map( (chunk) => htl.html`

${hex(chunk)}

` )}

`; thing.value = bytes; return thing; } }3b:[{"id":"03c3ddc2894159722263547f50bf71e1823310bd67cc986d2bbcc59c7bf9b101dc3346121c96f68795bc0173062d8fc5dd376d0b96e093439c805405b49305ed","href":"https://static.observableusercontent.com/files/03c3ddc2894159722263547f50bf71e1823310bd67cc986d2bbcc59c7bf9b101dc3346121c96f68795bc0173062d8fc5dd376d0b96e093439c805405b49305ed","name":"how-it-might-actually-look.svg","type":"image/svg+xml","lastModified":1740413019511.108,"size":40167},{"id":"a64606d4d1f9a57751455d53eb8be2f303ff500aab8eed3bbf1e5c60fa246c7367df47e25fe4a4c632f0e55ad67cda4b9723fabc9f25f8b693501b1813b055ac","href":"https://static.observableusercontent.com/files/a64606d4d1f9a57751455d53eb8be2f303ff500aab8eed3bbf1e5c60fa246c7367df47e25fe4a4c632f0e55ad67cda4b9723fabc9f25f8b693501b1813b055ac","name":"zip-basic-structure.excalidraw.svg","type":"image/svg+xml","lastModified":1740413026747.806,"size":27510},{"id":"cd37288dfd563f7b0c82f09fc9678ba59b339f2a4dea1629be6873e72ee5184eeb1fafee2f3baceffd7b851608343ed5e1d7d0f4b6526cccd614fa433e80187e","href":"https://static.observableusercontent.com/files/cd37288dfd563f7b0c82f09fc9678ba59b339f2a4dea1629be6873e72ee5184eeb1fafee2f3baceffd7b851608343ed5e1d7d0f4b6526cccd614fa433e80187e","name":"import-sample-single-version.zip","type":"application/zip","lastModified":1740416120868.116,"size":2318},{"id":"e823cb076e4161d0414b7d1bd95c67e619920bf80be0cf0698f7968f88916ecb20c761ebe8341bcd8c415ae563ea7bad0eb35785b83c36c2fec96eba3b1cbcb0","href":"https://static.observableusercontent.com/files/e823cb076e4161d0414b7d1bd95c67e619920bf80be0cf0698f7968f88916ecb20c761ebe8341bcd8c415ae563ea7bad0eb35785b83c36c2fec96eba3b1cbcb0","name":"find-central-directory.excalidraw.svg","type":"image/svg+xml","lastModified":1740489136457.469,"size":32111}] 2e:{"title":"Zip reader","cells":[{"id":0,"value":"# Zip reader\n\nZip reader using `CompressionStream` and other web protocols.\n\n### Resources\n- [PKWare APPNOTE.txt](https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT)\n- [ZIP file format - Wikipedia](https://en.wikipedia.org/wiki/ZIP_%28file_format%29)\n- [Writing a simple browser zip file decompressor with CompressionStreams](https://dev.to/ndesmic/writing-a-simple-browser-zip-file-decompressor-with-compressionstreams-5che) by ndesmic\n- [\"How are zlib, gzip and zip related? What do they have in common and how are they different?\" answered by Mark Adler](https://stackoverflow.com/questions/20762094/how-are-zlib-gzip-and-zip-related-what-do-they-have-in-common-and-how-are-they/20765054#20765054)\n- [Zip64 - Go Big Or Go Home](https://blog.yaakov.online/zip64-go-big-or-go-home/)\n- [Extra field documentation](https://libzip.org/specifications/extrafld.txt)\n","pinned":false,"hidden":false,"mode":"md"},{"id":1081,"value":"## Working Example\n\nThe code until the next header is not related to loading the zip, only to showing it in a nice way!","pinned":false,"hidden":false,"mode":"md"},{"id":4,"value":"viewof upload_file = Inputs.file()","pinned":false,"hidden":false,"mode":"ojs"},{"id":1264,"value":"zip_blob = upload_file == null ? example_zip.blob() : upload_file.blob()","pinned":false,"hidden":false,"mode":"ojs"},{"id":439,"value":"viewof selected_file = Inputs.select(\n _.sortBy(central_directory, (x) => x.file_name),\n {\n label: \"File\",\n format: (x) => x.file_name,\n disabled: central_directory.filter((x) => x.file_name.endsWith(\"/\"))\n // value: teams.find((t) => t.file === \"Warriors\")\n }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":659,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":1062,"value":"loaded_file = new File([await result_file.arrayBuffer()], result_file.name, {\n type: file_type\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1025,"value":"display_image = (file) => {\n let url = URL.createObjectURL(file);\n let element = htl.html`

\n \n

`;\n Inputs.disposal(element).then(() => {\n URL.revokeObjectURL(url);\n });\n return element;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1005,"value":"display_font = async (file) => {\n let url = URL.createObjectURL(file);\n let font_id = `font-${(Math.random() * 1_000_000).toFixed(0)}`;\n\n const font = new FontFace(font_id, `url(${url})`);\n await font.load();\n document.fonts.add(font);\n\n let text = `Whereas disregard and contempt for human rights have resulted. 0 1 2 3 4 5 6 7 8 9 ? . ( ) | ' \"`;\n\n let element = htl.html`

\n ${text}\n

`;\n Inputs.disposal(element).then(() => {\n URL.revokeObjectURL(url);\n document.font.delete(font);\n });\n return element;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1621,"value":"/// This is fun!\n/// https://docs.oracle.com/javase/specs/jvms/se6/html/ClassFile.doc.html\njava_class_protocol = combine([\n { protocol: exact(new Uint8Array([0xca, 0xfe, 0xba, 0xbe])) },\n { name: \"minor_version\", protocol: bytes.uint16 },\n { name: \"major_version\", protocol: bytes.uint16 },\n { name: \"constant_pool_count\", protocol: bytes.uint16 },\n { name: \"rest\", protocol: uint8array }\n])","pinned":false,"hidden":false,"mode":"ojs"},{"id":1619,"value":"display_java = async (file) => {\n let x = 0xcafebabe;\n\n let { rest, ...data } = consume(\n java_class_protocol,\n new Uint8Array(await file.arrayBuffer())\n );\n\n return htl.html`

${JSON.stringify(\n    data,\n    null,\n    2\n  )}

`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1655,"value":"display_json = async (file) => {\n let string = await file.text();\n\n return htl.html`

${JSON.stringify(\n    JSON.parse(string),\n    null,\n    2\n  )}

`;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":849,"value":"file_type = Mime.lookup(result_file.name) || \"binary\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":1083,"value":"","pinned":false,"hidden":false,"mode":"html"},{"id":1260,"value":"example_zip = FileAttachment(\"import-sample-single-version.zip\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":1095,"value":"## How does it work?\n\n> 📘 To keep this post simple I am **not** considering:\n> - Multiple disks/files\n> - Encryption\n> - ZIP64\n\nMore than anything this post is a dive into the zip format itself.\n\nTo start of, we need to know the general format for a ZIP file:","pinned":false,"hidden":false,"mode":"md"},{"id":1110,"value":"","pinned":false,"hidden":false,"mode":"html"},{"id":1175,"value":"$31","pinned":false,"hidden":false,"mode":"md"},{"id":1194,"value":"","pinned":false,"hidden":false,"mode":"html"},{"id":505,"value":"### Find our end of central directory record\n\nWe start by finding end of central directory record signature by reading from the end of the file.\n\nZip's header signatures are all variations of `PK` follow by some bytes. For the end of central directory record this ends up being `0x50 0x4b 0x05 0x06` (`\"PK\"` = `0x50 0x4b`)\n\n","pinned":false,"hidden":false,"mode":"md"},{"id":147,"value":"#### End Of Central Directory Protocol\n\n> This is taken verbatim from wikipedia, but I checked it with the APPNOTE.txt\n\noffset | length | thing\n------ | ------ | ------------------------------------------------\n0 | 4 | End of central directory signature = 0x06054b50\n4 | 2 | Number of this disk (or 0xffff for ZIP64)\n6 | 2 | Disk where central directory starts (or 0xffff for ZIP64)\n8 | 2 | Number of central directory records on this disk (or 0xffff for ZIP64)\n10 | 2 | Total number of central directory records (or 0xffff for ZIP64)\n12 | 4 | Size of central directory (bytes) (or 0xffffffff for ZIP64)\n16 | 4 | Offset of start of central directory, relative to start of archive (or 0xffffffff for ZIP64)\n20 | 2 | Comment length (n)\n22 | n | Comment","pinned":false,"hidden":false,"mode":"md"},{"id":1224,"value":"For this post I'm ignoring the ability to story zips over multiple disks, so we assume\n- `\"Number of this disk\"` is `0`\n- `\"Disk where central directory starts\"` is `0`\n- `\"Number of central directory records on this disk\"` is the same as `\"Total number of central directory records\"`\n\nI want this reader to be \"web-savy\", so we can't just load in the whole file and read from the end. Instead, I want to load as little as possible. For now, I'm going to assume the `\"Comment length\"` is `0`, so we can load the last 22 bytes of a file and find the end of directory signature at the start of it.\n\nIf we can't find the signature at the start of these 22 bytes we can load progressively more until we find the the signature.\n\n> ⚠️ Wikipedia mentions the comment length is limited to 65,535 bytes, however in APPNOTE.txt I can only find a reference to the maximum size of a central directory header.","pinned":false,"hidden":false,"mode":"md"},{"id":71,"value":"END_OF_DIRECTORY_SIGNATURE = new TextEncoder().encode(\"PK\\u{05}\\u{06}\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":508,"value":"expected_end_of_directory = new Uint8Array(\n await zip_blob.slice(-22).arrayBuffer()\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":1242,"value":"> ⚠️ Currently an error is thrown if there is a comment present in the end of central directory record","pinned":false,"hidden":false,"mode":"md"},{"id":74,"value":"viewof end_of_directory_header_bytes = hexview(\n _.isEqual(expected_end_of_directory.slice(0, 4), END_OF_DIRECTORY_SIGNATURE)\n ? expected_end_of_directory\n : error(\"No End-Of-Directory found in the last 22 bytes\")\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1256,"value":"### Parse the end of central directory record\n\nFor parsing binary protocols I wrote [a bunch of utility functions](#protocols).\n\nThe most important part to understand is that `combine()` will parse the bytes sequentially with the protocols that you provide it, and saves them to an object for every `name: ...` defined. This makes a rather neat way of quickly defining basic sequences, like our end of central directory record!\n\n> 📘 Here we see the first appearance of `le`, standing for \"Little Endian\". Turns out the web generally uses \"Big Endian\", but windows computer (and the zip format) use \"Little Endian\". Of course we can't agree on even that... [Endianness](https://developer.mozilla.org/en-US/docs/Glossary/Endianness)","pinned":false,"hidden":false,"mode":"md"},{"id":234,"value":"end_of_central_directory_protocol = combine([\n { protocol: exact(string.encode(\"PK\\x05\\x06\")) },\n {\n name: \"disk\",\n protocol: bytes.uint16le\n },\n {\n name: \"start_disk\",\n protocol: bytes.uint16le\n },\n {\n name: \"records_on_disk\",\n protocol: bytes.uint16le\n },\n {\n name: \"records\",\n protocol: bytes.uint16le\n },\n {\n name: \"size_of_central_directory\",\n protocol: bytes.uint32le\n },\n {\n name: \"offset_of_central_directory\",\n protocol: bytes.uint32le\n },\n {\n name: \"comment\",\n protocol: with_length(bytes.uint16le, string)\n }\n])","pinned":true,"hidden":false,"mode":"ojs"},{"id":1350,"value":"And use this protocol to decode the bytes we know contain our end of central directory record (because it starts with the signature).\n\nClick on the \"▶︎ Object\" to expand the data.","pinned":false,"hidden":false,"mode":"md"},{"id":236,"value":"end_of_central_directory = consume(\n end_of_central_directory_protocol,\n end_of_directory_header_bytes\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1322,"value":"Huray!\n\nWe parsed the first thing we need from our zip file!\n\nThis doesn't give us too much informations, most of it is to support a multi-disk zip file, which we will not cover in this post.\n\nThe important properties we care about are `size_of_central_directory` and `offset_of_central_directory`. Using those we can load all the bytes that contain the central directory headers.","pinned":false,"hidden":false,"mode":"md"},{"id":1330,"value":"findCentralDirectoryExcalidraw = FileAttachment(\n \"find-central-directory.excalidraw.svg\"\n).image({\n width: 500\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1336,"value":"### Parse the central directory headers\n\nNow we are getting somewhere!\n\nWe'll get the bytes that our end of central directory record has pointed us to.","pinned":false,"hidden":false,"mode":"md"},{"id":1338,"value":"viewof central_directory_bytes = hexview(\n new Uint8Array(\n await zip_blob\n .slice(\n end_of_central_directory.offset_of_central_directory,\n end_of_central_directory.offset_of_central_directory +\n end_of_central_directory.size_of_central_directory\n )\n .arrayBuffer()\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":316,"value":"$32","pinned":false,"hidden":false,"mode":"md"},{"id":306,"value":"$33","pinned":true,"hidden":false,"mode":"ojs"},{"id":320,"value":"central_directory = central_directory_protocol.decode(\n new Uint8Array(\n await zip_blob\n .slice(\n end_of_central_directory.offset_of_central_directory,\n end_of_central_directory.offset_of_central_directory +\n end_of_central_directory.size_of_central_directory\n )\n .arrayBuffer()\n )\n)[0]","pinned":false,"hidden":false,"mode":"ojs"},{"id":366,"value":"### Local file header\n\nOffset | Length | Description\n------------- | ------------ | --------------\n0 | 4 | Local file header signature = 0x04034b50 (PK♥♦ or \"PK\\3\\4\")\n4 | 2 | Version needed to extract (minimum)\n6 | 2 | General purpose bit flag\n8 | 2 | Compression method; e.g. none = 0, DEFLATE = 8 (or \"\\0x08\\0x00\")\n10 | 2 | File last modification time\n12 | 2 | File last modification date\n14 | 4 | CRC-32 of uncompressed data\n18 | 4 | Compressed size (or 0xffffffff for ZIP64)\n22 | 4 | Uncompressed size (or 0xffffffff for ZIP64)\n26 | 2 | File name length (n)\n28 | 2 | Extra field length (m)\n30 | n | File name\n30+n | m | Extra field","pinned":false,"hidden":false,"mode":"md"},{"id":368,"value":"local_file_header_protocol = combine([\n { protocol: exact(string.encode(\"PK\\x03\\x04\")) },\n { name: \"version_required\", protocol: bytes.uint16le },\n { name: \"flags\", protocol: bytes.uint16le },\n {\n name: \"compression\",\n protocol: switch_protocol(\n bytes.uint16le,\n new Map([\n [0, \"uncompressed\"],\n [8, \"deflate\"]\n ])\n )\n },\n { name: \"modification_time\", protocol: bytes.uint16le },\n { name: \"modification_date\", protocol: bytes.uint16le },\n { name: \"crc_32\", protocol: bytes.uint32le },\n { name: \"compressed_size\", protocol: bytes.uint32le },\n { name: \"uncompressed_size\", protocol: bytes.uint32le },\n { name: \"name_length\", protocol: bytes.uint16le },\n { name: \"extra_field_length\", protocol: bytes.uint16le }\n ])","pinned":true,"hidden":false,"mode":"ojs"},{"id":840,"value":"selected_file","pinned":true,"hidden":false,"mode":"ojs"},{"id":796,"value":"selected_file_entry_size = selected_file.offset +\n 30 +\n selected_file.file_name_length +\n selected_file.extra_field_length","pinned":true,"hidden":false,"mode":"ojs"},{"id":962,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":913,"value":"$35","pinned":false,"hidden":false,"mode":"ojs"},{"id":370,"value":"$36","pinned":false,"hidden":false,"mode":"ojs"},{"id":1510,"value":"### Gzip\n\nIf we want to be really awesome, we can wrap our deflate-raw data in some gzip data, and the `DecompressionStream` will then verify the CRC32 for us!\n\n#### Resources\n- [Gzip file format spec](https://www.rfc-editor.org/rfc/rfc1952#section-2.2)\n- [WhatWG Compression spec](https://compression.spec.whatwg.org/#supported-formats) - more readable than I expected!\n- [Walkthrough a full gzip file](https://commandlinefanatic.com/cgi-bin/showarticle.cgi?article=art053)","pinned":false,"hidden":false,"mode":"md"},{"id":1415,"value":"gzip_header_protocol = combine([\n /// ID\n { protocol: exact(new Uint8Array([0x1f, 0x8b])) },\n /// Compression: 8 = DEFLATE\n { protocol: exact(bytes.uint8.encode(8)) },\n /// Flags - Sets some extra flags that allow for extra data.\n /// ....... If we set it to 0 no extra data is expected, which is great!\n { protocol: exact(bytes.uint8.encode(0)) },\n /// MTIME - Modification time in Unix format\n /// IGNORED BY DECOMPRESSIONSTREAM\n { name: \"modification_time\", protocol: bytes.uint32le },\n /// XFL - 0x00000010 is slowest compression, 0x00000100 is fastest\n /// IGNORED BY DECOMPRESSIONSTREAM\n { name: \"extra_flags\", protocol: bytes.uint8 },\n /// OS 255 = UNKNOWN\n { name: \"OS\", protocol: bytes.uint8 }\n])","pinned":true,"hidden":false,"mode":"ojs"},{"id":1446,"value":"gzip_trailer_protocol = combine([\n { name: \"crc32\", protocol: bytes.uint32le },\n { name: \"uncompressed_size\", protocol: bytes.uint32le }\n])","pinned":true,"hidden":false,"mode":"ojs"},{"id":1451,"value":"viewof STRICT_CRC = Inputs.toggle({ label: \"Strict CRC32\", value: true })","pinned":false,"hidden":false,"mode":"ojs"},{"id":618,"value":"$37","pinned":true,"hidden":false,"mode":"ojs"},{"id":269,"value":"## Protocols","pinned":false,"hidden":false,"mode":"md"},{"id":1236,"value":"consume = (protocol, buffer) => {\n let [result, offset] = protocol.decode(buffer);\n if (offset !== buffer.length) {\n throw new Error(`consume() left with ${buffer.length - offset} bytes`);\n }\n return result;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":194,"value":"concat = (buffers) => {\n let length = _.sumBy(buffers, (buffer) => buffer.length);\n let result = new Uint8Array(length);\n let offset = 0;\n for (let buffer of buffers) {\n result.set(buffer, offset);\n offset = offset + buffer.length;\n }\n return result;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":222,"value":"to_dataview = (buffer) => {\n return new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":964,"value":"flags = (values) => {\n if (values.length > 8) {\n throw new Error(\"For now flags() is capped at 8 values (1 byte)\");\n }\n return {\n decode: (buffer) => {\n let [value, offset] = bytes.uint8.decode(buffer);\n\n let present_flags = new Set();\n\n for (let [bit, result] of [\n [0b00000001, values[0]],\n [0b00000010, values[1]],\n [0b00000100, values[2]],\n [0b00001000, values[3]],\n [0b00010000, values[4]],\n [0b00100000, values[5]],\n [0b01000000, values[6]],\n [0b10000000, values[7]]\n ]) {\n console.log(\"bit:\", bit.toString(2));\n console.log(\"value & bit:\", value & bit);\n console.log(\"result:\", result);\n if (result != null && value & bit) {\n present_flags.add(result);\n console.log(\"present_flags:\", present_flags);\n }\n }\n return [present_flags, offset];\n }\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":894,"value":"function just_repeat(protocol) {\n return {\n decode: (buffer) => {\n let offset = 0;\n let entries = [];\n while (offset !== buffer.length) {\n let [data, new_offset] = protocol.decode(buffer.subarray(offset));\n entries.push(data);\n offset = offset + new_offset;\n }\n\n return [entries, offset];\n }\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":248,"value":"uint8array = ({\n encode: (value) => value,\n decode: (buffer) => [buffer, buffer.length]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1299,"value":"with_exact_length = (length_in_bytes, payload_protocol) => {\n return {\n encode: (value) => {\n let payload = payload_protocol.encode(value);\n return concat([length_in_bytes, payload]);\n },\n decode: (buffer) => {\n try {\n let [value, value_offset] = payload_protocol.decode(\n buffer.subarray(0, length_in_bytes)\n );\n\n if (length_in_bytes !== value_offset) {\n console.log(`buffer:`, buffer.subarray(0, value_offset));\n console.log(`length_in_bytes:`, length_in_bytes);\n // prettier-ignore\n throw new Error(`Decoded too little (expected ${length_in_bytes}, but only decoded ${value_offset})`);\n }\n\n return [value, length_in_bytes];\n } catch (e) {\n // e.stack = stack;\n throw e;\n }\n }\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":210,"value":"$38","pinned":false,"hidden":false,"mode":"ojs"},{"id":993,"value":"dynamic_byte_le = ({\n decode: (buffer) => {\n if (buffer.lengh === 1) {\n return bytes.uint8.decode(buffer);\n } else if (buffer.length === 2) {\n return bytes.uint16le.decode(buffer);\n } else if (buffer.length === 4) {\n return bytes.uint32le.decode(buffer);\n } else {\n throw new Error(\"Too big a byte\");\n }\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":214,"value":"$39","pinned":false,"hidden":false,"mode":"ojs"},{"id":204,"value":"string = ({\n encode: (value) => new TextEncoder().encode(value),\n decode: (buffer) => [new TextDecoder().decode(buffer), buffer.length]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":460,"value":"switch_protocol = (protocol, values) => {\n return {\n encode: (value) => {\n let entry = Array.from(values.entries()).find((x) => x[1] === value);\n if (entry == null) {\n throw new Error(`Invalid switch encode value: ${value}`);\n }\n return protocol.encode(entry[0]);\n },\n decode: (buffer) => {\n let [value, offset] = protocol.decode(buffer);\n let mapped_value = values.get(value);\n if (mapped_value == null) {\n throw new Error(`Invalid switch decode value: ${value}`);\n }\n return [mapped_value, offset];\n }\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":322,"value":"exact = (needle) => {\n return {\n encode: () => needle,\n decode: (buffer) => {\n let decoded = buffer.slice(0, needle.length);\n if (!_.isEqual(decoded, needle)) {\n throw new Error(`Expected \"${hex(needle)}\", got \"${hex(decoded)}\"`);\n }\n return [undefined, needle.length];\n }\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":308,"value":"prefilled = (protocol, value) => {\n return {\n encode: () => protocol.encode(value),\n decode: (buffer) => {\n let [decoded, offset] = protocol.decode(buffer);\n if (decoded !== value) {\n throw new Error(`Expected ${value}, got ${decoded}`);\n }\n return [undefined, offset];\n }\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":188,"value":"combine = (parts) => {\n return {\n encode: (value) => {\n return concat(\n parts.map(({ name, protocol }) =>\n name == null\n ? protocol.encode(undefined)\n : protocol.encode(value[name])\n )\n );\n },\n decode: (buffer) => {\n let result = {};\n let offset = 0;\n for (let { name, protocol } of parts) {\n let [value, value_offset] = protocol.decode(buffer.slice(offset));\n if (name != null) result[name] = value;\n offset = offset + value_offset;\n }\n return [result, offset];\n }\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":287,"value":"## Utils","pinned":false,"hidden":false,"mode":"md"},{"id":1384,"value":"read_all = async (stream) => {\n let results = [];\n for await (let x of stream) {\n results.push(x);\n }\n return results;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1371,"value":"concat_streams = (streams) => {\n let output = new TransformStream();\n\n async(async () => {\n try {\n for (let stream of streams) {\n await stream.pipeTo(output.writable, { preventClose: true });\n }\n output.writable.close();\n } catch (error) {\n console.log(\"error:\", error);\n /// Can ignore the error, it will bubble up through our output stream\n }\n });\n\n return output.readable;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1380,"value":"async = async (async) => async()","pinned":false,"hidden":false,"mode":"ojs"},{"id":599,"value":"LazyFileModule = await import(\"https://cdn.jsdelivr.net/npm/@mjackson/lazy-file/+esm\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":606,"value":"LazyFile = LazyFileModule.LazyFile","pinned":false,"hidden":false,"mode":"ojs"},{"id":589,"value":"Mime = {\n let Mime = await import(\"https://cdn.jsdelivr.net/npm/mime-types/+esm\");\n Mime.types[\"ts\"] = \"application/typescript\";\n return Mime;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":76,"value":"_ = require(\"lodash\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":37,"value":"function hex(arraybuffer) {\n return Array.from(arraybuffer)\n .map((x) => x.toString(16).padStart(2, \"0\"))\n .join(\" \");\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":397,"value":"function ReadableStreamFrom(buffers) {\n return new ReadableStream({\n start(controller) {\n for (let buffer of buffers) {\n controller.enqueue(buffer);\n }\n controller.close();\n }\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":531,"value":"error = (error) => {\n if (error instanceof Error) {\n throw error;\n } else {\n throw new Error(error);\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":636,"value":"class ByteSliceStream extends TransformStream {\n #offsetStart = 0;\n #offsetEnd = 0;\n\n constructor(start = 0, end = Infinity) {\n super({\n start: () => {\n // assert(start >= 0, \"`start` must be greater than 0\");\n end += 1;\n },\n transform: (chunk, controller) => {\n this.#offsetStart = this.#offsetEnd;\n this.#offsetEnd += chunk.byteLength;\n if (this.#offsetEnd > start) {\n if (this.#offsetStart < start) {\n chunk = chunk.slice(start - this.#offsetStart);\n }\n if (this.#offsetEnd >= end) {\n chunk = chunk.slice(0, chunk.byteLength - this.#offsetEnd + end);\n controller.enqueue(chunk);\n controller.terminate();\n } else {\n controller.enqueue(chunk);\n }\n }\n }\n });\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":753,"value":"blob_to_bytes = async (blob) => new Uint8Array(await blob.arrayBuffer())","pinned":false,"hidden":false,"mode":"ojs"},{"id":1549,"value":"$3a","pinned":false,"hidden":false,"mode":"ojs"},{"id":1609,"value":"humanreadable_bytes = (bytes) => {\n if (bytes > 10 ** 3) {\n return `${(bytes / 10 ** 3).toFixed(2)} MB`;\n } else {\n return `${bytes} bytes`;\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":810,"value":"## DOS Date\n\n[Microsoft documentation](https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-dosdatetimetofiletime)\n\n### wFatDate\n\nThe MS-DOS date. The date is a packed value with the following format.\n\nBits | Description\n------- | ---------------\n0-4 | Day of the month (1–31)\n5-8 | Month (1 = January, 2 = February, and so on)\n9-15 | Year offset from 1980 (add 1980 to get actual year)\n\n### wFatTime\n\nThe MS-DOS time. The time is a packed value with the following format.\n\nBits | Description\n------- | ---------------\n0-4 | Second divided by 2\n5-10 | Minute (0–59)\n11-15 | Hour (0–23 on a 24-hour clock)\n","pinned":false,"hidden":false,"mode":"md"},{"id":812,"value":"/// Thanks ChatGPT\nfunction parseDosDateTime(dosDate, dosTime) {\n const day = dosDate & 0x1F; // Bits 0-4\n const month = (dosDate >> 5) & 0x0F; // Bits 5-8\n const year = ((dosDate >> 9) & 0x7F) + 1980; // Bits 9-15\n\n const seconds = (dosTime & 0x1F) * 2; // Bits 0-4 (multiply by 2)\n const minutes = (dosTime >> 5) & 0x3F; // Bits 5-10\n const hours = (dosTime >> 11) & 0x1F; // Bits 11-15\n\n return new Date(year, month - 1, day, hours, minutes, seconds);\n}","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W3b","stdlib":"1","titling":"auto"}