30:T45d,// https://observablehq.com/@mbostock/authentication-simulator viewof user = { try { // no popup, token already available in session const login = await authenticate(); const hello = md`

✅ Hello, ${login.user.name}! You’re authenticated.

`; hello.value = login; return hello; } catch (error) { // try popup const form = html`

`; const clearBtn = html``; const field = viewof textarea.querySelector("textarea"); const fileInputControl = viewof fileInput.querySelector("input[type=file]"); const fileInputStatus = viewof fileInput.querySelector("#status"); const updateButtons = () => { const hasText = field?.value?.trim().length > 0; const hasFile = fileInputControl?.files?.length > 0; const hasPaths = hasText || hasFile; submitBtn.disabled = !hasPaths; clearBtn.disabled = !hasPaths; }; updateButtons(); field?.addEventListener("input", () => { updateButtons(); if (fileInputStatus && fileInputStatus.textContent.trim().length > 0) { fileInputStatus.textContent = "⚠️ Text area edited after file upload. Contents may no longer match uploaded file."; } }); fileInputControl?.addEventListener("change", updateButtons); submitBtn.onclick = () => { submitBtn.disabled = true; // Prevent double submission viewof textarea.dispatchEvent(new CustomEvent("input")); // Sync value mutable submitCounter += 1; }; clearBtn.onclick = () => { if (field) field.value = ""; field.dispatchEvent(new Event("input")); if (fileInputControl) fileInputControl.value = ""; if (fileInputStatus) fileInputStatus.textContent = ""; updateButtons(); mutable submitCounter += 1; }; container.append(submitBtn, clearBtn); container.value = null; return container; }33:T468,zipWarning = { // Trigger recompute on submit or clear submitCounter; const fileCount = paths?.length ?? 0; const estimatedBytes = fileCount * 1.1 * 1024 * 1024; // 1MB average GNSS obs file size. const exceedsLimit = estimatedBytes > 2 * 1024 * 1024 * 1024; if (exceedsLimit) { return html`

Your estimated download size: ~${bytesToString( estimatedBytes )} of data across ${d3.format(",")(fileCount)} files.

⚠️ Note: You are attempting to zip ~${bytesToString( estimatedBytes )} of data across ${d3.format(",")(fileCount)} files. This exceeds browser memory limits (~2GB). Our Zip file archive generator will use multiple zip files (chunks) to accommodate your download. We recommend using command line tools for large downloads.

`; } return html`

Your estimated download size: ~${bytesToString( estimatedBytes )} of data across ${d3.format(",")(fileCount)} files.

`; }34:Tfb3,downloadButton = { if ((paths?.length ?? 0) === 0) return md``; return html`

${chunks(paths).map((chunk, i) => { const container = html`

`; const controls = html`

`; const estimatedSizeBytes = chunk.length * 1 * 1024 * 1024; const estimatedSizeText = bytesToString(estimatedSizeBytes); const createBtn = html``; const cancelBtn = html``; const progressBar = html``; const progressText = html`

`; let downloadedBytes = 0; const totalBytesEstimate = estimatedSizeBytes; let downloadAborted = false; cancelBtn.onclick = () => { downloadAborted = true; mutable abortController?.abort(); mutable abortController = null; cancelBtn.style.visibility = "hidden"; createBtn.disabled = false; progressText.textContent = "❌ Canceled"; }; createBtn.onclick = async () => { if (mutable activeDownload) { progressText.textContent = "⏳ Please wait for the current zip to finish."; progressText.style.color = "gray"; progressText.style.fontStyle = "italic"; return; } mutable activeDownload = true; createBtn.disabled = true; cancelBtn.style.visibility = "visible"; progressBar.style.display = "block"; progressBar.value = 0; progressText.textContent = ""; downloadedBytes = 0; downloadAborted = false; try { mutable reset *= -1; const zipBlob = await blob(chunk, { onProgress: (bytes) => { downloadedBytes += bytes; const percent = Math.min( (downloadedBytes / totalBytesEstimate) * 100, 100 ); progressBar.value = percent; progressText.textContent = `${bytesToString( downloadedBytes )} of ~${bytesToString(totalBytesEstimate)}`; } }); if (downloadAborted) { mutable activeDownload = false; return; } const zipReadyLabel = html`✅ Zip Ready`; const downloadBtn = html``; const url = URL.createObjectURL(zipBlob); downloadBtn.onclick = () => { const a = html``; document.body.appendChild(a); a.click(); a.remove(); URL.revokeObjectURL(url); mutable activeDownload = false; }; const rightControls = html`

`; rightControls.append(zipReadyLabel, downloadBtn); controls.append(rightControls); } catch (err) { progressText.textContent = err.name === "AbortError" ? "❌ Canceled" : "⚠️ Error during zip."; console.error(err); } finally { cancelBtn.style.visibility = "hidden"; createBtn.disabled = false; mutable activeDownload = false; } }; controls.append(createBtn, cancelBtn); container.append(controls, progressBar, progressText); return container; })}

`; }35:T49c,savePathsButtonCell = { const saveButton = html`

1) (e.g., file_urls_to_download.txt)

`; const warning = html`

`; saveButton.onclick = () => { //const text = textarea?.value ?? ""; // assumes `textarea` is another cell const text = viewof textarea.value ?? ""; const lines = text .trim() .split("\n") .map((d) => d.trim()) .filter(Boolean); if (lines.length === 0) { warning.textContent = "⚠️ Please submit at least one valid URL."; return; } warning.textContent = ""; const blob = new Blob([lines.join("\n")], { type: "text/plain" }); const url = URL.createObjectURL(blob); const a = html``; document.body.appendChild(a); a.click(); a.remove(); URL.revokeObjectURL(url); }; const container = html`

`; container.append(saveButton, warning); return container; }36:Tca9,viewof examplesWithToken = { const div = html`

`; const wgetButton = html``; const curlButton = html``; div.append(wgetButton, curlButton); let tokenPromise = null; let token = null; let wgetShown = false; let curlShown = false; let wgetContainer, curlContainer; const getToken = async () => { if (!tokenPromise) tokenPromise = authenticate(); token = await tokenPromise; return token; }; const createCommandBlock = (command) => { const pre = html`

${command}

`; const copyBtn = html``; copyBtn.onclick = () => { navigator.clipboard.writeText(command).then(() => { copyBtn.innerHTML = ``; setTimeout(() => { copyBtn.innerHTML = ``; }, 2000); }); }; const container = html`

`; container.append(pre, copyBtn); return container; }; wgetButton.onclick = async () => { if (!wgetShown) { const { accessToken } = await getToken(); const command = `wget -i file_urls_to_download.txt --header "Authorization: Bearer ${accessToken}"`; wgetContainer = createCommandBlock(command); div.appendChild(wgetContainer); wgetButton.textContent = "Hide wget command"; wgetShown = true; } else { div.removeChild(wgetContainer); wgetButton.textContent = "Show wget command"; wgetShown = false; } }; curlButton.onclick = async () => { if (!curlShown) { const { accessToken } = await getToken(); const command = `xargs -n 1 curl -O --header "Authorization: Bearer ${accessToken}" < file_urls_to_download.txt`; curlContainer = createCommandBlock(command); div.appendChild(curlContainer); curlButton.textContent = "Hide curl command"; curlShown = true; } else { div.removeChild(curlContainer); curlButton.textContent = "Show curl command"; curlShown = false; } }; div.value = () => token; return div; }37:T95c,// Fetch multiple files concurrently using the given access token. // Asynchronously yields bytes fetched until all fetches have resolved (or failed) async function authFetch( paths, // Array of EarthScope resources that require authenticated download. { concurrency = 8, // Default, download serially. Use higher whole numbers to fetch in parallel. controller, // Call controller.abort() accessToken, max = MAX_SIZE, progress = (bytes) => { // Called each time a fetch resolves with current cumulative fetched bytes. } } = {} ) { let limit = pLimit.default(concurrency); // Default is serial (1) let total = 0; let aborted = false; // Issue all fetch requests limited by pLimit. return Promise.all( paths.map( ( url // All files are returned in the order corresponding to the input parths. ) => limit(() => { if (!aborted) { // Paths are fetched limited by the concurrency level and are not fetched in strict order. return fetch(url, { signal: controller.signal, // Abort signal to cancel in-flight fetch if user selection changes. credentials: "include", headers: { Authorization: `Bearer ${accessToken}` } }) .then(async (response) => { if (response.status == STATUS_OK) { let buffer = await response.arrayBuffer(); total += buffer.byteLength; progress(buffer.byteLength); if (!aborted && total > max) { aborted = true; limit.clearQueue(); // Do not send any more fetches. controller.abort(); // Cancel in-flight fetches. console.log(`Aborted fetches after ${total} bytes`); } return buffer; } else { // Push file path instead of buffer to indicate which paths were not fetched. console.log("failed to fetch:", url); return url; } }) .catch((e) => { console.log(e); return url; // Not fetched, but push path string to record failure. }); } else { return url; } }) ) ); }38:T469,async function blob(chunkedPaths = paths, options = {}) { const { onProgress } = options; mutable reset != -1; let zip = new jszip(); const controller = new AbortController(); mutable abortController = controller; invalidation.then(() => { console.log("abort initiated from invalidation"); controller.abort(); }); try { const all = await authFetch(chunkedPaths, { controller, accessToken: user.accessToken, concurrency, progress: (bytes) => { mutable filesDownloaded += 1; mutable totalBytes += bytes; if (onProgress) { onProgress(bytes); } } }); all.filter(Boolean).forEach((buffer, i) => { zip.file(chunkedPaths[i].split("/").slice(-1), buffer); }); zip.file("README.md", readme()); const buffer = await zip.generateAsync({ type: "arraybuffer" }); return new Blob([buffer], { type: "arraybuffer" }); } catch (error) { if (controller.signal.aborted) { console.warn("Download aborted."); } else { throw error; } } finally { mutable abortController = null; } }39:[] 2e:{"title":"Data Downloader v2","cells":[{"id":0,"value":"title = md`# Data Downloader v2\nFetch multiple NSF NGF Facility data archive files by pasting in paths, after searching and filtering data with our **GNSS Data Access Notebook**.\\n\n**Notes:** \n* You need to register and sign in with your EarthScope User Profile to access data.\\n\n* You may need to turn off browser pop-up/tracking security options for this site in order to log in.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":635,"value":"viewof step1 = htl.html`

1) Sign in

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":56,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":107,"value":"visit = html`Visit your EarthScope Consortium User Profile to sign-in or sign-out without pop-ups enabled.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":589,"value":"loadPaths = html`

2) Load Paths

Paste a list of paths here that you copied to your clipboard from other Data Search tools or upload paths from a file.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":46,"value":"viewof textarea = Inputs.textarea({\n label: \"A) Paste paths here or load from file below\",\n width: 960,\n rows: 12,\n placeholder: `https://gage-data.earthscope.org/archive/gnss/rinex/obs/2023/102/jro11020.23d.Z\nhttps://gage-data.earthscope.org/archive/gnss/rinex/obs/2023/101/jro11010.23d.Z\nhttps://gage-data.earthscope.org/archive/gnss/rinex/obs/2023/100/jro11000.23d.Z\n⋮`\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":604,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":603,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":587,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":586,"value":"chooseDownloadMethod = html`

3) Choose Download Method

You have two general methods for downloading your list of files. For large downloads (> 2GB) and to automate downloads, we recommend using command line tools or scripts. For one-off downloads, the Zip file archive(s) created here are convenient.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":585,"value":"optionA = html`

Option A) Create and Download Zip Archive(s)

Best for smaller downloads (< 2GB). It will create multiple zip archives for downloads of more than 2GB of data.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":97,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":583,"value":"optionB = html`

Option B) Command Line/Scripting

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":580,"value":"$35","pinned":false,"hidden":false,"mode":"ojs"},{"id":579,"value":"copying_description = html`

2) Run wget or curl with your paths file by copying an example below and running it on your computer. You may need to update the path to the file_urls_to_download.txt file, or change your working dir to the folder where you downloaded the file. If you are on a windows PC you may need to adjust the commands accordingly.

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":578,"value":"$36","pinned":false,"hidden":false,"mode":"ojs"},{"id":41,"value":"---\n## Appendix","pinned":false,"hidden":false,"mode":"md"},{"id":639,"value":"pageview","pinned":false,"hidden":false,"mode":"ojs"},{"id":458,"value":"MAX_SIZE = 2 * 1e9","pinned":false,"hidden":false,"mode":"ojs"},{"id":577,"value":"mutable activeDownload = false","pinned":false,"hidden":false,"mode":"ojs"},{"id":576,"value":"chunkedPaths = chunks(paths)","pinned":false,"hidden":false,"mode":"ojs"},{"id":575,"value":"function chunks(\n paths,\n maxBytes = 2 * 1024 * 1024 * 1024,\n avgFileSize = 1.2 * 1024 * 1024\n) {\n const result = [];\n let currentChunk = [];\n let currentSize = 0;\n\n for (const file of paths ?? []) {\n if (currentSize + avgFileSize > maxBytes && currentChunk.length > 0) {\n result.push(currentChunk);\n currentChunk = [];\n currentSize = 0;\n }\n currentChunk.push(file);\n currentSize += avgFileSize;\n }\n\n if (currentChunk.length > 0) result.push(currentChunk);\n\n return result;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":574,"value":"mutable chunkProgress = new Map()","pinned":false,"hidden":false,"mode":"ojs"},{"id":454,"value":"concurrency = 8","pinned":false,"hidden":false,"mode":"ojs"},{"id":435,"value":"mutable reset = 1","pinned":false,"hidden":false,"mode":"ojs"},{"id":410,"value":"baseUrl = document.baseURI.split(\"?\")[0]","pinned":false,"hidden":false,"mode":"ojs"},{"id":352,"value":"mutable filesDownloaded = 0","pinned":false,"hidden":false,"mode":"ojs"},{"id":284,"value":"mutable totalBytes = 0","pinned":false,"hidden":false,"mode":"ojs"},{"id":160,"value":"$37","pinned":false,"hidden":false,"mode":"ojs"},{"id":84,"value":"function readme() {\n return `© ${d3.timeFormat(\"%Y\")(new Date())} EarthScope Consortium\n\nContact brooks.mershon@earthscope.org with questions, concerns, or bug reports. You can also leave comments on the notebook that created this file at the link above.\n\nTo download this data again visit:\n\nhttps://observablehq.com/@earthscope/auth-fetch\n\nand copy paste the following filepaths:\n\n${textarea}`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":79,"value":"$38","pinned":false,"hidden":false,"mode":"ojs"},{"id":62,"value":"STATUS_OK = 200","pinned":false,"hidden":false,"mode":"ojs"},{"id":598,"value":"mutable submitCounter = 0","pinned":false,"hidden":false,"mode":"ojs"},{"id":51,"value":"paths = {\n submitCounter; // reactivity trigger\n\n const raw = viewof textarea?.value ?? \"\";\n return raw.trim().length ? raw.trim().split(\"\\n\").filter(Boolean) : [];\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":16,"value":"mutable fileDownloadSignal = 1","pinned":false,"hidden":false,"mode":"ojs"},{"id":570,"value":"mutable abortController = null","pinned":false,"hidden":false,"mode":"ojs"},{"id":34,"value":"// TODO (brooks): risky import\npLimit = import(\"https://cdn.skypack.dev/p-limit@4.0.0?min\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":59,"value":"import { signin, authenticate } from \"@earthscope/auth0\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":92,"value":"jszip = require(\"jszip@3/dist/jszip.min.js\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":99,"value":"import { download } from \"@mbostock/lazy-download\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":298,"value":"import { bytesToString } from \"@earthscope/bytes\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":637,"value":"import {pageview} from \"@earthscope/pageview\"","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W39","stdlib":"1","titling":"auto"}