${feature._traceId}`,\n svg`<g fill=none stroke-width=1 stroke=black>${feature.features.map(\n (f) => svg`<path d=\"${geoPath(f)}\" pathLength=${pathLength} />`\n )}`\n );\n return img;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":875,"value":"function featureFromTrace(xml, validTypes = [\"LineString\"]) {\n const dom = new DOMParser().parseFromString(xml, \"text/xml\");\n const json = toGeoJSON.gpx(dom);\n const valid = new Set(validTypes);\n\n json.features = json.features.filter((f) => {\n return valid.has(f.geometry.type);\n });\n if (!json.features.length) return null;\n return json;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":463,"value":"randomTraces = {\n more;\n const total = 100;\n const concurrency = 10;\n\n const abort = resolvablePromise();\n invalidation.then(abort.resolve);\n\n const valids = validsWithSize.filter((d) => d[1] < 10000); // all files smaller than 10kb\n const ids = Array.from(new Set(d3.shuffle(valids).map(([id]) => id)));\n let i = 0;\n const results = [];\n\n const next = async () => {\n if (abort.resolved || i >= ids.length) return;\n if (results.length >= total) return abort.resolve();\n const id = ids[i++];\n try {\n const xml = await fetchTrace(id, { invalidated: abort });\n if (xml) {\n const json = featureFromTrace(xml);\n if (json) {\n json._traceId = id;\n results.push(json);\n }\n }\n } catch (err) {\n console.warn(`Error for trace ID ${id}: ${err}`);\n }\n return next();\n };\n\n mutable isLoading = true;\n await Promise.all(Array.from({ length: concurrency }, next));\n mutable isLoading = false;\n return results;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":873,"value":"$32","pinned":true,"hidden":false,"mode":"ojs"},{"id":872,"value":"// A promise that can be resolved externally, and has a flag for its resolution status.\n// API:\n// - *promise*.resolve()\n// - *promise*.resolved\nfunction resolvablePromise() {\n let resolve,\n resolved = false;\n const promise = new Promise((res) => {\n resolve = res;\n });\n return Object.defineProperties(promise, {\n resolve: {\n value: (value = undefined) => {\n resolved = true;\n return resolve(value);\n }\n },\n resolved: {\n get: () => resolved\n }\n });\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":0,"value":"feedURL = \"https://www.openstreetmap.org/traces/rss\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":630,"value":"mutable isLoading = true","pinned":true,"hidden":false,"mode":"ojs"},{"id":152,"value":"traces = Promise.all(\n items.map((e) => fetch(e.dataURL).then((res) => res.text()))\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":11,"value":"items = {\n const feed = await parseRSSCORS(feedURL);\n return feed.items.map((item) => {\n const linksplit = item.link.split(\"/\");\n return {\n ...item,\n dataURL: `https://www.openstreetmap.org/trace/${\n linksplit[linksplit.length - 1]\n }/data`\n };\n });\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":663,"value":"validsWithSize = FileAttachment(\"validIdsSize.json\").json()","pinned":true,"hidden":false,"mode":"ojs"},{"id":13,"value":"import { fetchp, parseRSSCORS } from \"@radames/hello-rss-parser\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":44,"value":"toGeoJSON = import(\"@tmcw/togeojson\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":677,"value":"import { Scrubber } from \"@mbostock/scrubber\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":467,"value":"fflate = import(\"https://cdn.skypack.dev/fflate?min\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":494,"value":"bzip2 = import(\"https://cdn.skypack.dev/bzip2\")","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"} </article></body></html>

31:T493,# Day 5 - OpenStreetMap GPS Traces OpenStreetMaps has a [GPS traces repository](https://www.openstreetmap.org/traces). The RSS feed only shows the 20 latest uploads. It seems like the millions of public GPS traces IDs follow a predictable and incremental pattern; thus, we can make good guesses and fetch a bunch of them. https://www.openstreetmap.org/trace/ID/data. I thought it would be cool to visualize random traces in a grid, they all are random, but they have unique and interesting shapes. To get the traces data, I randomly search valid GPS traces IDs. I've pre-fetched valid GPS traces IDs for this notebook and filtered the files with less than 10kb, so the experience is smooth here. There are traces with too many points, and the fetch size could reach [hundreds of MBs](https://help.openstreetmap.org/questions/3280/how-many-gps-traces-are-in-osm) **todo:** The other way to retrieve GPS traces is to provide a lat/long bounding box https://wiki.openstreetmap.org/wiki/API_v0.6#GPS_traces. The response will be pages of 5000 points aggregated points for that specific bbox. *thanks [Fabian](https://observablehq.com/@mootari) for all the improvements*32:T565,async function fetchTrace(id, { invalidated = invalidation } = {}) { const controller = new AbortController(); invalidated.then(() => controller.abort()); return fetch(`https://www.openstreetmap.org/trace/${id}/data`, { signal: controller.signal }) .then((res) => (res.status !== 200 ? null : res)) .catch((err) => null) .then((res) => (res ? getText(res) : null)) .catch((err) => { if (err.name === "AbortError") return null; throw err; }); function getText(res) { const mime = res.headers.get("Content-Type"); switch (mime) { // sometimes it's pure XML text (e.g. ID 3826980) case "application/gpx+xml": return res.text(); // sometimes it's GZIP (e.g. ID 3878474) case "application/x-gzip": return res.arrayBuffer().then((buffer) => { const decompressed = fflate.decompressSync(new Uint8Array(buffer)); return fflate.strFromU8(decompressed); }); // e.g. ID 3853356 //case "application/x-zip": // todo case "application/x-bzip2": return res.arrayBuffer().then((buffer) => { const decompressed = bzip2.simple( bzip2.array(new Uint8Array(buffer)) ); return fflate.strFromU8(decompressed); }); default: throw Error(`unhandled MIME type: ${mime}`); } } }33:[{"id":"5695dc6908c83eb940aabf4ddfdc793d66aaa56b9c00bbffb203ec908364012fa285e320c93b459ab613cc15717b3867b212c426e6d2802214ffc9c11aa88318","href":"https://static.observableusercontent.com/files/5695dc6908c83eb940aabf4ddfdc793d66aaa56b9c00bbffb203ec908364012fa285e320c93b459ab613cc15717b3867b212c426e6d2802214ffc9c11aa88318","name":"validIdsSize.json","type":"application/json","lastModified":1636065588921.006,"size":752500}] 2f:{"title":"Day 5 - OpenStreetMap GPS Traces","cells":[{"id":7,"value":"$31","pinned":false,"hidden":false,"mode":"md"},{"id":549,"value":"viewof more = Inputs.button(\"more\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":680,"value":"viewof time = Scrubber(d3.range(1, 101), {\n delay: 10,\n loop: true,\n alternate: true\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":642,"value":"{\n if (isLoading) {\n return html`

Loading ...

`;\n }\n return html``;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":173,"value":"grid = {\n const id = DOM.uid(\"grid\").id;\n const view = html`

\n ${images}\n \n `;\n yield view;\n for await (const t of Generators.input(viewof time)) {\n view.style.setProperty(\"--t\", t);\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":877,"value":"images = randomTraces.map((feature) => getTraceDrawing(feature))","pinned":false,"hidden":false,"mode":"ojs"},{"id":61,"value":"function getTraceDrawing(feature, options = {}) {\n const {\n width: w = 100,\n height: h = w,\n padding: pad = 5,\n pathLength = 100 // Reference length for 100%\n } = options;\n\n const extent = [\n [pad, pad],\n [w - pad, h - pad]\n ];\n const projection = d3.geoIdentity().fitExtent(extent, feature);\n const geoPath = d3.geoPath().projection(projection);\n\n const img = DOM.svg(w, h);\n img.append(\n svg`${feature._traceId}`,\n svg`<g fill=none stroke-width=1 stroke=black>${feature.features.map(\n (f) => svg`<path d=\"${geoPath(f)}\" pathLength=${pathLength} />`\n )}`\n );\n return img;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":875,"value":"function featureFromTrace(xml, validTypes = [\"LineString\"]) {\n const dom = new DOMParser().parseFromString(xml, \"text/xml\");\n const json = toGeoJSON.gpx(dom);\n const valid = new Set(validTypes);\n\n json.features = json.features.filter((f) => {\n return valid.has(f.geometry.type);\n });\n if (!json.features.length) return null;\n return json;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":463,"value":"randomTraces = {\n more;\n const total = 100;\n const concurrency = 10;\n\n const abort = resolvablePromise();\n invalidation.then(abort.resolve);\n\n const valids = validsWithSize.filter((d) => d[1] < 10000); // all files smaller than 10kb\n const ids = Array.from(new Set(d3.shuffle(valids).map(([id]) => id)));\n let i = 0;\n const results = [];\n\n const next = async () => {\n if (abort.resolved || i >= ids.length) return;\n if (results.length >= total) return abort.resolve();\n const id = ids[i++];\n try {\n const xml = await fetchTrace(id, { invalidated: abort });\n if (xml) {\n const json = featureFromTrace(xml);\n if (json) {\n json._traceId = id;\n results.push(json);\n }\n }\n } catch (err) {\n console.warn(`Error for trace ID ${id}: ${err}`);\n }\n return next();\n };\n\n mutable isLoading = true;\n await Promise.all(Array.from({ length: concurrency }, next));\n mutable isLoading = false;\n return results;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":873,"value":"$32","pinned":true,"hidden":false,"mode":"ojs"},{"id":872,"value":"// A promise that can be resolved externally, and has a flag for its resolution status.\n// API:\n// - *promise*.resolve()\n// - *promise*.resolved\nfunction resolvablePromise() {\n let resolve,\n resolved = false;\n const promise = new Promise((res) => {\n resolve = res;\n });\n return Object.defineProperties(promise, {\n resolve: {\n value: (value = undefined) => {\n resolved = true;\n return resolve(value);\n }\n },\n resolved: {\n get: () => resolved\n }\n });\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":0,"value":"feedURL = \"https://www.openstreetmap.org/traces/rss\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":630,"value":"mutable isLoading = true","pinned":true,"hidden":false,"mode":"ojs"},{"id":152,"value":"traces = Promise.all(\n items.map((e) => fetch(e.dataURL).then((res) => res.text()))\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":11,"value":"items = {\n const feed = await parseRSSCORS(feedURL);\n return feed.items.map((item) => {\n const linksplit = item.link.split(\"/\");\n return {\n ...item,\n dataURL: `https://www.openstreetmap.org/trace/${\n linksplit[linksplit.length - 1]\n }/data`\n };\n });\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":663,"value":"validsWithSize = FileAttachment(\"validIdsSize.json\").json()","pinned":true,"hidden":false,"mode":"ojs"},{"id":13,"value":"import { fetchp, parseRSSCORS } from \"@radames/hello-rss-parser\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":44,"value":"toGeoJSON = import(\"@tmcw/togeojson\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":677,"value":"import { Scrubber } from \"@mbostock/scrubber\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":467,"value":"fflate = import(\"https://cdn.skypack.dev/fflate?min\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":494,"value":"bzip2 = import(\"https://cdn.skypack.dev/bzip2\")","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"} </article></body></html>