31:T420,function* parsePage({ find, follow, capture }) { const updated = find("time").attr("datetime"); // skip parsing if content is stale if (new Date(updated) <= new Date(cache.updated)) { yield { ...cache, updated }; // use the cached data return { ...cache, updated }; // and do not proceed } /** * Sections look like: * label + div > (h4 + ul)* */ let schedules = []; for (const $regionLabel of find("label.dr-h")) { // eg. "back bay" const region = trim($regionLabel.text()); for (const $locationHeader of find("h4", $regionLabel.next())) { // eg. "copley square" const location = trim($locationHeader.text()); for (const $schedule of find("li", $locationHeader.next())) { // schedules look like "datetime : truck name" const [schedule, truck] = $schedule.text().split(":").map(trim); schedules.push({ truck, region, location, schedule }); } } } yield { updated, schedules }; }32:T822,/** * The https://github.com/epegzz/node-scraper/ modified to work in Observable * - removed NodeJS dependencies */ async function* scrape(requestUrlOrArgs, parserFunction, context) { // Allow passing either request url or request args const { url: scrapeUrl, ...fetchOpts } = { ...(typeof requestUrlOrArgs === "string" ? { url: requestUrlOrArgs } : requestUrlOrArgs) }; const res = await fetch(scrapeUrl, { ...fetchOpts, method: "get" }).then( (res) => res.text() ); const $ = cheerio.load(res); const find = (queryString, node) => { const find = node ? node.find.bind(node) : $; const result = find(queryString); // make result iterable using for ... of statement result[Symbol.iterator] = function* () { const elements = []; result.each((i, element) => elements.push($(element))); for (const element of elements) { yield element; } }; return result; }; const followUpRequests = []; const follow = (url, parser, context) => { followUpRequests.push({ requestArgs: { ...fetchOpts, url: `${scrapeUrl}${url}` }, parser, context }); }; // follows a link but collects all results and returns them as an array const capture = async function (url, parserFunction, newContext) { const iterator = scrape( { ...fetchOpts, url: `${scrapeUrl}${url}` }, parserFunction, newContext || context ); const collectResults = []; for await (const collectResult of iterator) { collectResults.push(collectResult); } return collectResults; }; yield* parserFunction({ request: { url: scrapeUrl }, response: { body: res.body, status: res.status }, find, follow, capture, context }); if (followUpRequests.length) { for (const followUpRequest of followUpRequests) { yield* scrape( followUpRequest.requestArgs, followUpRequest.parser || parserFunction, followUpRequest.context || context ); } } }33:[{"id":"828903641754c6ab559b58812bea2a271ead4fd4b5ae4e0104b4747aebd94678cae211ca3c1fc513693970318e559435f766776d7c4816b7661a125657e5955e","href":"https://static.observableusercontent.com/files/828903641754c6ab559b58812bea2a271ead4fd4b5ae4e0104b4747aebd94678cae211ca3c1fc513693970318e559435f766776d7c4816b7661a125657e5955e","name":"schedule data.json","type":"application/json","lastModified":1664294746373.608,"size":5647},{"id":"c3aea9b1c362ef6c2a83b372878f480b0f5adf544a7e79fac8c30fb2c86abfbd8f7aca72693bb3a56a74d26ccc64cad84e76a7ad76aff3729f4436bd3f0257f2","href":"https://static.observableusercontent.com/files/c3aea9b1c362ef6c2a83b372878f480b0f5adf544a7e79fac8c30fb2c86abfbd8f7aca72693bb3a56a74d26ccc64cad84e76a7ad76aff3729f4436bd3f0257f2","name":"schedules@1.json","type":"application/json","lastModified":1664228846966.811,"size":6211},{"id":"d353320f0bbb69978931bd7e404ccfb5192c95c68dfa9190633a01e1d38d3833c56ca0db4d590de93a55030e32b37f31b547563c999c6fede79dc677a7dcec97","href":"https://static.observableusercontent.com/files/d353320f0bbb69978931bd7e404ccfb5192c95c68dfa9190633a01e1d38d3833c56ca0db4d590de93a55030e32b37f31b547563c999c6fede79dc677a7dcec97","name":"schedule data (2).json","type":"application/json","lastModified":1664294855005.787,"size":5576},{"id":"b9819fc21dbf7cbb6922ad2c5fc77f12e99d6d6b9eed71b11ef2107a1640520489f3b414b74a192520192ed95229a03d0c56838a958cd94098855895af7b1397","href":"https://static.observableusercontent.com/files/b9819fc21dbf7cbb6922ad2c5fc77f12e99d6d6b9eed71b11ef2107a1640520489f3b414b74a192520192ed95229a03d0c56838a958cd94098855895af7b1397","name":"schedule data (3).json","type":"application/json","lastModified":1664298278677.255,"size":5615}] 2f:{"title":"Boston Food Truck Schedule","cells":[{"id":0,"value":"# Boston Food Truck Schedule\nQuick drill-down and search for the foodtruck ${html`map and schedule`}.","pinned":false,"hidden":false,"mode":"md"},{"id":507,"value":"viewof search = Inputs.search(data)","pinned":false,"hidden":false,"mode":"ojs"},{"id":62,"value":"md`_OR filter by place and time (${htl.html` {\n e.preventDefault();\n setSearch(viewof search, defaultOptions.search);\n set(viewof regions, defaultOptions.regions);\n set(viewof days, defaultOptions.days);\n}}>reset`}):_`","pinned":false,"hidden":false,"mode":"ojs"},{"id":510,"value":"viewof regions = Inputs.select(\n d3.group(data, (d) => d.region),\n {\n multiple: true,\n label: html`Regions
cmd-click to deselect or select multiple`,\n value: defaultOptions.regions,\n valueof: ([k, v]) => k\n }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":513,"value":"viewof days = Inputs.select(\n d3.group(data, (d) => d.schedule.split(\",\")[0]),\n {\n label: html`Day`,\n multiple: true,\n value: defaultOptions.days,\n valueof: ([k, v]) => k\n }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":65,"value":"viewof table = Inputs.table(\n search.length < data.length\n ? search\n : data.filter(\n ({ region, location, schedule }) =>\n (regions.length === 0 || regions.includes(region)) &&\n (days.length === 0 ||\n days.some((dayFilter) => schedule.includes(dayFilter)))\n ),\n { columns: [\"truck\", \"schedule\", \"location\", \"region\"] }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":364,"value":"defaultOptions = ({\n search: \"\",\n regions: [\"Back Bay\", \"South End\"],\n days: [`${d3.timeFormat(\"%A\")(new Date())}s`]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":67,"value":"---\n## Data\nThis notebook uses cached data to not spam _boston.gov_ with automation. Web scraping [has been a problem](https://www.boston.gov/news/web-analytics-cautionary-tale) for them in the past, so we're trying to be a good citizen.\n\nComment on this cell if the data is significantly out of date, but I'll check periodically ${useCached ? \"\" : new Date(data.updated) > new Date(cache.updated) ? notice(md`Schedule updated since last captured`, \"warn\") : notice(md`still up to date - last updated ${data.updated}`)}","pinned":false,"hidden":false,"mode":"md"},{"id":13,"value":"data = {\n let source;\n if (useCached) {\n source = cache.schedules;\n source.updated = cache.updated;\n } else {\n // call scraper with URL and parser\n const scrapeResults = scrape(\n `https://sgregson-cors-anywhere.herokuapp.com/${THE_URL}`,\n parsePage\n );\n\n // I'd like to simply return [...scrapeResults] but it's not iterable ???\n let results = [];\n for await (const result of scrapeResults) {\n results.push(result);\n }\n\n source = results[0].schedules;\n source.updated = results[0].updated;\n }\n\n return source;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":250,"value":"viewof useCached = Inputs.toggle({\n label: html`Use Cached Data
(2022-09-27)`,\n value: true\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"THE_URL = \"https://www.boston.gov/departments/small-business-development/city-boston-food-trucks-schedule\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"$31","pinned":true,"hidden":false,"mode":"ojs"},{"id":10,"value":"// cleans up some HTML-related whitespace\ntrim = (str) => str.replace(/[\\s\\n]*/, \" \").trim()","pinned":true,"hidden":false,"mode":"ojs"},{"id":403,"value":"cache = FileAttachment(\"schedule data (3).json\").json()","pinned":false,"hidden":false,"mode":"ojs"},{"id":265,"value":"## Externals","pinned":false,"hidden":false,"mode":"md"},{"id":300,"value":"// scraping utilities. basically the node-scraper module and its dependencies","pinned":true,"hidden":false,"mode":"ojs"},{"id":3,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":190,"value":"url = import(\"https://cdn.skypack.dev/url@0.11.0?min\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":179,"value":"axios = import(\"https://cdn.skypack.dev/axios@0.27.2?min\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":187,"value":"cheerio = import(\"https://cdn.skypack.dev/cheerio@1.0.0-rc.12?min\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":342,"value":"// input state control","pinned":true,"hidden":false,"mode":"ojs"},{"id":343,"value":"import { set } from \"@observablehq/synchronized-inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":534,"value":"// Search input API varies\nfunction setSearch(input, value) {\n input.query = value;\n input.dispatchEvent(new Event(\"input\", { bubbles: true }));\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":302,"value":"// styling","pinned":true,"hidden":false,"mode":"ojs"},{"id":150,"value":"import {styles} from \"@sgregson/styles\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":216,"value":"styles","pinned":false,"hidden":false,"mode":"ojs"},{"id":567,"value":"import {notice} from \"@sgregson/components\"","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"}