31:T7d8,scrapPage = async function (params, page) { let _ = params; let url = `${_.proxy}${_.url}?lang=${_.lang}&minu=${_.minu}&maxu=${_.maxu}&minki=${_.minki}&minau=${_.minau}&black=${_.black}&creg=${_.creg}&appr=${_.appr}&lcok=${_.lcok}&ord=tusersd`; let htmlString = await fetch(`${url}&p=${page}`).then((res) => res.text()); const clean = (str) => { return str ? str.replace(" ", "") : ""; }; let $ = cheerio.load(htmlString); let nb = $(".ibody").length; let result = []; for (let i = 0; i < nb; i++) { let registrwarn = $(".ibody")[i]?.children[5]?.children[9]?.children[4]?.children[0]?.data; registrwarn = registrwarn == undefined ? "" : ` ${registrwarn}`; result.push({ instance: $(".ihead")[i]?.children[0]?.children[0]?.data, language: $(".ibody")[i]?.children[3]?.children[3]?.children[1]?.data.substr(1), users: +$(".ibody")[i]?.children[3]?.children[5]?.children[1]?.data, active_users_last_month: +$(".ibody")[i]?.children[3]?.children[7]?.children[1]?.data, active_users_last_6_month: +$(".ibody")[i]?.children[3]?.children[9]?.children[1]?.data, known_instances: +$(".ibody")[i]?.children[3]?.children[11]?.children[1]?.data, blacklisted: $(".ibody")[i]?.children[5]?.children[3]?.children[2]?.children[0] ?.data, registration: $(".ibody")[i]?.children[5]?.children[9]?.children[2]?.children[0] ?.data + registrwarn, email: $(".ibody")[i]?.children[5]?.children[11]?.children[2]?.children[0] ?.data // admin_account: // $(".ibody")[i]?.children[13]?.children[1]?.children[1]?.data, // date_of_creation: clean( // $(".ibody")[i]?.children[13]?.children[2]?.children[1]?.data // ), // admin_name: clean( // $(".ibody")[i]?.children[13]?.children[3]?.children[1]?.data // ) //image: $(".iimgc")[i]?.children[0]?.attribs?.src }); } return result; }32:[{"id":"dda53e85bca038e611d5436ebb16ff0f3b7a3fa916b3134402f006ffdc1aa305965348b43767a9d3bf028016f6345d50a209c80b8e5181be1333fbef665ab169","href":"https://static.observableusercontent.com/files/dda53e85bca038e611d5436ebb16ff0f3b7a3fa916b3134402f006ffdc1aa305965348b43767a9d3bf028016f6345d50a209c80b8e5181be1333fbef665ab169","name":"mastodon20221225.csv","type":"text/csv","lastModified":1671998574728.608,"size":710481},{"id":"95814e8df40ffc8a1a16e0d50cf6fd1b881470052127378009bf5c8c2fd789abdc2db9e8bb89d76ebcdf8e98af941af7c9c0c286eb353daa682a72d6cf4b4866","href":"https://static.observableusercontent.com/files/95814e8df40ffc8a1a16e0d50cf6fd1b881470052127378009bf5c8c2fd789abdc2db9e8bb89d76ebcdf8e98af941af7c9c0c286eb353daa682a72d6cf4b4866","name":"mastodon_22221114.csv","type":"text/csv","lastModified":1668546057291.866,"size":442336}] 2f:{"title":"Mastodon instances list","cells":[{"id":0,"value":"# Mastodon instances list","pinned":false,"hidden":false,"mode":"md"},{"id":1164,"value":"There are currently __${intro.instances}__ Mastodon instances. \nSee the list on the [mastodon website](https://mastodon.help/instances).","pinned":false,"hidden":false,"mode":"md"},{"id":1921,"value":"You can now download the data as of December 25, 2022.","pinned":false,"hidden":false,"mode":"md"},{"id":1903,"value":"button(\n await FileAttachment(\"mastodon20221225.csv\").csv(),\n \"mastodon20221225.csv\"\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1918,"value":"Or wait while up-to-date data is being retrieved.","pinned":false,"hidden":false,"mode":"md"},{"id":1183,"value":"$${data.length < intro.instances ? `*Scraping in progress ${Math.round((data.length / intro.instances) * 100) }% (${data.length} / ${intro.instances})*` : button(data, \"mastodon.csv\") }","pinned":false,"hidden":false,"mode":"md"},{"id":1136,"value":"{\n let output;\n\n if (data.length < intro.instances) {\n output = html``;\n } else {\n output = html`Complete 👍`;\n }\n\n return output;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1127,"value":"mastodon = data","pinned":false,"hidden":false,"mode":"ojs"},{"id":1232,"value":"### Appendix","pinned":false,"hidden":false,"mode":"md"},{"id":15,"value":"params = ({\n proxy: \"https://corsproxy.io/?\",\n url: \"https://mastodon.help/instances\",\n lang: 0,\n minu: 0,\n maxu: 1000000000,\n minki: 0,\n minau: 0,\n black: 0,\n creg: 0,\n appr: 0,\n lcok: 0\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":65,"value":"cheerio = require(\"https://bundle.run/cheerio@1.0.0-rc.5\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":1969,"value":"proxyUrl = (url) => `https://corsproxy.io/?${encodeURIComponent(url)}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1967,"value":"getUrl = ({ proxy, url, ...params }, page) =>\n proxyUrl(\n `${url}?${new URLSearchParams({\n ...params,\n ord: \"tusersd\",\n p: page\n })}`\n )","pinned":false,"hidden":false,"mode":"ojs"},{"id":89,"value":"intro = {\n let htmlString = await fetch(getUrl(params, 1)).then((res) => res.text());\n\n let $ = cheerio.load(htmlString);\n let data = $(\".introe\")[0]\n .children.filter((d) => d.type == \"tag\")\n .map((d) => d.children[0].data.replaceAll(\",\", \"\"));\n\n // let data = $(\".introe\")[0]\n // .children.filter((d) => d.type == \"tag\")\n // .map((d) => +d.children[0].data.replaceAll(\".\", \"\"));\n\n return {\n instances: +data[0],\n users: +data[1],\n activelastmonth: +data[2],\n statuses: +data[3],\n nb_pages: Math.floor(data[0] / 10) + 1\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1612,"value":"getInstances_slow = async function* () {\n let output = [];\n let p = 1;\n for (let i = 1; i <= intro.nb_pages; i++) {\n let x = await scrapPage(params, i);\n output.push(x);\n yield output.flat();\n }\n}\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":1721,"value":"getInstances = function* () {\n let output = [];\n let ids = [];\n let count = 0;\n\n for (let i = 1; i <= intro.nb_pages; i++) {\n scrapPage(params, i).then((d) => {\n let result = output.push(d);\n ids.push(i);\n count = ids.length;\n });\n }\n\n while (count <= intro.nb_pages + 1) {\n yield output.flat();\n if (count >= intro.nb_pages) count++;\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1084,"value":"data = getInstances()","pinned":false,"hidden":false,"mode":"ojs"},{"id":884,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":1723,"value":"button = (data, filename = \"data.csv\") => {\n if (!data) throw new Error(\"Array of data required as first argument\");\n\n let downloadData;\n if (filename.includes(\".csv\")) {\n downloadData = new Blob([d3.csvFormat(data)], { type: \"text/csv\" });\n } else if (filename.includes(\".json\")) {\n downloadData = new Blob([JSON.stringify(data, null, 2)], {\n type: \"application/json\"\n });\n } else {\n downloadData = new Blob([data]);\n }\n\n const size = (downloadData.size / 1024).toFixed(0);\n const button = DOM.download(\n downloadData,\n filename,\n `Download ${filename} (~${size} KB)`\n );\n return button;\n}","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}