30:T430,fetch_members = async function*({ propublica_key, from = 97, to = 116 // chamber = 'house' } = {}) { let responses = []; let max_requests = Math.abs(from - to) + 1; for (let i of _.range(from, to + 1)) { let house_result = await fetch_propublica( propublica_key, `${i}/house/members` ); responses.push( house_result.results[0].members.map(member => { return { ...member, congress: i, chamber: 'House' }; }) ); let senate_result = await fetch_propublica( propublica_key, `${i}/senate/members` ); responses.push( senate_result.results[0].members.map(member => { return { ...member, congress: i, chamber: 'Senate' }; }) ); yield { members: responses.flat(), done: false, max_requests: max_requests, requests: i - from }; } return yield { members: responses.flat(), done: true, max_requests: max_requests, requests: max_requests }; }31:[] 2e:{"title":"Propublica API","cells":[{"id":0,"value":"md`\n# Propublica API\n\nLoad data about the US congress using the propublica api at https://projects.propublica.org/api-docs/congress-api/\n\nBills (and votes) have a type, which can be pretty confusing (and I surely don't want all types in my analysis): https://govtracknews.wordpress.com/2009/11/11/what-are-the-different-types-of-bills/\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"fetch_propublica = {\n let cache = new Map();\n return async (propublica_key, endpoint, offset = 0) => {\n let cache_key = `${endpoint}.json?${offset}`;\n if (cache.has(cache_key)) {\n return cache.get(cache_key);\n }\n\n let response = await fetch(\n `https://api.propublica.org/congress/v1/${endpoint}.json?offset=${offset}`,\n {\n headers: { \"X-API-Key\": propublica_key }\n }\n );\n let data = await response.json();\n\n cache.set(cache_key, data);\n return data;\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":366,"value":"create_propublica_api = propublica_key => {\n return (...args) => {\n return fetch_propublica(propublica_key, ...args);\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":212,"value":"md`\n### Fetch Votes\n\nBefore, I did this with propublicas \"recent votes\" api, which caps at 20 votes per request.\nWith the new request, I ask for all votes in a specific month. That can be waaay more than 20 results. \nhttps://projects.propublica.org/api-docs/congress-api/votes/#get-votes-by-date\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"fetch_votes_months = async function*({ propublica_key, chamber = 'house', from, to } = {}) {\n let all_votes = [];\n let months_difference =\n 1 + Math.abs(datefns.differenceInCalendarMonths(from, to));\n\n let base_month = from > to ? to : from;\n\n for (let month_delta of _.range(0, months_difference)) {\n let request_count = month_delta + 1;\n\n let date_to_fetch = datefns.addMonths(base_month, month_delta);\n\n let YYYY = datefns.format(date_to_fetch, \"yyyy\");\n let MM = datefns.format(date_to_fetch, \"MM\");\n let result = await fetch_propublica(propublica_key, `${chamber}/votes/${YYYY}/${MM}`);\n let votes = result.results.votes;\n\n all_votes = [...all_votes, ...votes];\n\n yield {\n all: false,\n done: false,\n max_requests: months_difference,\n requests: request_count,\n votes: all_votes\n };\n }\n\n return yield {\n all: true,\n done: true,\n max_requests: months_difference,\n requests: months_difference,\n votes: all_votes\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":240,"value":"md`\n### Fetch Members\n\nQuickest way to fetch members is cycling through all \"editions\" of congress and calling https://projects.propublica.org/api-docs/congress-api/members/#lists-of-members\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":217,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":205,"value":"md`\n### Fetch bills\n\nToo bad you have to specify a \\`type\\` property, making bulk scraper way harder. Maybe I can find a better method than https://projects.propublica.org/api-docs/congress-api/bills/#get-recent-bills\n\nAlso I can only get 20 per request :/\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":204,"value":"fetch_bills = async function*({\n propublica_key,\n congress = 116,\n chamber = \"house\",\n type = \"enacted\",\n request_limit = 20\n}) {\n let all_bills = [];\n for (let i of _.range(0, request_limit)) {\n let response = await fetch_propublica(\n propublica_key,\n `${congress}/${chamber}/bills/${type}`,\n i * 20\n );\n let bills = response.results[0].bills;\n\n all_bills = [...all_bills, ...bills];\n\n yield { all: false, done: false, requests: i + 1, bills: all_bills };\n\n if (bills.length < 20) {\n return yield {\n all: true,\n done: true,\n requests: i + 1,\n bills: all_bills\n };\n }\n }\n return yield {\n all: false,\n done: true,\n requests: request_limit,\n bills: all_bills\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":28,"value":"md`\n## Downloads\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":292,"value":"md`### Propublica key`","pinned":false,"hidden":false,"mode":"ojs"},{"id":286,"value":"viewof propublica_key = html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":244,"value":"md`### Votes download`","pinned":false,"hidden":false,"mode":"ojs"},{"id":318,"value":"viewof votes_download_options = form(html`

\n ${md`\n### Votes download \nBulk vote downloader\n `}\n\n \n

\n \n till\n \n

\nEnter propublica key above`;\n return button;\n }\n\n let button = html``;\n button.onclick = async () => {\n if (mutable all_house_cache.done) {\n return;\n }\n\n mutable all_house_cache = { done: false, data: null };\n let generator = fetch_votes_months({\n propublica_key,\n chamber: votes_download_options.chamber,\n from: new Date(`${votes_download_options.from}-01-01`),\n to: new Date(`${votes_download_options.to}-12-31`)\n });\n invalidation.then(() => generator.throw(new Error('Cancelled')));\n\n for await (let results of generator) {\n mutable all_house_cache = { done: false, data: results };\n }\n\n mutable all_house_cache = {\n done: true,\n data: mutable all_house_cache.data\n };\n };\n return button;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":170,"value":"{\n return downloader(all_house_cache, {\n download: data => ({\n json: data.votes,\n filename: `${votes_download_options.chamber}-${votes_download_options.from}-till-${votes_download_options.to}.json`\n }),\n progress: data => data.requests / data.max_requests\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":353,"value":"{\n let sorted = _.sortBy(all_house_cache.data.votes, x => x.date);\n return `${_.first(sorted).date} - ${_.last(sorted).date}`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":248,"value":"md`\n### Members download\n\nDownload all members of congress from the 97th to the 116th. \nI still need to find out the best way to seperate the changes in individual members over different congresses. Right now I add a \\`congress\\` property to every entry.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":251,"value":"mutable members_cache = ({ done: false, data: null })","pinned":false,"hidden":false,"mode":"ojs"},{"id":253,"value":"{\n let button = html``;\n button.disabled = propublica_key === \"\" ? true : false;\n button.onclick = async () => {\n if (mutable members_cache.done) {\n return;\n }\n\n mutable members_cache = { done: false, data: null };\n let generator = fetch_members({ propublica_key });\n invalidation.then(() => generator.throw(new Error('Cancelled')));\n\n for await (let results of generator) {\n mutable members_cache = { done: false, data: results };\n }\n\n mutable members_cache = {\n done: true,\n data: mutable members_cache.data\n };\n };\n return button;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":266,"value":"downloader(members_cache, {\n download: data => ({ json: data.members, filename: 'members.json' }),\n progress: data => data.requests / data.max_requests\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":414,"value":"{\n let unique_members = _.map(_.keyBy(members_cache.data.members, x => x.id))\n .length;\n return unique_members;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":371,"value":"md`### Bills download`","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"md`### Appendix`","pinned":false,"hidden":false,"mode":"ojs"},{"id":111,"value":"downloader = (cache, { download = () => null, progress = () => 0 }) => {\n if (cache.data == null) return html``;\n\n if (cache.done) {\n let { json, filename, title = `Download ${filename}` } = download(\n cache.data\n );\n return DOM.download(new Blob([JSON.stringify(json)]), filename, title);\n }\n\n return html`\n \n

\n \n \n ${_.round(progress(cache.data) * 100)}%\n \n \n \n `;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":12,"value":"_ = require('lodash')","pinned":false,"hidden":false,"mode":"ojs"},{"id":16,"value":"datefns = require('https://bundle.run/date-fns@2.9.0')","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"last = async (iterator, invalidation, tap_fn = () => {}) => {\n let last_x = null;\n\n invalidation.then(() => {\n iterator.throw(new Error('Cell reloaded'));\n });\n\n for await (let x of iterator) {\n last_x = x;\n tap_fn(x);\n }\n return last_x;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":321,"value":"import { form } from \"@mbostock/form-input\"","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}