30:T411,getUserTweets = async () => { document.getElementById("progress").innerHTML = "⬇️ User timeline: Download in progress..."; progress.value = 0; progress.max = profile.data.public_metrics.tweet_count > 3200 ? 3200 : profile.data.public_metrics.tweet_count; let userTweets = []; const params = { "max_results": 100, "tweet.fields": "created_at,public_metrics,entities", } let hasNextPage = true; let nextToken = null; while (hasNextPage) { let resp = await getPage(params, options, nextToken); if (resp.meta.result_count > 0) { if (resp.data) { userTweets = userTweets.concat(resp.data); progress.value += resp.meta.result_count; } if (resp.meta.next_token) { nextToken = resp.meta.next_token; } else { hasNextPage = false; } } else { hasNextPage = false; } } document.getElementById("progress").innerHTML = `✅ User timeline: data loaded (${userTweets.length} Tweets)`; return userTweets; }31:[] 2e:{"title":"Querying the Twitter API","cells":[{"id":0,"value":"# Querying the Twitter API\n

[Twitter Starter Pack](https://observablehq.com/@floatingpurr/twitter-starter) > Get the data

\n\n> ⚠️ **Important notice: this notebook uses a CORS proxy hosted on Heroku. Unfortunately it'll be shut down on November 28, 2022 with the [end of the Heroku free plan](https://devcenter.heroku.com/articles/free-dyno-hours).**\n\nA convenient way to query the Twitter API to dump Timeline data and to explore them in the [_Visualizing Twitter data_ 🐦](https://observablehq.com/@floatingpurr/visualizing-twitter-data) notebook.\n\nℹ️ Due to the CORS mechanism, this notebook is backed by [this CORS proxy ](#corsProxy).\n\n📚 Read the [documentation](https://observablehq.com/@floatingpurr/twitter-starter).","pinned":false,"hidden":false,"mode":"md"},{"id":262,"value":"## 1. Get your very own Bearer Token from Twitter\n\nHead to the Twitter Developer Platform and set up your Bearer Token for the API v2 endpoints (see [here](https://developer.twitter.com/en/docs/authentication/oauth-2-0/bearer-tokens)). \n\nℹ️ Standard developer accounts are [limited](https://developer.twitter.com/en/docs/projects/overview#tweet-cap) to 500,000 Tweets per month and to the ~3,200 most recent Tweets, Retweets, replies and Quote Tweets posted by a Twitter user.","pinned":false,"hidden":false,"mode":"md"},{"id":261,"value":"## 2. Set your token for this notebook ","pinned":false,"hidden":false,"mode":"md"},{"id":260,"value":"Did you get it? You need it to authenticate to the API. It will be attached to each request!\n\nEnter and set your bearer token here:","pinned":false,"hidden":false,"mode":"md"},{"id":269,"value":"viewof token = text({\n description: \"This is your Bearer Token whose value is only available to you. Don't forget to hit the \\\"Set 🔒\\\" button!\",\n placeholder: \"Enter a valid token...\",\n value: getSecretKey, // Just for test\n submit: \"Set 🔒\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":296,"value":"token ? md`✅ _Token set!_` : md`❌ _Insert your token before moving ahead._`","pinned":false,"hidden":false,"mode":"ojs"},{"id":258,"value":"## 3. Select a Twitter user and download data","pinned":false,"hidden":false,"mode":"md"},{"id":257,"value":"Type in the username of the Twitter user you want to analyze:","pinned":false,"hidden":false,"mode":"md"},{"id":256,"value":"viewof account = text({placeholder: \"Twitter username...\", description: \"Enter a valid twitter username without the \\\"@\\\"\", submit: \"Fetch data 🔥\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":255,"value":"## 4. Check the download status","pinned":false,"hidden":false,"mode":"md"},{"id":254,"value":"Downloading progress:","pinned":false,"hidden":false,"mode":"md"},{"id":253,"value":"progress = html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":252,"value":"$${profile.data ? `✅ **Profile data**: data loaded (**${account}**)` : '❌ **Profile data**: not available'}","pinned":false,"hidden":false,"mode":"md"},{"id":251,"value":"","pinned":false,"hidden":false,"mode":"html"},{"id":250,"value":"## 5. Data preview","pinned":false,"hidden":false,"mode":"md"},{"id":249,"value":"tweets.length > 0 ? md`You got _${tweets.length}_ tweets out of _${profile.data.public_metrics.tweet_count}_ for *${profile.data.username}*` : md`_Waiting for data..._`","pinned":false,"hidden":false,"mode":"ojs"},{"id":166,"value":"Inputs.table(tweets, {columns: [\"id\", \"created_at\", \"text\"]})","pinned":false,"hidden":false,"mode":"ojs"},{"id":247,"value":"## 6. Save data to file","pinned":false,"hidden":false,"mode":"md"},{"id":246,"value":"Download the file _twitter.json_ with your [data](#data).","pinned":false,"hidden":false,"mode":"md"},{"id":245,"value":"button(data, 'twitter.json')","pinned":false,"hidden":false,"mode":"ojs"},{"id":244,"value":"You are all set. Now, you can load that file into the Twitter Starter Pack notebook, _coming soon..._","pinned":false,"hidden":false,"mode":"md"},{"id":243,"value":"## Caveat\n\nThe Twitter API does not support [CORS](https://stackoverflow.com/questions/35879943/twitter-api-authorization-fails-cors-preflight-in-browser), so we cannot query the API directly from the browser. Therefore, this notebook routes http requests to a special [proxy](#corsProxy) that expects requests to come from here. Do not use a fork of this notebook, otherwise the proxy will refuse to forward your queries to the Twitter API.\n\n> 🤔 _Mmmh, I don't feel comfortable with that_\n\nOh, I see. If you prefer to create your own version of this notebook, you will need to address requests to another CORS proxy. Build yours from scratch or pick one from this collection of [free CORS proxies](https://nordicapis.com/10-free-to-use-cors-proxies/).","pinned":false,"hidden":false,"mode":"md"},{"id":105,"value":"## Implementation","pinned":false,"hidden":false,"mode":"md"},{"id":241,"value":"// http options with auth\noptions = ({\n headers: {\n \"authorization\": `Bearer ${token}`\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":93,"value":"corsProxy = \"https://observablecorsproxy.herokuapp.com/\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":240,"value":"profileUrl = `${corsProxy}https://api.twitter.com/2/users/by/username/${account.trim()}?user.fields=description,created_at,location,pinned_tweet_id,profile_image_url,public_metrics,url,verified`","pinned":false,"hidden":false,"mode":"ojs"},{"id":124,"value":"tweetsUrl = profile.data ? `${corsProxy}https://api.twitter.com/2/users/${profile.data.id}/tweets` : md`Timeline URL not yet available, insert a valid [username](#account)`","pinned":false,"hidden":false,"mode":"ojs"},{"id":113,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":115,"value":"getPage = async (params, options, nextToken) => {\n if (nextToken) {\n params.pagination_token = nextToken;\n }\n try {\n \n return await d3.json(`${tweetsUrl}?${new URLSearchParams(params)}`, options);\n\n } \n catch (err) {\n throw new Error(`Request failed: ${err}`);\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":234,"value":"getSecretKey = {\n // just for testing, ignore it!\n try {\n return Secret(\"TWITTER_BEARER_TOKEN\")\n } catch (error) {\n return ''\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":233,"value":"profile = account ? d3.json(profileUrl, options) : {}","pinned":false,"hidden":false,"mode":"ojs"},{"id":131,"value":"tweets = {\n if(profile.data) {\n return getUserTweets();\n } else {\n document.getElementById(\"progress\").innerHTML = \"❌ User timeline: not available\";\n return [];\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":319,"value":"timestamp = {\n account\n return new Date().toISOString()\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":341,"value":"**Data saved to file**:","pinned":false,"hidden":false,"mode":"md"},{"id":230,"value":"data = ({\n data_timestamp: timestamp,\n profile: profile.data,\n tweets: tweets\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":229,"value":"## References\n\n- [CORS-Anywhere - for Observables](https://observablehq.com/@asg017/cors-anywhere-for-observables)\n- [The Twitter API](https://developer.twitter.com/en/docs/twitter-api)","pinned":false,"hidden":false,"mode":"md"},{"id":228,"value":"## Appendix","pinned":false,"hidden":false,"mode":"md"},{"id":226,"value":"import {text} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":225,"value":"import {button} from \"@jeremiak/download-data-button\"","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}