31:T85b,{ const form = html``; let _interval; let _results = []; let counter = 0; async function _fetch(i) { const url = urls[i] console.log(url) let response; try { response = await d3.json(url) } catch (err) { console.error(err, response) mutable status = `There is an error, open the JS console. Trying again in ${timeInterval/1000}sec.` counter-- return; } console.log(response) response.result["original_url"] = list_of_links[i] _results.push(response) mutable results = _results mutable status = `Looks good, retrieved ${_results.length} urls` } function _start() { console.log("start"); form.start.setAttribute("disabled", true) form.pause.removeAttribute("disabled", true) form.reset.setAttribute("disabled", true) console.log("Fetching N°", counter) _fetch(counter); counter++; _interval = setInterval(()=>{ // stop if (counter >= urls.length) { console.log("Finished!") mutable status = `Finished! Retrieved ${mutable results.length} urls` return } console.log("Fetching N°", counter) _fetch(counter); counter++; },timeInterval+250) } function _pause(){ console.log("pause"); form.start.removeAttribute("disabled", true) form.pause.setAttribute("disabled", true) form.reset.removeAttribute("disabled", true) clearInterval(_interval) } function _reset(){ console.log("reset"); form.start.removeAttribute("disabled", true) form.pause.setAttribute("disabled", true) form.reset.setAttribute("disabled", true) counter=0 _results = []; mutable results = _results; clearInterval(_interval); } form.start.onclick = () => _start() form.pause.onclick = () => _pause() form.reset.onclick = () => _reset() return form; }32:[] 2f:{"title":"CrowdTangle API - links","cells":[{"id":0,"value":"md`# CrowdTangle API - links`","pinned":false,"hidden":false,"mode":"ojs"},{"id":609,"value":"md`\nThis notebook accepts a list of URLs and queries CrowTangle for social media posts that share it.\nThe API documentation is available [here](https://github.com/CrowdTangle/API/wiki).\n\nEven though stated differently in documentation, the Reddit platform is not currently supported (https://github.com/CrowdTangle/API/issues/26).\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":32,"value":"viewof links = textarea({\n title: \"List of links\",\n description: \"Paste here a link on each row, nothing else :)\",\n placeholder: \"Insert links here...\",\n value:\"https://www.youtube.com/watch?v=xCz6S91CU1c\\nhttps://www.youtube.com/watch?v=5lGjSHy4TKE\\nhttps://www.youtube.com/watch?v=NfzgOvpbx-I\\nhttps://www.youtube.com/watch?v=vLn2YwkRBIs\\nhttps://www.youtube.com/watch?v=Zhx_dPWYXuM\",\n spellcheck: true,\n width: \"100%\",\n rows: 10\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":44,"value":"viewof apiKey = text({\n title: \"Your API key\",\n placeholder: \"your-api-key\",\n description: \"It is better to keep secret your API key\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":588,"value":"viewof timeInterval = slider({\n min: 0, \n max: 60000, \n step: 100, \n value: 3000, \n title: \"Speed of API calls\", \n description: \"Calling API too fast will result in exceding rate limits.
Separate sequential calls with appropriate delay intervals.\",\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":75,"value":"viewof platforms = checkbox({\n title: \"Platforms\",\n description: \"The platforms from which to retrieve posts. This value can be comma-separated.\",\n options: [\n { value: \"facebook\", label: \"Facebook\" },\n { value: \"instagram\", label: \"Instagram\" },\n // Reddit is not supported even if stated differently in documentation\n //{ value: \"reddit\", label: \"Reddit\" }\n ],\n value: [\"facebook\"],\n disabled: false\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":83,"value":"viewof count = slider({\n min: 1, \n max: 1000, \n step: 1, \n value: 1000, \n title: \"Count\", \n description: \"Number of posts to return\",\n // submit: \"Set\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":62,"value":"viewof startDate = date({\n title: \"Start date (it is optional but please use this!)\",\n description: \"The earliest date at which a post could be posted. Time zone is UTC.\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":121,"value":"viewof endDate = date({\n title: \"End date (optional)\",\n description: \"The latest date at which a post could be posted. Time zone is UTC.\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":145,"value":"viewof sortBy = radio({\n options: [\n {value: \"date\", label: \"Date\"},\n {value: \"subscriber_count\", label: \"Subscriber count\"},\n {value: \"total_interactions\", label: \"Total interactions\"}\n ], \n title: \"Sort by\", \n value: \"date\",\n description: \"The method by which to order posts. If you set 'subscriber_count', a startDate is required.\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"urls = list_of_links.map(link=>{\n const params = new URLSearchParams()\n if (!apiKey) return \"missing API key\"\n params.append('token', apiKey)\n params.append('platforms',platforms.join(','))\n params.append('count', count)\n params.append('link', link)\n params.append('sortBy', sortBy)\n \n if (startDate) params.append('startDate', startDate)\n if (endDate) params.append('endDate', endDate)\n\n return \"https://api.crowdtangle.com/links\" + '?' + params.toString()\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":308,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":535,"value":"md`Currently limited to one API call every ${timeInterval/1000} seconds.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":571,"value":"mutable status = \"You can press start\";","pinned":false,"hidden":false,"mode":"ojs"},{"id":515,"value":"data = {\n let data = results.map(d=>{\n const original_url = d.result.original_url\n return d.result.posts.map(dd=>(flattenObj({original_url:original_url, ...dd})))\n }).flat()\n return data\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":175,"value":"{\n if (data.length>0) {\n return table(data,{\n sortable: true,\n style: 'compact'\n })\n } else {\n return md`Hit the 'retrieve' button to show results in form of table.\\nIf finished with zero data, might be no posts sharing the input link.`\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":6,"value":"md`### References\n- Documentation: https://github.com/CrowdTangle/API/wiki\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":12,"value":"md`### Dependencies`","pinned":false,"hidden":false,"mode":"ojs"},{"id":56,"value":"import {text, textarea, date, checkbox, slider, radio, button} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":172,"value":"import { table } from \"@gampleman/table\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":249,"value":"d3 = require(\"d3@6\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":645,"value":" function flattenObj(obj, parent, res = {}){\n // taken from https://stackoverflow.com/a/33158929/4667165\n // https://creativecommons.org/licenses/by-sa/3.0/\n for(let key in obj){\n let propName = parent ? parent + '_' + key : key;\n if(typeof obj[key] == 'object'){\n flattenObj(obj[key], propName, res);\n } else {\n res[propName] = obj[key];\n }\n }\n return res;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":623,"value":"md`### Some global variables`","pinned":false,"hidden":false,"mode":"ojs"},{"id":553,"value":"list_of_links = links.split(\"\\n\").filter(d=>d.includes(\"http\"))","pinned":false,"hidden":false,"mode":"ojs"},{"id":402,"value":"mutable results = []","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}