2a:T5ba,md ` - **app.py** - This is our main application. It contains required libraries, runs scripts to extract raw data, transform it, upload it to AWS S3, and send notifications in case of errors while fetching data. - **config.py** - Config specs with logging info for AWS/S3 - **country-info.csv** - Second dataset with information of different countries - **PredictiveDaily.R** - R Scripts for Data Transformation - **emailapps.py** - Scripts to send notifications emails with Python, if errors while fetching/uploading data - en var (environment variable) code to call the Gmail credentials on Heroku config vars. - **init.R** - Installs R packages on Heroku - **pipeline_flowchart** - A flow diagram visualising the pipeline. - **Procfile** - Required by Heroku (specifies the commands that are executed by the app on startup). In this case is empty because we'll use the *Heroku scheduler* add-on. - **requirements.txt** - Indicates required libraries on Heroku - **upload_to_s3.py** - Uploads the database to AWS/S3 - **Main challenges or difficulties** - Getting familiar with Python / R. - Setup on different platforms - Workflow and sync - How we addressed it : Iterative programming, extensive comments on code, consult Professors / Classmates, Official Documentation, Code Academy, Stack Overflow, Reddit forums. Constant and open communication + collaboration on Slack, Trello, Google Drive `2b:T700,{ const line = vl.markLine({ interpolate: 'monotone' }) .encode( vl.x() .fieldT('date') .timeUnit('utcyearmonthdate') .axis({ tickCount: 'day', format: '%b', grid: false, tickSize: 2, labelColor: '#929292' }) .title(null), vl.y() .fieldQ('deaths') .axis(null) .scale({ nice: true }) .title(null), vl.color({ value: 'brown' }) ) const text = vl.markText({ align: 'right', dx: -13, dy: -4, baseline: 'top', fontSize: 8 }) .encode( vl.text() .fieldQ('argmax.mean_ceil') .format(',d'), vl.x() .fieldT('argmax.date') .timeUnit('utcyearmonthdate') .title(null), vl.y() .fieldQ('argmax.mean') .axis(null) .title(null), ) .transform( vl.calculate('ceil(datum.mean)').as('mean_ceil'), vl.aggregate({ op: 'argmax', field: 'mean', as: 'argmax' }), ) const tick = text.markTick({ size: 8, xOffset: -4, align: 'right', fill: '#000' }); return vl.layer(line.markArea({ opacity: 0.2 }), line, text, tick) .width(180) .height(100) .facet(vl.facet().fieldN('location').title(null)) .columns(4) .transform( vl.window({ op: 'mean', field: 'deaths', as: 'mean' }) .groupby('location') .frame([-3, 3]), ) .resolve({ axis: { x: 'deaths' }, scale: { y: 'deaths' } }) .config({ style: { cell: { stroke: 'transparent' } }, header: { labelFontSize: 13, labelPadding: 0 }, padding: { top: 5, right: 0, bottom: 5, left: 0 }, }) .data(country_data .filter(d => d.total_deaths_per_million) .map(d => d.map(v => Object.assign({ location: d.location }, v))) .flat()) .render() }2c:T1142,viewof paramaters = form(html`

`)2d:[{"id":"51d1b6d78f800a35ca9dc7000fba107686bbbe799c08bd271fc62341e90bdb9d778d8c417478c1f42233de9d034c36df22d56c7fd2d56b4e6754bab926ae3c5b","href":"https://static.observableusercontent.com/files/51d1b6d78f800a35ca9dc7000fba107686bbbe799c08bd271fc62341e90bdb9d778d8c417478c1f42233de9d034c36df22d56c7fd2d56b4e6754bab926ae3c5b","name":"forecasts (3)@2.csv","type":"text/csv","lastModified":1609847632287.187,"size":983233},{"id":"a1bed5d45182ef9f16d06ff25885041efd634379ba469c8bfd43faaa68eacd46d1f1bd29022f94d74eafd1fa4bb204ad7fe089517e0c966f799222e462267e0d","href":"https://static.observableusercontent.com/files/a1bed5d45182ef9f16d06ff25885041efd634379ba469c8bfd43faaa68eacd46d1f1bd29022f94d74eafd1fa4bb204ad7fe089517e0c966f799222e462267e0d","name":"pipeline_flowchart-v3.png","type":"image/png","lastModified":1609951140521.338,"size":91145}] 29:{"title":"3. Predictive model","cells":[{"id":0,"value":"md`# 3. Predictive model`","pinned":false,"hidden":false,"mode":"ojs"},{"id":504,"value":"md`Once the data were analyzed with R, we had the number of deaths for selected countries and our prediction for the first days of January. The pipeline updates this information daily. These are the visualizations generated at the end of the pipeline.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":585,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":513,"value":"md `## Data pipeline`","pinned":false,"hidden":false,"mode":"ojs"},{"id":516,"value":"md `We built an automated data pipeline in the cloud. The goal is to read a remote data source, transform it, and push the resulting data to our own AWS S3 bucket. This forms the basis for a simple ETL process (Extract/Transform/Load).`","pinned":false,"hidden":false,"mode":"ojs"},{"id":520,"value":"FileAttachment(\"pipeline_flowchart-v3.png\").image()","pinned":false,"hidden":false,"mode":"ojs"},{"id":522,"value":"$2a","pinned":false,"hidden":false,"mode":"ojs"},{"id":565,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":526,"value":"md `## Data Visualization`","pinned":false,"hidden":false,"mode":"ojs"},{"id":568,"value":"md `To show similarities and differences between our chosen countries we found that a multiple line chart works effectively. Is a simple way to illustrate and compare data patterns.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":498,"value":"md`### COVID-19 RELATED DEATHS SINCE SEPTEMBER 2020`","pinned":false,"hidden":false,"mode":"ojs"},{"id":39,"value":"$2b","pinned":false,"hidden":false,"mode":"ojs"},{"id":495,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":573,"value":"md`To visualize more detailed data per country, we also decided to plot a series of small multiples with a drop down menu. We can see the prediction line marked in blue and confirm if our predictions were accurate.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":491,"value":"md`### FORECAST OF COVID-19 RELATED DEATHS BY COUNTRY, ON JANUARY 2021`","pinned":false,"hidden":false,"mode":"ojs"},{"id":438,"value":"$2c","pinned":false,"hidden":false,"mode":"ojs"},{"id":441,"value":"chart = {\n const svg = d3.create(\"svg\").attr(\"viewBox\", [0, 0, width, height]);\n\n svg.append(\"g\").call(xAxis);\n\n svg.append(\"g\").call(yAxis);\n \n svg.append(\"g\").call(grid);\n\n svg\n .append(\"path\")\n .datum(df)\n .attr(\"fill\", \"none\")\n .attr(\"stroke\", \"brown\")\n .attr(\"stroke-width\", 2.5)\n .attr(\"stroke-linejoin\", \"round\")\n .attr(\"stroke-linecap\", \"round\")\n .attr(\"d\", line);\n \n svg\n .append(\"path\")\n .datum(df)\n .attr(\"fill\", \"brown\")\n .attr('opacity', .15)\n .attr(\"d\", area);\n\n svg\n .append(\"path\")\n .datum(df)\n .attr(\"fill\", \"none\")\n .attr(\"stroke\", \"steelblue\")\n .attr(\"stroke-width\", 2.5)\n .attr(\"stroke-linejoin\", \"round\")\n .attr(\"stroke-linecap\", \"round\")\n .attr(\"d\", line2);\n \n svg\n .append(\"path\")\n .datum(df)\n .attr(\"fill\", \"steelblue\")\n .attr('opacity', .15)\n .attr(\"d\", area2);\n\n return svg.node();\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":534,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":544,"value":"md `## Improvements`","pinned":false,"hidden":false,"mode":"ojs"},{"id":547,"value":"md`As previously mentioned, we found some difficulties in programming these visualizations, mostly due to our proficiency with D3 and the timeframe of the project. These are the most important issues to be improved or corrected:\n\n- **Multiple charts:** The initial purpose of doing these charts was to illustrate the prediction line, but it didn't work.\n- **Line chart with the selector:** The drop down menu was done manually due to some technical difficulties. The y axis is done for the value of cases and it doesn't count the prediction part, so it isn't shown perfectly for a few countries.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":558,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":531,"value":"md `## Conclusions`","pinned":false,"hidden":false,"mode":"ojs"},{"id":538,"value":"md `These two visualizations were more technically challenging, since we wanted to show the prediction line in both charts. However, we think that such visualizations would correctly show the required information. The pipeline runs efficiently and generates the charts as required for its last step.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":426,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":423,"value":"md`## Appendix: Multiple charts`","pinned":false,"hidden":false,"mode":"ojs"},{"id":244,"value":"OWiDdata = d3.csv(\"https://mvtec-group1.s3-eu-west-1.amazonaws.com/mvtec-group1/forecasts.csv\");","pinned":false,"hidden":false,"mode":"ojs"},{"id":12,"value":"country_data = d3.groups(OWiDdata, d => d.location)\n .map(([, rows]) => process_data(rows))","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"process_data = data => {\n const output = data.map((d, i) => ({\n date: d.date,\n deaths: d.total_deaths_per_million,\n prediction: d.fcasted_deaths_avg\n \n }));\n\n output.unshift({\n date: d3.utcFormat('%Y-%m-%d'),\n deaths: 0,\n prediction: 0,\n\n });\n\n Object.assign(output, data[0], {\n past_week: d3.sum(output.slice(-7), d => d.total_deaths_per_million)\n });\n\n ['date', 'deaths', 'prediction'].forEach(key => delete output[key]);\n\n return output;\n};","pinned":false,"hidden":false,"mode":"ojs"},{"id":14,"value":"d3 = require(\"d3@5\", \"d3-array@2\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":17,"value":"import {vl} from '@vega/vega-lite-api'","pinned":false,"hidden":false,"mode":"ojs"},{"id":430,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs"},{"id":434,"value":"md`## Appendix: Line chart`","pinned":false,"hidden":false,"mode":"ojs"},{"id":482,"value":"data2 = OWiDdata\n .map(d => ({\n value: +d.total_deaths_per_million,\n date: new Date(d.date),\n value1: d.fcasted_deaths_avg,\n location: d.location\n }))","pinned":false,"hidden":false,"mode":"ojs"},{"id":444,"value":"df = data2.filter(d => d.location == paramaters.location);","pinned":false,"hidden":false,"mode":"ojs"},{"id":446,"value":"grid = g => g\n .attr(\"stroke\", \"currentColor\")\n .attr(\"stroke-opacity\", 0.05)\n .call(g => g.append(\"g\")\n .selectAll(\"line\")\n .data(x.ticks())\n .join(\"line\")\n .attr(\"x1\", d => 0.3 + x(d))\n .attr(\"x2\", d => 0.3 + x(d))\n .attr(\"y1\", margin.top +10)\n .attr(\"y2\", height - margin.bottom +10))\n .call(g => g.append(\"g\")\n .selectAll(\"line\")\n .data(y.ticks())\n .join(\"line\")\n .attr(\"y1\", d => 0.5 + y(d))\n .attr(\"y2\", d => 0.5 + y(d))\n .attr(\"x1\", margin.left)\n .attr(\"x2\", width - margin.right));","pinned":false,"hidden":false,"mode":"ojs"},{"id":449,"value":"line = d3\n .line()\n .defined(d => !isNaN(d.value))\n .x(d => x(d.date))\n .y(d => y(d.value))","pinned":false,"hidden":false,"mode":"ojs"},{"id":452,"value":"line2 = d3\n .line()\n .defined(d => !isNaN(d.value1))\n .x(d => x(d.date))\n .y(d => y(d.value1))","pinned":false,"hidden":false,"mode":"ojs"},{"id":455,"value":"x = d3\n .scaleUtc()\n .domain(d3.extent(df, d => d.date))\n .range([margin.left, width - margin.right])","pinned":false,"hidden":false,"mode":"ojs"},{"id":458,"value":"y = d3\n .scaleLinear()\n .domain([0, d3.max(df, d => d.value)])\n .nice()\n .range([height - margin.bottom, margin.top])","pinned":false,"hidden":false,"mode":"ojs"},{"id":461,"value":"xAxis = g =>\n g.attr(\"transform\", `translate(0,${height - margin.bottom})`).call(\n d3\n .axisBottom(x)\n .ticks(width / 80)\n .tickSizeOuter(0)\n )","pinned":false,"hidden":false,"mode":"ojs"},{"id":464,"value":"yAxis = g =>\n g\n .attr(\"transform\", `translate(${margin.left},0)`)\n .call(d3.axisLeft(y))\n .call(g => g.select(\".domain\").remove())\n .call(g =>\n g\n .select(\".tick:last-of-type text\")\n .clone()\n .attr(\"x\", 3)\n .attr(\"text-anchor\", \"start\")\n .attr(\"font-weight\", \"bold\")\n .text(df.y)\n )","pinned":false,"hidden":false,"mode":"ojs"},{"id":467,"value":"area = d3.area()\n .curve(d3.curveLinear)\n .defined(d => !isNaN(d.value))\n .x(d => x(d.date))\n .y1(d => y(d.value))\n .y0(y(0));","pinned":false,"hidden":false,"mode":"ojs"},{"id":470,"value":"area2 = d3.area()\n .curve(d3.curveLinear)\n .defined(d => !isNaN(d.value1))\n .x(d => x(d.date))\n .y1(d => y(d.value1))\n .y0(y(0));","pinned":false,"hidden":false,"mode":"ojs"},{"id":473,"value":"margin = ({ top: 80, right: 30, bottom: 30, left: 50 })","pinned":false,"hidden":false,"mode":"ojs"},{"id":476,"value":"height = 400","pinned":false,"hidden":false,"mode":"ojs"},{"id":479,"value":"import {form} from \"@mbostock/form-input\"","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W2d","stdlib":"1","titling":"auto"}