3f:null 40:["$","title",null,{"children":"1. Predictive method | Santiago Salcido | Observable"}] 46:T5bb,md ` - **app.py** - This is our main application. It contains required libraries, runs scripts to extract raw data, transform it, upload it to AWS S3, and send notifications in case of errors while fetching data. - **config.py** - Config specs with logging info for AWS/S3 - **country-info.csv** - Second dataset with information of different countries - **PredictiveDaily.R** - R Scripts for Data Transformation - **emailapps.py** - Scripts to send notifications emails with Python, if errors while fetching/uploading data - en var (environment variable) code to call the Gmail credentials on Heroku config vars. - **init.R** - Installs R packages on Heroku - **pipeline_flowchart** - A flow diagram visualising the pipeline. - **Procfile** - Required by Heroku (specifies the commands that are executed by the app on startup). In this case is empty because we'll use the *Heroku scheduler* add-on. - **requirements.txt** - Indicates required libraries on Heroku - **upload_to_s3.py** - Uploads the database to AWS/S3 --- - **Main challenges or difficulties** - Getting familiar with Python / R. - Setup on different platforms - Workflow and sync - How we addressed it : Iterative programming, extensive comments on code, consult Professors / Classmates, Official Documentation, Code Academy, Stack Overflow, Reddit forums. Constant and open communication + collaboration on Slack, Trello, Google Drive `47:T700,{ const line = vl.markLine({ interpolate: 'monotone' }) .encode( vl.x() .fieldT('date') .timeUnit('utcyearmonthdate') .axis({ tickCount: 'day', format: '%b', grid: false, tickSize: 2, labelColor: '#929292' }) .title(null), vl.y() .fieldQ('deaths') .axis(null) .scale({ nice: true }) .title(null), vl.color({ value: 'brown' }) ) const text = vl.markText({ align: 'right', dx: -13, dy: -4, baseline: 'top', fontSize: 8 }) .encode( vl.text() .fieldQ('argmax.mean_ceil') .format(',d'), vl.x() .fieldT('argmax.date') .timeUnit('utcyearmonthdate') .title(null), vl.y() .fieldQ('argmax.mean') .axis(null) .title(null), ) .transform( vl.calculate('ceil(datum.mean)').as('mean_ceil'), vl.aggregate({ op: 'argmax', field: 'mean', as: 'argmax' }), ) const tick = text.markTick({ size: 8, xOffset: -4, align: 'right', fill: '#000' }); return vl.layer(line.markArea({ opacity: 0.2 }), line, text, tick) .width(180) .height(100) .facet(vl.facet().fieldN('location').title(null)) .columns(4) .transform( vl.window({ op: 'mean', field: 'deaths', as: 'mean' }) .groupby('location') .frame([-3, 3]), ) .resolve({ axis: { x: 'deaths' }, scale: { y: 'deaths' } }) .config({ style: { cell: { stroke: 'transparent' } }, header: { labelFontSize: 13, labelPadding: 0 }, padding: { top: 5, right: 0, bottom: 5, left: 0 }, }) .data(country_data .filter(d => d.total_deaths_per_million) .map(d => d.map(v => Object.assign({ location: d.location }, v))) .flat()) .render() }48:T1142,viewof paramaters = form(html`

`)49:[{"id":"51d1b6d78f800a35ca9dc7000fba107686bbbe799c08bd271fc62341e90bdb9d778d8c417478c1f42233de9d034c36df22d56c7fd2d56b4e6754bab926ae3c5b","href":"https://static.observableusercontent.com/files/51d1b6d78f800a35ca9dc7000fba107686bbbe799c08bd271fc62341e90bdb9d778d8c417478c1f42233de9d034c36df22d56c7fd2d56b4e6754bab926ae3c5b","name":"forecasts (3)@2.csv","type":"text/csv","lastModified":1609880162545.564,"size":983233},{"id":"a1bed5d45182ef9f16d06ff25885041efd634379ba469c8bfd43faaa68eacd46d1f1bd29022f94d74eafd1fa4bb204ad7fe089517e0c966f799222e462267e0d","href":"https://static.observableusercontent.com/files/a1bed5d45182ef9f16d06ff25885041efd634379ba469c8bfd43faaa68eacd46d1f1bd29022f94d74eafd1fa4bb204ad7fe089517e0c966f799222e462267e0d","name":"pipeline_flowchart-v3.png","type":"image/png","lastModified":1609885906844.991,"size":91145}] 44:{"id":"8334c37a9432c5bf","slug":"1-predictive-method","title":"1. Predictive method","thumbnail":"208c3cc7d004928e8dbc504a6e5e219bc2ae8ca459119de6159f2246f70fe9d9","access_level":"public","create_time":"$D2021-01-05T20:56:02.545Z","update_time":"$D2021-01-05T22:42:15.525Z","view_time":null,"first_public_time":"$D2021-01-06T14:25:39.328Z","has_unpublished_edits":null,"fork":{"id":"b7ecc6fa91ee115b","slug":"predictive-method/2","title":"3. Predictive model","owner":{"id":"b1ca7d0d24548993","login":"oriolvidal","name":""}},"stars":1,"starred":false,"owner":{"type":"team","id":"26ad3d0499b8e73a","login":"ssalcido","name":"Santiago Salcido","avatar_url":"https://avatars-next.observableusercontent.com/86ad7be14e345939560ec8a976ed8fc64a40a466b4baadef7e738548ec71d8e3","tier":"starter_2024","bio":"","following":false},"version":583,"publish_version":583,"role":null,"authored":false,"authors":[{"type":"individual","id":"42e7544461f30790","login":"ssalcido","name":"Santiago Salcido","avatar_url":"https://avatars-next.observableusercontent.com/86ad7be14e345939560ec8a976ed8fc64a40a466b4baadef7e738548ec71d8e3","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"42e7544461f30790","login":"ssalcido","name":"Santiago Salcido","avatar_url":"https://avatars-next.observableusercontent.com/86ad7be14e345939560ec8a976ed8fc64a40a466b4baadef7e738548ec71d8e3","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":{"id":"4ede8333b4dad5cc","slug":"mvtec-group-assignment-1","title":"MVTEC | Group Assignment 1","owner":{"id":"26ad3d0499b8e73a","login":"ssalcido","name":"Santiago Salcido"}},"type":"notebook","body":{"titling":"auto","title":"1. Predictive method","cells":[{"id":0,"value":"md`# 1. Predictive method`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":510,"value":"md `## Introduction`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":504,"value":"md`For the predictive method we decide to… lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum purus turpis, euismod sit amet bibendum nec, mollis et ante. Aenean at nisl id eros sollicitudin luctus vitae sit amet diam.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":549,"value":"md `## Data pipeline`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":551,"value":"md `We built an automated data pipeline in the cloud. The goal is to read a remote data source, transform it, and push the resulting data to our own AWS S3 bucket. This forms the basis for a simple ETL process (Extract/Transform/Load). \n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":566,"value":"FileAttachment(\"pipeline_flowchart-v3.png\").image()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":570,"value":"$46","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":521,"value":"md `## Data Visualization`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":498,"value":"md`### COVID-19 RELATED DEATHS SINCE SEPTEMBER 2020`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":39,"value":"$47","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":495,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":491,"value":"md`### FORECAST OF COVID-19 RELATED DEATHS BY COUNTRY, ON JANUARY 2021`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":438,"value":"$48","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":441,"value":"chart = {\n const svg = d3.create(\"svg\").attr(\"viewBox\", [0, 0, width, height]);\n\n svg.append(\"g\").call(xAxis);\n\n svg.append(\"g\").call(yAxis);\n \n svg.append(\"g\").call(grid);\n\n svg\n .append(\"path\")\n .datum(df)\n .attr(\"fill\", \"none\")\n .attr(\"stroke\", \"brown\")\n .attr(\"stroke-width\", 2.5)\n .attr(\"stroke-linejoin\", \"round\")\n .attr(\"stroke-linecap\", \"round\")\n .attr(\"d\", line);\n \n svg\n .append(\"path\")\n .datum(df)\n .attr(\"fill\", \"brown\")\n .attr('opacity', .15)\n .attr(\"d\", area);\n\n svg\n .append(\"path\")\n .datum(df)\n .attr(\"fill\", \"none\")\n .attr(\"stroke\", \"steelblue\")\n .attr(\"stroke-width\", 2.5)\n .attr(\"stroke-linejoin\", \"round\")\n .attr(\"stroke-linecap\", \"round\")\n .attr(\"d\", line2);\n \n svg\n .append(\"path\")\n .datum(df)\n .attr(\"fill\", \"steelblue\")\n .attr('opacity', .15)\n .attr(\"d\", area2);\n\n return svg.node();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":426,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":538,"value":"md `## Conclusion`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":540,"value":"md `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum purus turpis, euismod sit amet bibendum nec, mollis et ante. Aenean at nisl id eros sollicitudin luctus vitae sit amet diam. Proin sed velit sit amet elit fermentum malesuada. Proin luctus nibh a neque finibus, vitae sagittis enim iaculis. `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":529,"value":"md `## Appendix`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":423,"value":"md`### Multiple charts`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":244,"value":"data = d3.dsv(';',\"https://mvtec-group1.s3-eu-west-1.amazonaws.com/mvtec-group1/forecasts.csv\");","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":414,"value":"data1 = parseFloat(data.reproduction_rate.replace(',','.'))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"OWiDdata = FileAttachment(\"forecasts (3)@2.csv\").csv()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"country_data = d3.groups(OWiDdata, d => d.location)\n .map(([, rows]) => process_data(rows))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":24,"value":"process_data = data => {\n const output = data.map((d, i) => ({\n date: d.date,\n deaths: d.total_deaths_per_million,\n prediction: d.fcasted_deaths_avg\n \n }));\n\n output.unshift({\n date: d3.utcFormat('%Y-%m-%d'),\n deaths: 0,\n prediction: 0,\n\n });\n\n Object.assign(output, data[0], {\n past_week: d3.sum(output.slice(-7), d => d.total_deaths_per_million)\n });\n\n ['date', 'deaths', 'prediction'].forEach(key => delete output[key]);\n\n return output;\n};","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":14,"value":"d3 = require(\"d3@5\", \"d3-array@2\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"import {vl} from '@vega/vega-lite-api'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":430,"value":"md` --------------------`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":434,"value":"md`### Line chart`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":482,"value":"data2 = d3\n .csvParse(await FileAttachment(\"forecasts (3)@2.csv\").text(), d3.autoType)\n .map(d => ({\n value: +d.total_deaths_per_million,\n date: new Date(d.date),\n value1: d.fcasted_deaths_avg,\n location: d.location\n }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":444,"value":"df = data2.filter(d => d.location == paramaters.location);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":446,"value":"grid = g => g\n .attr(\"stroke\", \"currentColor\")\n .attr(\"stroke-opacity\", 0.05)\n .call(g => g.append(\"g\")\n .selectAll(\"line\")\n .data(x.ticks())\n .join(\"line\")\n .attr(\"x1\", d => 0.3 + x(d))\n .attr(\"x2\", d => 0.3 + x(d))\n .attr(\"y1\", margin.top +10)\n .attr(\"y2\", height - margin.bottom +10))\n .call(g => g.append(\"g\")\n .selectAll(\"line\")\n .data(y.ticks())\n .join(\"line\")\n .attr(\"y1\", d => 0.5 + y(d))\n .attr(\"y2\", d => 0.5 + y(d))\n .attr(\"x1\", margin.left)\n .attr(\"x2\", width - margin.right));","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":449,"value":"line = d3\n .line()\n .defined(d => !isNaN(d.value))\n .x(d => x(d.date))\n .y(d => y(d.value))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":452,"value":"line2 = d3\n .line()\n .defined(d => !isNaN(d.value1))\n .x(d => x(d.date))\n .y(d => y(d.value1))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":455,"value":"x = d3\n .scaleUtc()\n .domain(d3.extent(df, d => d.date))\n .range([margin.left, width - margin.right])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":458,"value":"y = d3\n .scaleLinear()\n .domain([0, d3.max(df, d => d.value)])\n .nice()\n .range([height - margin.bottom, margin.top])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":461,"value":"xAxis = g =>\n g.attr(\"transform\", `translate(0,${height - margin.bottom})`).call(\n d3\n .axisBottom(x)\n .ticks(width / 80)\n .tickSizeOuter(0)\n )","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":464,"value":"yAxis = g =>\n g\n .attr(\"transform\", `translate(${margin.left},0)`)\n .call(d3.axisLeft(y))\n .call(g => g.select(\".domain\").remove())\n .call(g =>\n g\n .select(\".tick:last-of-type text\")\n .clone()\n .attr(\"x\", 3)\n .attr(\"text-anchor\", \"start\")\n .attr(\"font-weight\", \"bold\")\n .text(df.y)\n )","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":467,"value":"area = d3.area()\n .curve(d3.curveLinear)\n .defined(d => !isNaN(d.value))\n .x(d => x(d.date))\n .y1(d => y(d.value))\n .y0(y(0));","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":470,"value":"area2 = d3.area()\n .curve(d3.curveLinear)\n .defined(d => !isNaN(d.value1))\n .x(d => x(d.date))\n .y1(d => y(d.value1))\n .y0(y(0));","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":473,"value":"margin = ({ top: 80, right: 30, bottom: 30, left: 50 })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":476,"value":"height = 400","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":479,"value":"import {form} from \"@mbostock/form-input\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W49","stdlib":"1"}} 45:"$44:body"