2e:null 2f:["$","title",null,{"children":"OpenAQ Direct File Access | OpenAQ | Observable"}] 34:[] 32:{"id":"5b6c659a5d59219b","slug":"openaq-direct-file-access","title":"OpenAQ Direct File Access","thumbnail":"3c535361e57fcf3864a6015785912f5db5dcf7eeecfc0d28c4f841cc4a29cb54","access_level":"public","create_time":"$D2022-01-24T22:17:01.670Z","update_time":"$D2023-11-13T19:11:03.761Z","view_time":null,"first_public_time":"$D2023-03-09T15:18:02.187Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"1f8b49ddb281b956","login":"openaq","name":"OpenAQ","avatar_url":"https://avatars-next.observableusercontent.com/74972890073630f6e976308e735e7c4300d84027710faa3b49c2e9c36a176e6e","tier":"starter_2024"},"version":571,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"a7bdcb7339765e3b","login":"openaq-russ","name":"","avatar_url":"https://avatars-next.observableusercontent.com/727b8f8d1b7a836462f95d96575087e82375b2f4be54d612b43806496b5ab102","tier":null}],"members":[{"type":"team","id":"1f8b49ddb281b956","login":"openaq","name":"OpenAQ","avatar_url":"https://avatars-next.observableusercontent.com/74972890073630f6e976308e735e7c4300d84027710faa3b49c2e9c36a176e6e","tier":"starter_2024","role":"editor"},{"type":"individual","id":"a7bdcb7339765e3b","login":"openaq-russ","name":"","avatar_url":"https://avatars-next.observableusercontent.com/727b8f8d1b7a836462f95d96575087e82375b2f4be54d612b43806496b5ab102","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":3,"collection":null,"type":"notebook","body":{"title":"OpenAQ Direct File Access","cells":[{"id":0,"value":"# OpenAQ Direct File Access","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":566,"value":"

⛔

This notebook is out of date. See the updated notebook at https://observablehq.com/@openaq/openaq-open-data-on-aws. As of November 2023 we have migrated the Open Data on AWS bucket to a new file structure. The file structure demonstrated in this notebook will persist until the end of November 2023, please migrate to the new structure ASAP

","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":97,"value":"OpenAQ has changed how we store our data on S3. An improved folder structure will allow for more refined search of data on the OpenAQ platform. Let's take a look how to use this new and improved structure to find data for a single day.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":564,"value":"https://openaq-data-archive.s3.amazonaws.com/","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":517,"value":"The S3 folder structure is as follows:\n\n```\n* records\n * csv.gz\n * provider={provider_name}\n * country={country_code}\n * locationid={OpenAQ location ID}\n * year={4 digit year}\n * month={2 digit month}\n * location-{locationid}-{year}{month}{day}.csv.gz\n```","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":513,"value":"Currently data are stored as gzipped csv files. Below the ```records``` folder we will look in the ```csv.gz``` if other file formats are supported in the future a folder will be available at this level.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":373,"value":"For this demo we will use OpenAQ location 2178, an EPA monitor in Albuquerque, New Mexico, USA originally from airnow.gov. The first three folders in the file path will be __provider=airnow__, __country=us__ and __locationid=2178__","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":379,"value":"Next we can filter the data to the year and month we are interested in.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":100,"value":"## Choose a date","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":456,"value":"Data are generated at the end of the day of the location, i.e. midnight of the local time.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":62,"value":"viewof date1 = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":571,"value":"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":122,"value":"For this date: ${date1.getUTCFullYear()}-${monthPad}-${date1.getUTCDate()}, we'll look in the __year=${date1.getUTCFullYear()}__ directory and __month=${monthPad}__ directory","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":540,"value":"Finally, the file is named in the format ```location-{location ID}-{year}{month}{day}.csv.gz``` or in this example ```location-2178-${formatDate(date1)}.csv.gz```","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":240,"value":"ultimately the URL for this file will be: ${url}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":233,"value":"This file includes the measurements for all parameters supported by the location. In the table below we can view the contents of the csv:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":230,"value":"Inputs.table(data)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":329,"value":"We can chart the series to see how the values change throughout the day (in local time). Choose a parameter and Y axis scale type below:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":317,"value":"viewof parameter = Inputs.select(parameters, {\n value: \"log\",\n label: \"Measurand\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":311,"value":"viewof scaleValue = Inputs.select([\"log\", \"linear\"], {\n value: \"log\",\n label: \"Y Axis Scale\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"Plot.plot({\n width: width,\n color: {\n type: \"categorical\",\n scheme: \"tableau10\"\n },\n x: { ticks: 24 },\n y: { type: scaleValue },\n marks: [\n Plot.line(series, {\n x: \"date\", \n y: \"value\",\n stroke: \"location\"\n }),\n Plot.dot(series, {\n r: 2,\n x: \"date\",\n y: \"value\",\n stroke: \"location\",\n fill: \"location\",\n title: d => `${d.value} ${d.units}`\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":500,"value":"To learn more about how to query data from the OpenAQ Open Data on AWS bucket read more at [https://docs.openaq.org](https://docs.openaq.org). If you have any questions or want to connect with others join our [Slack community](https://join.slack.com/t/openaq/shared_invite/zt-yzqlgsva-v6McumTjy2BZnegIK9XCVw)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":223,"value":"series = data.filter(o => o.parameter == parameter).map(o => {o.date = new Date(o.datetime); o.value = parseFloat(o.value); return o})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":320,"value":"parameters = [...new Set(data.map(x => x.parameter))].filter(e => e != null);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":219,"value":"data.map(o => {o.date = new Date(o.datetime); o.value = parseFloat(o.value); return o}).filter(o => o.measurand == \"o3\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"data = {\n const bytes = new Uint8Array(await res.arrayBuffer());\n const text = new TextDecoder().decode(gunzip(bytes));\n return parseCsv(text);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":175,"value":"parseCsv = value => {\n const rows = value.split('\\n').map(o => o.split(','));\n rows.pop();\n const columns = rows.shift();\n let data = [];\n for (const row of rows) {\n let record = {}\n for (let i = 0; i < row.length; i++) {\n record[columns[i]] = row[i]\n } \n data.push(record);\n }\n return data;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3,"value":"res = await fetch(url);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":464,"value":"url = `https://openaq-data-archive.s3.amazonaws.com/records/csv.gz/provider%3Dairnow/country%3Dus/locationid%3D2178/year%3D${date1.getUTCFullYear() }/month%3D${monthPad}/location-2178-${formatDate(date1)}.csv.gz`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":544,"value":"monthPad = month.toString().padStart(2,\"0\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":477,"value":"month = date1.getUTCMonth() + 1","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"import {gunzip} from \"@mourner/wasm-gunzip\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":64,"value":"max = new Date()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":67,"value":"function formatDate(date) {\n return [\n (date.getFullYear()).toString().padStart(4, \"0\"),\n (date.getMonth() + 1).toString().padStart(2, \"0\"),\n (date.getDate() + 1).toString().padStart(2, \"0\")\n ].join(\"\")\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":559,"value":"html``","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":485,"value":"import {Plot} from \"@mkfreeman/plot-tooltip\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W34","stdlib":"1","titling":"auto"}} 33:"$32:body"