2b:null 2c:["$","title",null,{"children":"Exploring the Office for National Statistics API | Alf Eaton | Observable"}] 31:[{"id":"39d9c87aab7fc1d135f7281b68683ee3f7937e86f9179fbcc3f6b672a95d30f3abb112dba371693a002695851eeb7a05e7d1bd93e10486606ca134836b0b53c3","href":"https://static.observableusercontent.com/files/39d9c87aab7fc1d135f7281b68683ee3f7937e86f9179fbcc3f6b672a95d30f3abb112dba371693a002695851eeb7a05e7d1bd93e10486606ca134836b0b53c3","name":"weekly-deaths-region-historical@1.csv","type":"text/csv","lastModified":1734816062670.156,"size":9734},{"id":"8ffc7165c8ee1750a5e9ead0aab853e695fa6f633a1f53399ae1b9fa10b86763d31a117053fda9fe3c81d26a2cd708fc410e7f3c3d6943e2cb452f4e918c7667","href":"https://static.observableusercontent.com/files/8ffc7165c8ee1750a5e9ead0aab853e695fa6f633a1f53399ae1b9fa10b86763d31a117053fda9fe3c81d26a2cd708fc410e7f3c3d6943e2cb452f4e918c7667","name":"weekly-deaths-region-historical.csv","type":"text/csv","lastModified":1615125859062.9,"size":6896}] 2f:{"id":"a6be78a38620793d","slug":"office-of-national-statistics-api","title":"Exploring the Office for National Statistics API","thumbnail":"c294d2f39f6e01b668448fab4783fefd9097da80a7b22b44a0b1b89caf2346b5","access_level":"public","create_time":"$D2021-03-06T20:09:59.852Z","update_time":"$D2024-12-21T21:22:30.151Z","view_time":null,"first_public_time":"$D2021-03-06T22:48:24.278Z","fork":false,"stars":3,"starred":false,"owner":{"type":"team","id":"1dccd75582c469d5","login":"hubgit","name":"Alf Eaton","avatar_url":"https://avatars-next.observableusercontent.com/7f025ee242055990272e8933562c11766adf04e1a256eda25874741e864d6f4f","tier":"starter_2024"},"version":579,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"0955fee430221920","login":"hubgit","name":"Alf Eaton","avatar_url":"https://avatars-next.observableusercontent.com/7f025ee242055990272e8933562c11766adf04e1a256eda25874741e864d6f4f","tier":null}],"members":[{"type":"individual","id":"0955fee430221920","login":"hubgit","name":"Alf Eaton","avatar_url":"https://avatars-next.observableusercontent.com/7f025ee242055990272e8933562c11766adf04e1a256eda25874741e864d6f4f","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":3,"collection":null,"type":"notebook","body":{"title":"Exploring the Office for National Statistics API","cells":[{"id":0,"value":"md`# Exploring the Office for National Statistics API`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":149,"value":"md`The Office for National Statistics produces [an API](https://developer.ons.gov.uk/) (still in Beta) that makes datasets and other data available programmatically using HTTP.` ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":156,"value":"md`First, let's fetch a list of the datasets:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":158,"value":"datasets = d3.json(\"https://api.beta.ons.gov.uk/v1/datasets?limit=1000\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":175,"value":"md`Then choose a dataset to explore:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":420,"value":"viewof datasetId = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":178,"value":"dataset = datasets.items.find(item => item.id === datasetId)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":566,"value":"Have a look at the editions","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":564,"value":"editions = d3.json(\n \"https://api.beta.ons.gov.uk/v1/datasets/weekly-deaths-age-sex/editions\"\n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":513,"value":"dataset.qmi.href","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":181,"value":"md`Fetch the latest version:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":184,"value":"version = d3.json(dataset.links.latest_version.href)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":193,"value":"md`The data is published as [CSVW](https://www.w3.org/TR/tabular-data-primer/), which means there's a CSV data file plus a JSON file describing it.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":269,"value":"md`First, examine the description:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":192,"value":"metadata = d3.json(version.downloads.csvw.href)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":511,"value":"version.downloads.csv.href","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":202,"value":"md`Next, fetch the data for this version as CSV:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":197,"value":"items = d3.csv(version.downloads.csv.href, d3.autoType)\n\n// items = d3.csv(\n// withCors(\n// \"https://static.ons.gov.uk/datasets/weekly-deaths-age-sex-covid-19-v143.csv\"\n// ),\n// d3.autoType\n// )","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":226,"value":"html`

To view a summary of the table, try running this locally: \n\n

\ncurl -L '${version.downloads.csv.href}' \\\\\n  | xsv stats --everything \\\\\n  | xsv table\n

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":207,"value":"md`In theory it would be possible to combine the description with the data and produce linked data, but this isn't easy yet.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":210,"value":"// compacted = jsonld.expand(json) // needs CORS on https://cdn.ons.gov.uk/assets/json-ld/context.json","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":212,"value":"md`Instead, let's take the raw data and filter it:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":215,"value":"filtered = items\n // .filter((item) => item[\"recorded-deaths\"] === \"total-registered-deaths\")\n .filter((item) => item[\"registration-or-occurrence\"] === \"registrations\") // \"occurrences\"\n .filter(\n (item) =>\n item[\"administrative-geography\"] === \"E92000001\" ||\n item[\"administrative-geography\"] === \"W92000004\"\n ) // England or Wales\n .filter((item) => item[\"V4_0\"] !== null) // not empty\n .filter((item) => item[\"age-groups\"] === \"all-ages\")\n .filter((item) => item[\"sex\"] === \"all\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":292,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":217,"value":"md`Next, map each item to produce cleaner data:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":220,"value":"mapped = filtered.map((item) => ({\n year: item[\"calendar-years\"],\n week: Number(item[\"week-number\"].replace(/^week-/, \"\")), // this should be a number already?\n deaths: item[\"v4_0\"]\n}))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":310,"value":"md`The API doesn't provide historical data, so we'll need to manually collate [the data for previous years](https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/deaths/datasets/weeklyprovisionalfiguresondeathsregisteredinenglandandwales) and attach it here.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":313,"value":"historical = d3.csv(await FileAttachment(\"weekly-deaths-region-historical@1.csv\").url())","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":239,"value":"md`## Visualisation`\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":234,"value":" md`Finally, use [Vega-Lite](https://vega.github.io/vega-lite/) to visualise the data:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":101,"value":"graph = vl\n .markBar()\n .title(\"Deaths registered per week in England and Wales\")\n .data(mapped.concat(historical))\n .width({ step: 2 })\n .encode(\n vl.x().fieldO(\"year\").axis(false),\n vl\n .y()\n .sum(\"deaths\")\n .title(\"Deaths registered per week\")\n .axis({ grid: false }),\n vl.column().fieldO(\"week\").title(\"Week\").spacing(0),\n vl\n .color()\n .field(\"year\")\n .title(\"Year\")\n .scale({\n range: [\n \"#aaa\",\n \"#aaa\",\n \"#aaa\",\n \"#aaa\",\n \"#aaa\",\n \"#aaa\",\n \"#aaa\",\n \"#aaa\",\n \"#aaa\",\n \"#aaa\",\n \"#f75123\",\n \"#4a88f1\",\n \"#51a723\",\n \"#f7a723\",\n \"#000000\"\n ]\n })\n .legend({ orient: \"bottom\" }),\n vl.tooltip([\n { field: \"week\", type: \"nominal\" },\n { field: \"year\", type: \"nominal\" },\n { field: \"deaths\", type: \"quantitative\" }\n ])\n )","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":471,"value":"graph.render()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":385,"value":"md`Tap the vertical ellipsis next to the chart and select \"Download SVG\" or \"Download PNG\" to export the chart as an image.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":237,"value":"md`## Dependencies`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":98,"value":"import { vl } from '@hubgit/vega-lite'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":8,"value":"// jsonld = require('jsonld@4.0.1/dist/jsonld.js')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2,"value":"// d3 = import(\"https://cdn.skypack.dev/d3@6\")\nd3 = import(\"https://cdn.skypack.dev/pin/d3@v6.6.0-xTKZt8vyZkRYDoC4a9ZR/mode=imports,min/optimized/d3.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":288,"value":"import { escapeHTML } from '@hubgit/escape-html'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":496,"value":"import { withCors } from \"@hubgit/cors\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":359,"value":"md`## Styles`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":362,"value":"html`\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W31","stdlib":"1","titling":"auto"}} 30:"$2f:body"