2c:null 2d:["$","title",null,{"children":"Discovering Date Patterns | Alessandra Sozzi | Observable"}] 32:T465,addTooltips( Plot.plot({ width, facet: { data: data, x: x_facet, // break out horizontally y: y_facet, // break out horizontally marginLeft: y_facet === "week" || y_facet === "hour" ? 30 : 80 }, fy: { label: null, ...get_facet_options(y_facet, "y") // get settings based on selection }, fx: { label: x_facet === "week" ? "Week" : null, ...get_facet_options(x_facet) // get settings based on selection }, marks: [ x_facet !== "week" ? Plot.frame() : null, // don't display a frame if viewing week // Create rectangles for background color Plot.rect( data, Plot.groupZ( { fill: "mean", title: get_title }, { fill: "value", fillOpacity: 0.4, } ) ), // Display ticks Plot.tickX(data, { x: "value", strokeOpacity: 0.2 }) ], color: { scheme: "plasma", reverse: true }, x: { label: "Electricity Demand (MWh) →" } }), { stroke: "black", "stroke-width": "3px" } // tooltip styles )33:[{"id":"459facd602c61caa398fee8016ddb564c1874ee9413c6ae3357fe1e3c05edf20e1d766ba2c8465403e0fb7c29a43d8728a1ce88e5ff36d49d122f2fdad33d04a","href":"https://static.observableusercontent.com/files/459facd602c61caa398fee8016ddb564c1874ee9413c6ae3357fe1e3c05edf20e1d766ba2c8465403e0fb7c29a43d8728a1ce88e5ff36d49d122f2fdad33d04a","name":"california.csv","type":"text/csv","lastModified":1686908702491.095,"size":764226},{"id":"246efc36145aac89fb4cd59349284b9dc157c7d5d9c9733e59a0df0f8693dc7e4a229de442fbf26764beecd79ff9c4ac76a7c6d6ab8b89f37972a7d4eca38aee","href":"https://static.observableusercontent.com/files/246efc36145aac89fb4cd59349284b9dc157c7d5d9c9733e59a0df0f8693dc7e4a229de442fbf26764beecd79ff9c4ac76a7c6d6ab8b89f37972a7d4eca38aee","name":"right@1.png","type":"image/png","lastModified":1686908702491.095,"size":195854},{"id":"6aac3c5783ed11e6aecd30a3e5878f0327fd0dab1d97affde8de0fe597d29b806c12887fd72ecf1472b0365bee196f13ecc49d4d49742544fa49ea87e403ed35","href":"https://static.observableusercontent.com/files/6aac3c5783ed11e6aecd30a3e5878f0327fd0dab1d97affde8de0fe597d29b806c12887fd72ecf1472b0365bee196f13ecc49d4d49742544fa49ea87e403ed35","name":"leftcropped.png","type":"image/png","lastModified":1686908702491.095,"size":202794},{"id":"7bd5d071fd038c265464a5445487b4a42380435032cc7bb8a58d411217ca5a8309564b06f1ef324caab0b625d1b5ba4100e1d7033e0586e3c703b469e0e189a1","href":"https://static.observableusercontent.com/files/7bd5d071fd038c265464a5445487b4a42380435032cc7bb8a58d411217ca5a8309564b06f1ef324caab0b625d1b5ba4100e1d7033e0586e3c703b469e0e189a1","name":"4-yearly.png","type":"image/png","lastModified":1686908702491.095,"size":38573},{"id":"19f915a7198ee3020f1e503615047c040c5161251fc0ba8b8e19637c5788b6a226ec44f2f414e6c8fb8f86a5dbc175cb519dc4e13f3bf606c990b4054f51d5c6","href":"https://static.observableusercontent.com/files/19f915a7198ee3020f1e503615047c040c5161251fc0ba8b8e19637c5788b6a226ec44f2f414e6c8fb8f86a5dbc175cb519dc4e13f3bf606c990b4054f51d5c6","name":"1-hourly.png","type":"image/png","lastModified":1686908702491.095,"size":41569},{"id":"173374ea1f17d75451f498d4f25b1f675ecf0113280429780ea60791dbc53bfefe0cf3631cdfba90baf717e2911ea753cbdf8d0818cf6c2ae88d050d279f9263","href":"https://static.observableusercontent.com/files/173374ea1f17d75451f498d4f25b1f675ecf0113280429780ea60791dbc53bfefe0cf3631cdfba90baf717e2911ea753cbdf8d0818cf6c2ae88d050d279f9263","name":"2-daily.png","type":"image/png","lastModified":1686908702491.095,"size":36927},{"id":"b4d20223aab0b21bb03c1bc7f1b29a413cd5ed3eec790984740b9d91d33a5de11e97f420708cb739c7c3d99f831be39be7430b667f7d73cd62a10a4ddb6041b0","href":"https://static.observableusercontent.com/files/b4d20223aab0b21bb03c1bc7f1b29a413cd5ed3eec790984740b9d91d33a5de11e97f420708cb739c7c3d99f831be39be7430b667f7d73cd62a10a4ddb6041b0","name":"3-weekly.png","type":"image/png","lastModified":1686908702491.095,"size":56056}] 30:{"id":"73097c728ce55931","slug":null,"title":"Discovering Date Patterns","thumbnail":"11b76bf57df7072d9a3d9c98b729dedfa183c0bd3e67412cbbeb85dcc34d9ee4","access_level":"public","create_time":"$D2023-06-16T09:45:02.491Z","update_time":"$D2023-06-16T09:45:02.491Z","view_time":null,"first_public_time":"$D2023-06-16T09:45:02.491Z","has_unpublished_edits":null,"fork":{"id":"af591412a9582f6a","slug":"discovering-date-patterns","title":"Discovering Date Patterns","owner":{"id":"f35c755083683fe5","login":"observablehq","name":"Observable"}},"stars":0,"starred":false,"owner":{"type":"team","id":"feafd296ad17b11e","login":"alessandrasozzi","name":"Alessandra Sozzi","avatar_url":"https://avatars-next.observableusercontent.com/5e9d0797182c08ccefb659313caa998626fdf2ce2bfc2ffc322b83243c07d18b","tier":"starter_2024"},"version":1762,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"6e90bfac93d169be","login":"alessandrasozzi","name":"Alessandra Sozzi","avatar_url":"https://avatars-next.observableusercontent.com/5e9d0797182c08ccefb659313caa998626fdf2ce2bfc2ffc322b83243c07d18b","tier":null}],"members":[{"type":"individual","id":"6e90bfac93d169be","login":"alessandrasozzi","name":"Alessandra Sozzi","avatar_url":"https://avatars-next.observableusercontent.com/5e9d0797182c08ccefb659313caa998626fdf2ce2bfc2ffc322b83243c07d18b","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Discovering Date Patterns","cells":[{"id":0,"value":"# Discovering Date Patterns\n## If the only feature you have is the date, how can you uncover meaningful patterns?","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1600,"value":"authorship([\"Mike Freeman\", \"Zan Armstrong\", \"Ian Johnson\"])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1661,"value":"---","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"Imagine you have a new project where you're tracking a performance metric hourly— you have **a full year of hourly data**, but no additional information beyond the timestamp and the metric. Before bringing in any external data, you're asked to extract meaningful patterns from the data. (_See [here](https://observablehq.com/@observablehq/correlation-over-time) for an example joining temperature data._)\n\nAs an example, we'll use hourly electricity demand data in the California region from the [OpenData API](https://www.eia.gov/opendata/), downloaded using [this notebook](https://observablehq.com/@observablehq/eia-opendata-electricity-grid-operation?collection=@observablehq/analyzing-time-series-data).","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"raw_data = await FileAttachment(\"california.csv\").csv({typed:true})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":983,"value":"Inputs.table(raw_data, {columns:[\"date\", \"value\"], width: width < 400 ? width : 400})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1335,"value":"*Note: The \"Z\" indicates that the timestamps are stored in UTC*","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":80,"value":"## Detecting meaningful patterns","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":92,"value":"You start by looking at the full timeseries, where you notice some _seasonal fluctuations_ as well as daily noise. To provide a clearer sense of the seasonality, you can use [Observable Plot](https://observablehq.com/collection/@observablehq/plot) to smooth the data using a rolling average, and also zoom in on any dates of interest using the controls below. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":124,"value":"viewof smooth = Inputs.radio(new Map([[\"Hourly (no smoothing)\", 1] ,[\"Daily\", 24], [\"Weekly\", 168], [\"28-day period\", 24 * 28]]), {value:1, label:\"Rolling Average:\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":153,"value":"viewof limits = brushFilterX(\n d3.extent(data, (d) => d.date),\n { defaultExtent: [d3.isoParse(\"2020-02-01\"), d3.isoParse(\"2020-10-10\")] }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":85,"value":"// Create a line plot showing the rolling average\nPlot.line(\n data,\n // compute the rolling average, show values in the brush selection\n Plot.windowY({\n x: d => (!limits || d.date >= limits[0] && d.date <= limits[1]) ? d.date : NaN,\n y: \"value\",\n k: smooth\n })\n).plot({\n marks: [Plot.ruleY([0])], // draw a line at x=0\n marginLeft: 50,\n x: { label: \"date (utc)→\", domain: limits || undefined },\n y: {// Filtering down based on the brush\n label: \"↑ Electricity Demand (MWh)\",\n domain: [0, d3.max(data, (d) => d.value)]\n },\n width,\n height: 400,\n caption: md`Note, this displays the x-axis in UTC time, not the local time for California.`\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1486,"value":"Here are a few patterns that emerge:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1682,"value":"

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":25,"value":"## Creating date features","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"So far you've just been calculating rolling averages over the hourly data to reveal patterns at the daily, weekly, and monthly-ish periods ([1](https://observablehq.com/@observablehq/discovering-date-patterns#cell-1240)). \n\nHowever, you can go further and augment the data, and then compare our data using those features. As an analyst, you know that embedded in the date are meaningful categories including:\n- Hour of the day\n- Day of the week\n- Weekend v.s. weekday\n- Week of the year\n- Month of the year\n- Quarter of the year","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":859,"value":"In the cell below we use the [Temporal Polyfill](https://github.com/tc39/proposal-temporal) to extract this information (in the appropriate timezone):","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":523,"value":"// Extract the desired information from each timestamp\ndata = raw_data.map((d) => {\n const tzdate = Temporal.Instant.from(d.date).toZonedDateTimeISO(\n \"America/Los_Angeles\"\n ); // date in localized timezone\n const date = new Date(tzdate.epochMilliseconds); // Javascript Date object, stores date in browser's local timezone\n const hour = tzdate.hour;\n const dayOfMonth = tzdate.day;\n const day = weekdays[tzdate.dayOfWeek + -1];\n const weekend = tzdate.dayOfWeek > 5 ? \"Weekend\" : \"Weekday\";\n const week = tzdate.weekOfYear;\n const month = months[tzdate.month - 1];\n const quarter =\n tzdate.month < 4\n ? \"Q1: Jan - Mar\"\n : tzdate.month < 7\n ? \"Q2: Apr - Jun\"\n : tzdate.month < 10\n ? \"Q3: Jul - Sep\"\n : \"Q4: Oct - Dec\";\n const value = d.value;\n return {\n date,\n hour,\n day,\n weekend,\n week,\n month,\n quarter,\n value,\n dayOfMonth,\n };\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":867,"value":"You can now compare our metric of interest (electricity demand) to **7 different date features** instead of just one:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":54,"value":"Inputs.table(data)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":879,"value":"---","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":284,"value":"## Breaking down hourly patterns into days, weeks, and months\nNow that you've augmented your data with features like \"Quarter\", you can use these to facet our data. This allows for comparisons across categories, and across intersections of categories (e.g., how does the hourly demand vary within a given quarter?). \n\nFrom the analysis above, it's clear that:\n- Certain hours of the day tend have the higher electricity demand\n- The fluctuations we see vary by time of year. \n\nBased on these observations, you want to look at how the distribution of energy usage compares across quarters and hours. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1692,"value":"

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1315,"value":"Using Observable, you create an interactive visualization so others can explore the data by selecting *Hour*, *Day*, *Weekend*, *Week*, *Month*, *Quarter*, or *None* to define the columns and rows of the chart (also known as _faceting_).\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":379,"value":"viewof x_facet = Inputs.radio(facet_options, {\n label: \"Break out Horizontally by:\",\n value: \"quarter\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":960,"value":"viewof y_facet = Inputs.radio(facet_options, {\n label: \"Break out Vertically by:\",\n value: \"hour\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":311,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1753,"value":"data[0]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1354,"value":"💡 *If you try \"Day\" and \"Week\", you'll see every day of the year!*","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":410,"value":"## Closing","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":414,"value":"Even when your data is “just” a timestamp and a value, you can explore it deeply through rolling averages with different windows or by augmenting the data into familiar time-based categories like quarters or weekends. Different breakdowns will reveal different aspects of the data. \n\nWant to do see this analysis with your own data? Just replace the `raw_data` cell with your own data in the same format (two columns: `date`, and `value`). ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1610,"value":"navigation(3)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1256,"value":"-------","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1226,"value":"#### Footnotes\n\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1240,"value":"1. 28-day period: Aggregating by month can lead to spurious variation when data has strong day of week patterns. A good alternative is to use 28-day, 4-week, periods to approximate months in data that is likely to have a strong day-of-week seasonal pattern. If the month boundaries are relevant to your data, perhaps when analyzing data about rent that's due monthly, then it's better to use months. Either way we want to match the aggregation to the context of the data we're analyzing. To learn more, watch [Everything is Seasonal](https://www.youtube.com/watch?v=IiF4-g001EQ>).","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1208,"value":"-----","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":67,"value":"## Appendix","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":150,"value":"import {brushFilterX} from \"@observablehq/brush-filter-x\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":911,"value":"import {addTooltips} from \"@mkfreeman/plot-tooltip\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1598,"value":"import {authorship, readMore, navigation, workshop} from \"@observablehq/timeseries-assets\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1006,"value":"// Options for breaking the charts into different facets\nfacet_options = new Map([\"Hour\", \"Day\", \"Weekend\", \"Week\", \"Month\", \"Quarter\", \"None\"].map(d => [d, d.toLowerCase()]))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1067,"value":"// Labels on hover\nget_title = (d) => {\n // Compute average\n const avg = d3.format(\",.0f\")(d3.mean(d, (d) => d.value));\n const x_value = x_facet === \"hour\" ? d[0].hour + \":00 PST\" : d[0][x_facet]\n const y_value = y_facet === \"hour\" ? d[0].hour + \":00 PST\" : d[0][y_facet]\n return `Avg. Demand: ${avg} MWh \\n` + \n (x_facet == 'none' ? \"\" : `${capitalize(x_facet)}: ${x_value}\\n `) + \n (y_facet == \"none\" ? \"\" : `${capitalize(y_facet)}: ${y_value}`);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1064,"value":"// Label for hours\nlabel_hour = (d) => d === \"hour\" ? \":00 PST\" : \"\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1082,"value":"// Setting the domain and tick format depending on the facet\nget_facet_options = (facet, direction = \"x\") => facet === \"day\"\n ? { domain: weekdays, tickFormat:d => (direction === \"x\" && width < 600) ?d.slice(0, 1) :d}\n : facet === \"month\"\n ? { domain: months, tickFormat: d => (direction === \"x\" && width < 600) ? d.slice(0, 1) : d }\n : (facet === \"hour\" && width > 800) ? {tickFormat: d => d + \":00\"}:{}\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1049,"value":"// Capitalize: https://stackoverflow.com/questions/1026069/how-do-i-make-the-first-letter-of-a-string-uppercase-in-javascript\ncapitalize = (string) => {\n return string.charAt(0).toUpperCase() + string.slice(1);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":57,"value":"months = [\"January\", \"February\", \"March\", \"April\", \"May\", \"June\", \"July\", \"August\", \"September\", \"October\", \"November\", \"December\"]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":59,"value":"weekdays = [\"Monday\", \"Tuesday\", \"Wednesday\", \"Thursday\", \"Friday\", \"Saturday\", \"Sunday\"]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1368,"value":"// Load the Temporal API using a Polyfill\nTemporal = {\n const TemporalLib = await require(\"@js-temporal/polyfill@0.3.0\")\n return TemporalLib.Temporal\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1531,"value":"\n","pinned":true,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}} 31:"$30:body"