29:null 2a:["$","title",null,{"children":"Generate Noisy Time Series Data | Jonathan Helfman | Observable"}] 2f:[] 2d:{"id":"2fc1e86d75a1fedb","slug":"generate-noisy-time-series-data","title":"Generate Noisy Time Series Data","thumbnail":"14fda240749e48fa8fa533d27c8d50add02d87cba02c0782fc648294b60b9e39","access_level":"public","create_time":"$D2023-03-01T15:28:43.754Z","update_time":"$D2023-08-11T17:01:06.268Z","view_time":null,"first_public_time":"$D2023-03-01T19:15:45.233Z","has_unpublished_edits":null,"fork":false,"stars":2,"starred":false,"owner":{"type":"team","id":"6c3b7072b5e94b97","login":"jonhelfman","name":"Jonathan Helfman","avatar_url":"https://avatars-next.observableusercontent.com/a404535763c16060490629d896d876b5b4cea8c603c4993b2aaed749a15b041a","tier":"starter_2024"},"version":321,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"ba5c813f0f621f19","login":"jonhelfman","name":"Jonathan Helfman","avatar_url":"https://avatars-next.observableusercontent.com/a404535763c16060490629d896d876b5b4cea8c603c4993b2aaed749a15b041a","tier":null}],"members":[{"type":"individual","id":"ba5c813f0f621f19","login":"jonhelfman","name":"Jonathan Helfman","avatar_url":"https://avatars-next.observableusercontent.com/a404535763c16060490629d896d876b5b4cea8c603c4993b2aaed749a15b041a","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"title":"Generate Noisy Time Series Data","cells":[{"id":0,"value":"# Generate Noisy Time Series Data\n\nRewritten from https://observablehq.com/@kgryte/stdlib-simulating-noisy-time-series-using-discrete-time-ma\nto use what seem to be more reliable dependencies (d3, Inputs, and Observable Plot)\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":237,"value":"Plot.plot({\n height: 150,\n caption: \"N-state corrupted time series\",\n marks: [\n //Plot.ruleY([0]),\n Plot.lineY(data, { x: \"date\", y: \"value\" })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":47,"value":"viewof n = Inputs.range([2, 1024], { label: \"n\", step: 1 })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":157,"value":"Binary Markov chains\n--","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":31,"value":"Plot.plot({\n height: 150,\n marks: [Plot.lineY(binarySequence)]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":276,"value":"Plot.plot({\n height: 150,\n marks: [Plot.lineY(binarySeries, { x: \"date\", y: \"value\" })]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"initial = [0.9, 0.1] // initial probabilities should sum to 1","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"transition = [\n [0.95, 0.05],\n [0.9, 0.1]\n] // transition probabilities, each row should sum to 1","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":104,"value":"initial_state = d3.randomUniform()() <= initial[0] ? 0 : 1","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":120,"value":"binarySequence = binaryStateSequence(initial_state, transition, n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":272,"value":"binarySeries = binaryStateSeries(initial_state, transition, start, end)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":154,"value":"N-state Markov chains\n--","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":52,"value":"Plot.plot({\n height: 150,\n caption: \"N-state sequence\",\n marks: [Plot.lineY(seq, { stroke: \"skyblue\" })]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":45,"value":"seq = nStateSequence(I, A, n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":95,"value":"Plot.plot({\n height: 150,\n caption: \"True N-state sequence\",\n marks: [Plot.lineY(trueObs, { stroke: \"skyblue\" })]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":183,"value":"Plot.plot({\n height: 150,\n caption: \"N-state corrupted sequence\",\n marks: [Plot.lineY(obs)]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":92,"value":"obs = corrupt(seq, means, sigmas)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":131,"value":"trueObs = trueStateSequence(seq, means)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":37,"value":"I = [ 0.1, 0.3, 0.2, 0.1, 0.3 ] // initial probabilities should sum to 1","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":39,"value":"A = {\n let transition = new Array(5);\n\n // Transition probabilities when in state s_0:\n transition[0] = [0.2, 0.7, 0.1, 0.0, 0.0]; // should sum to 1\n\n // Transition probabilities when in state s_1:\n transition[1] = [0.4, 0.3, 0.25, 0.05, 0.0]; // should sum to 1\n\n // Transition probabilities when in state s_2:\n transition[2] = [0.05, 0.2, 0.4, 0.3, 0.05]; // should sum to 1\n\n // Transition probabilities when in state s_3:\n transition[3] = [0.02, 0.18, 0.2, 0.2, 0.4]; // should sum to 1\n\n // Transition probabilities when in state s_4:\n transition[4] = [0.3, 0.0, 0.0, 0.2, 0.5]; // should sum to 1\n\n return transition; // transition probabilities\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":169,"value":"Also assign each state an observation value (i.e., the value you would observe in the absence of noise, which for Gaussian noise, is the mean observed value when a simulated process is in state 's_i')","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":82,"value":"means = [0.2, 0.35, 0.5, 0.6, 0.8]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":174,"value":"md`and an expected standard deviation (i.e., how much do we expect each observed value to vary from the true state value)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":84,"value":"sigmas = [0.1, 0.1, 0.1, 0.1, 0.1]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":192,"value":"Adding dates\n--\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":223,"value":"Plot.plot({\n height: 150,\n caption: \"N-state corrupted time series\",\n marks: [\n //Plot.ruleY([0]),\n Plot.lineY(data, { x: \"date\", y: \"value\" })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":218,"value":"data = obs.map((d, i) => ({ value: d, date: hours[i] }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":216,"value":"hours = time.timeHour.range(start, end)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":194,"value":"start = new Date(2023, 0, 1)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":233,"value":"Assume we want a value for every hour","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":197,"value":"end = new Date(2023, 0, 1 + parseInt(n / 24), n % 24)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":201,"value":"d3.timeHour.count(start, end)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":248,"value":"Nicer API\n--","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":252,"value":"Plot.plot({\n height: 150,\n caption: \"N-state corrupted time series\",\n marks: [\n //Plot.ruleY([0]),\n Plot.lineY(data2, { x: \"date\", y: \"value\" })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":250,"value":"data2 = generate(I, A, means, sigmas, start, end, 23)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":311,"value":"Plot.plot({\n height: 150,\n caption: \"Simplex noise time series\",\n marks: [\n //Plot.ruleY([0]),\n Plot.lineY(simplexData, { x: \"date\", y: \"value\" })\n ]\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":306,"value":"simplexData = simplex1d(n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":318,"value":"d3.extent(simplexData, (d) => d.value)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":56,"value":"Functions\n--","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":24,"value":"binaryStateSequence = (initialState, transition, n = 100) => {\n let states;\n let curr;\n let i;\n\n states = new Array(n);\n states[0] = initialState;\n curr = initialState;\n for (i = 1; i < n; i++) {\n curr = d3.randomUniform()() <= transition[curr][0] ? 0 : 1;\n states[i] = curr;\n }\n return states;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":268,"value":"binaryStateSeries = (initialState, transition, start, end) => {\n let states;\n let curr;\n let i;\n const n = d3.timeHour.count(start, end);\n\n states = new Array(n);\n states[0] = initialState;\n curr = initialState;\n for (i = 1; i < n; i++) {\n curr = d3.randomUniform()() <= transition[curr][0] ? 0 : 1;\n states[i] = curr;\n }\n const hours = time.timeHour.range(start, end);\n return states.map((d, i) => ({ date: hours[i], value: d }));\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":41,"value":"function nStateSequence (initial, transition, n) {\n let nStates;\n let states;\n let probs;\n let r;\n let i;\n let j;\n const randu = d3.randomUniform();\n\n nStates = initial.length;\n\n // Convert the initial probabilities to cumulative probabilities:\n initial = d3.cumsum(initial);\n\n // Convert each set of transition probabilities to cumulative probabilities:\n transition = transition.slice(); // avoid mutation\n for (i = 0; i < nStates; i++) {\n transition[i] = d3.cumsum(transition[i]);\n }\n\n // Define the output state array:\n states = new Array(n);\n\n // Determine the initial state:\n r = randu();\n for (i = 0; i < nStates; i++) {\n if (r < initial[i]) {\n states[0] = i;\n break;\n }\n }\n // Simulate the next `n-1` steps:\n for (i = 0; i < n - 1; i++) {\n probs = transition[states[i]];\n r = randu();\n for (j = 0; j < nStates; j++) {\n if (r < probs[j]) {\n states[i + 1] = j;\n break;\n }\n }\n }\n return states;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":86,"value":"function trueStateSequence(seq, means) {\n var states;\n var i;\n\n states = new Array(seq.length);\n for (i = 0; i < seq.length; i++) {\n states[i] = means[seq[i]];\n }\n return states;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":89,"value":"function corrupt(states, means, sigmas, scaleFactor = 1.0) {\n var state;\n var out;\n var i;\n\n // Initialize an output array:\n out = new Array(states.length);\n\n // For each state, simulate a corrupted observation:\n for (i = 0; i < states.length; i++) {\n state = states[i];\n out[i] = scaleFactor * d3.randomNormal(means[state], sigmas[state])();\n }\n return out;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":240,"value":"generate = (\n initial,\n transition,\n means,\n sigmas,\n start,\n end,\n scaleFactor = 1.0\n) => {\n const n = d3.timeHour.count(start, end);\n const seq = nStateSequence(initial, transition, n);\n const obs = corrupt(seq, means, sigmas, scaleFactor);\n const hours = time.timeHour.range(start, end);\n return obs.map((d, i) => ({ value: d, date: hours[i] }));\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":320,"value":"generateMinutes = (\n initial,\n transition,\n means,\n sigmas,\n start,\n end,\n scaleFactor = 1.0\n) => {\n const n = d3.timeMinute.count(start, end);\n const seq = nStateSequence(initial, transition, n);\n const obs = corrupt(seq, means, sigmas, scaleFactor);\n const minutes = time.timeMinute.range(start, end);\n return obs.map((d, i) => ({ value: d, date: minutes[i] }));\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":299,"value":"simplex = new SimplexNoise()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":297,"value":"simplex1d = (n) => {\n const hours = time.timeHour.range(start, end);\n\n const sdata = d3.range(n).map((d, i) => simplex.noise2D(d, 1));\n\n return sdata.map((d, i) => ({ date: hours[i], value: d }));\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":189,"value":"Imports\n--","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":187,"value":"time = require(\"d3-time@1.1.0\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":293,"value":"SimplexNoise = require(\"simplex-noise@2.4\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W2f","stdlib":"1","titling":"auto"}} 2e:"$2d:body"