41:null 42:["$","title",null,{"children":"A Random Walk Through the Grid World | Jim Kan | Observable"}] 48:T42d,displayedEpisodes = { const [newEpisodeNumber, newEpisode] = currentEpisode; if (!this) return [currentEpisode]; const [curEpisodeNumber, curEpisode] = this[this.length - 1]; if (newEpisodeNumber < curEpisodeNumber) { // reset return [currentEpisode]; } else if (newEpisodeNumber === curEpisodeNumber) { this[this.length - 1] = currentEpisode; return this; } else { return this.concat([currentEpisode]).slice(-numberOfEpisodesToDisplay); } } // randomWalks = d3.range(numberOfEpisodes).map(() => episode(randomWalkPolicy, maxPathLength)) // walks = slowEpisodes(env, numberOfEpisodes, epsilonGreedyPolicy, maxPathLength, firstVisitMC, 500) // walks = slowEpisodes(env, numberOfEpisodes, randomWalkPolicy, maxPathLength, noLearning, 50) // walks = slowEpisodes(env, numberOfEpisodes, epsilonGreedyPolicy, maxPathLength, false, 50) // walks = entireEpisodes(env, numberOfEpisodes, epsilonGreedyPolicy, maxPathLength, sarsaLearnQ, 200) // walks = genExperiment(env, currentAgent) // walks = genExperiment(env, sarsa)49:T544,function* genEpisodeStep(env, policyFn, episodeNumber, limit) { let { state: state, observation: observation } = reset(env); let action = chooseAction(env, observation, policyFn); let done = false, i = 0; while (!done && i < limit) { const result = step(env, state, action); const nextAction = !result.done && chooseAction(env, result.observation, policyFn); yield { current: { observation: observation, action: action, _state: state // Technically, the agent can not directly access the current state; // it can only make observations that are possibly noisy measurements of // the current state -- but it's convenient to keep it here for // visualization purposes, so we slap the underscore in front here. }, reward: result.reward, next: { observation: result.observation, action: nextAction, _state: result.state // technically, state is hidden... hence the underscore }, done: result.done, episodeNumber: episodeNumber, stepNumber: i }; done = result.done; state = result.state; observation = result.observation; action = nextAction; ++i; } } // episode = (policy, limit) => [...genEpisodeStep(env, policy, limit)]4a:T48e,viewof sigma = { const form = html` `; function update() { const sigma_x = form.sigma_x.valueAsNumber; const sigma_y = form.sigma_y.valueAsNumber; const p = form.p.valueAsNumber; const cov_xy = p * sigma_x * sigma_y const sigma = [[sigma_x*sigma_x, cov_xy], [cov_xy, sigma_y * sigma_y]].map(vv => vv.map(round)) form.value = sigma; form.value.sigma_x = sigma_x; form.value.sigma_y = sigma_y; form.value.p = p; } form.sigma_x.oninput = update; form.sigma_y.oninput = update; form.p.oninput = update; update(); return form; }4b:T1dea,displayGridWorld = (displayDimensions, rewardGrid, displayedEpisodes, Q, targetPolicyEpisode) => { const svgW = displayDimensions.margin.left + displayDimensions.x + displayDimensions.margin.right; const svgH = displayDimensions.margin.bottom + displayDimensions.y + displayDimensions.margin.top; const scales = ({ x: d3.scaleLinear() .domain([0, rewardGrid.length]) .range([displayDimensions.margin.left, displayDimensions.margin.left + displayDimensions.x]), y: d3.scaleLinear() .domain([0, rewardGrid[0].length]) .range([displayDimensions.y + displayDimensions.margin.top, displayDimensions.margin.top + 1]) }); const cellW = 1. + scales.x(1)-scales.x(0); const cellH = 1. + scales.y(0)-scales.y(1); function translateState(state) { const x = scales.x(state[0])+cellW/2; const y = scales.y(state[1])-cellH/2; return `translate(${x}, ${y})`; } const arrowRot = { N: 0, E: 90, W: -90, S: 180 }; const radius = Math.min(cellW, cellH) * 0.4; function rewardSurface(svg) { const surface = svg.append('g'); const colorScale = Math.max(Math.abs(matrixMin(rewardGrid)), Math.abs(matrixMax(rewardGrid))) const fillFn = d3.scaleSequential(d3.interpolateRdBu).domain([-colorScale, colorScale]); surface.selectAll('g') .data(rewardGrid) .enter().append('g') .attr("transform", (d, i) => `translate(${scales.x(i)}, 0)`) .selectAll("rect") .data(d => d) .enter().append("rect") .attr("fill", fillFn) .attr("y", (d, i) => scales.y(i)-cellH) .attr("opacity", 0.5) .attr("width", cellW) .attr("height", cellH); return svg; } const returnExtent = d3.extent(displayedEpisodes, (episode) => discountedReturn(episode)); const finalFillFn = d3.scaleSequential(d3.interpolateReds).domain( [Math.min(0, returnExtent[0]), returnExtent[1]]); const lineFn = d3.line() .x(([x,y]) => scales.x(x)) .y(([x,y]) => scales.y(y)) .curve(d3.curveLinear); // .curve(d3.curveCardinal.tension(0.75)); function episodePath(svg, episode, emphasizePath = false) { if (!episode.length) return; const trajectory = svg.append('g'); const finalStep = episode[episode.length-1]; // const finalFill = finalFillFn(discountedReturn(episode)); if (emphasizePath) { const steps = trajectory.selectAll('g') .data(episode) .enter().append('g') .attr('transform', (d) => translateState(d.current._state)); steps.append('circle') .attr('stroke', 'none') .attr('fill', '#555') .attr('opacity', 0.2) .attr("cx", '0') .attr("cy", '0') .attr("r", radius); steps.append('text') .attr('stroke', 'white') .attr('fill', 'none') .attr('opacity', 0.2) .attr('font-size', `${radius * 0.7}px`) .attr('text-anchor', 'middle') .attr('transform', (d) => `rotate(${arrowRot[d.current.action]}) translate(0, -${radius * 0.3})`) .attr('x', '0') .attr('y', '0') .text('⇧'); } trajectory.append('circle') .attr('transform', translateState(episode[0].current._state)) .attr('stroke', 'none') .attr('fill', 'green') .attr('opacity', 1) .attr("cx", '0') .attr("cy", '0') .attr("r", radius); trajectory.append('circle') .attr('transform', translateState(finalStep.next._state)) .attr('stroke', 'red') .attr('stroke-width', 3) .attr('fill', finalStep.done ? 'red' : 'none') // finalFill .attr('opacity', emphasizePath ? 1.0 : 0.4) .attr("cx", '0') .attr("cy", '0') .attr("r", radius); const path = episodeToPath(episode); trajectory.append('path') .datum(path) .attr('transform', `translate(${cellW/2}, -${cellH/2})`) .attr('fill', 'none') .attr('stroke', '#555') .attr('stroke-width', 7) .attr('stroke-linejoin', 'round') .attr('stroke-linecap', 'round') .attr('opacity', 0.2) .attr('d', lineFn); if (emphasizePath) { // let's draw it twice for emphasis... trajectory.append('path') .datum(path) .attr('transform', `translate(${cellW/2}, -${cellH/2})`) .attr('fill', 'none') .attr('stroke', 'white') .attr('stroke-width', 3) .attr('stroke-linejoin', 'round') .attr('stroke-linecap', 'round') .attr('opacity', 0.8) .attr('d', lineFn); } return svg; } function displayQ(svg, Q) { const keys = Object.keys(Q).map((s) => s.split(',').map((s) => parseInt(s, 10))); const qValues = Object.values(Q).map((v) => Object.values(v)); const colorScale = Math.max(Math.abs(matrixMin(qValues)), Math.abs(matrixMax(qValues))) // const colorScale = 100; const colorFn = d3.scaleSequential(d3.interpolateRdBu).domain([-colorScale, colorScale]); const qLayer = svg.append('g'); const steps = qLayer.selectAll('g') .data(keys) .enter().append('g') .attr('transform', (d) => translateState(d)); steps.selectAll('text') .data((d) => { const maxValue = Math.max(...Object.values(Q[d])); return Object.entries(Q[d]).map(([action, value]) => ({ action: action, value: value, isMax: (value == maxValue) })); }) .enter().append('text') .attr('stroke', d => colorFn(d.value)) .attr('fill', d => colorFn(d.value)) .attr('font-size', `${radius * 0.7}px`) // d => `${radius * (d.isMax ? 0.9 : 0.7)}px`) .attr('opacity', d => d.isMax ? 1 : 0.6) .attr('text-anchor', 'middle') // .attr('text-shadow', '5px 5px 20px black') .attr('transform', d => `rotate(${arrowRot[d.action]}) translate(0, -${radius * 0.3})`) .attr("x", '0') .attr("y", '0') // .text(([action, value]) => action); .text(d => d.isMax ? '⇪' : '⇧'); return svg; } function axes(svg) { const xAxis = d3.axisBottom() .scale(scales.x); const yAxis = d3.axisLeft() .scale(scales.y); svg.append('g') .attr('transform', `translate(0, ${displayDimensions.y + displayDimensions.margin.top})`) .call(xAxis); svg.append('g') .attr('transform', `translate(${displayDimensions.margin.left}, 0)`) .call(yAxis); return svg; } const svg = d3.select(DOM.svg(svgW, svgH)); const _showPathCircles = !targetPolicyEpisode && showPathCircles; // if there's a target policy episode, show the path circles on that instead of the most recent // episode with the behavior policy rewardSurface(svg); displayedEpisodes.slice(0, -1).forEach(([i, episode]) => episodePath(svg, episode, false)); displayedEpisodes.slice(-1).forEach(([i, episode]) => episodePath(svg, episode, _showPathCircles)); // only show circles on most recent episode (unless there's a target policy episode) if (targetPolicyEpisode) { episodePath(svg, targetPolicyEpisode, true); } if (Q) displayQ(svg, Q); axes(svg); return svg.node(); }4c:T7b1,function plotStats(displayDimensions, data, ...statFns) { const width = displayDimensions.x, height = displayDimensions.y * 0.4, marginLeft = displayDimensions.margin.left * 2, marginTop = displayDimensions.margin.top * 2; const svgW = marginLeft * 2 + width; const svgH = marginTop * 2 + height; const allYs = [].concat(...data.map(d => statFns.map(fn => fn(d)))); const yExtent = d3.extent(allYs); const scales = { x: d3.scaleLinear() .domain([0, data.length]) .range([marginLeft, marginLeft + width]), y: d3.scaleLinear() .domain([Math.min(yExtent[0], 0), yExtent[1]]) .range([marginTop + height, marginTop]) }; const lines = statFns.map(statFn => d3.line() .x((d, i) => scales.x(i)) .y(d => scales.y(statFn(d)))); const xAxis = d3.select(svg``) .attr('transform', `translate(0, ${marginTop + height})`) .call(d3.axisBottom(scales.x)) .node(); const yAxis = d3.select(svg``) .attr('transform', `translate(${marginLeft}, 0)`) .call(d3.axisLeft(scales.y)) .node(); const lineColors = d3.scaleOrdinal(d3.schemeCategory10); // ${memory.stats.map(([len, ret], i) => svg` // // `)} return svg` `; }4d:T751,statsPlot = (statFn, data, displayDimensions) => plotStats(displayDimensions, data, statFn) // { // const width = displayDimensions.x, // height = displayDimensions.y * 0.4, // marginLeft = displayDimensions.margin.left * 2, // marginTop = displayDimensions.margin.top * 2; // const svgW = marginLeft * 2 + width; // const svgH = marginTop * 2 + height; // const yExtent = d3.extent(data, d => statFn(d)); // const scales = { // x: d3.scaleLinear() // .domain([0, data.length]) // .range([marginLeft, marginLeft + width]), // y: d3.scaleLinear() // .domain([Math.min(yExtent[0], 0), yExtent[1]]) // .range([marginTop + height, marginTop]) // }; // const line = d3.line() // .x((d, i) => scales.x(i)) // .y(d => scales.y(statFn(d))); // const xAxis = d3.select(svg``) // .attr('transform', `translate(0, ${marginTop + height})`) // .call(d3.axisBottom(scales.x)) // .node(); // const yAxis = d3.select(svg``) // .attr('transform', `translate(${marginLeft}, 0)`) // .call(d3.axisLeft(scales.y)) // .node(); // // ${memory.stats.map(([len, ret], i) => svg` // // // // `)} // return svg` // `; // }4e:[] 46:{"id":"1746ae8c4bf84902","slug":"a-random-walk-through-the-grid-world","title":"A Random Walk Through the Grid World","thumbnail":"d6fc42151a897276e7cf3397b006262e750ecad603cfdca2d0d8dc71d22472e8","access_level":"public","create_time":"$D2018-04-29T21:34:48.074Z","update_time":"$D2019-04-15T00:12:31.921Z","view_time":null,"first_public_time":"$D2018-04-29T23:24:22.851Z","has_unpublished_edits":null,"fork":{"id":"3fed94c043aee0a3","slug":"temporal-difference-learning-sarsa-0","title":"Temporal-Difference Learning: SARSA(0)","owner":{"id":"df82e3ae90c3ba68","login":"j-kan","name":"Jim Kan"}},"stars":4,"starred":false,"owner":{"type":"team","id":"df82e3ae90c3ba68","login":"j-kan","name":"Jim Kan","avatar_url":"https://avatars-next.observableusercontent.com/6cf6d5dc58f463a8174b46278a86f73fa681db6ba244bb56e46eb254d64af045","tier":"starter_2024","bio":"","following":false},"version":3161,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"07757bcd007742e1","login":"j-kan","name":"Jim Kan","avatar_url":"https://avatars-next.observableusercontent.com/6cf6d5dc58f463a8174b46278a86f73fa681db6ba244bb56e46eb254d64af045","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"07757bcd007742e1","login":"j-kan","name":"Jim Kan","avatar_url":"https://avatars-next.observableusercontent.com/6cf6d5dc58f463a8174b46278a86f73fa681db6ba244bb56e46eb254d64af045","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":7,"collection":{"id":"d2769ec0679b5a7e","slug":"reinforcement-learning","title":"Reinforcement Learning","owner":{"id":"df82e3ae90c3ba68","login":"j-kan","name":"Jim Kan"}},"type":"notebook","body":{"titling":"auto","title":"A Random Walk Through the Grid World","cells":[{"id":0,"value":"md`# A Random Walk Through the Grid World`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2867,"value":"md`This serves as a testbed for simple implementations of reinforcement learning algorithms -- primarily for my own edification as I make my way through [this](http://www0.cs.ucl.ac.uk/staff/d.silver/web/Teaching.html) and [this](http://incompleteideas.net/book/the-book-2nd.html), and then maybe [this](https://sites.google.com/view/deep-rl-bootcamp/home) (my notes from these can be found [here](https://beta.observablehq.com/@j-kan/reinforcement-learning-notes)).\n\nThis notebook in particular doesn't implement any interesting algorithms; it just sets up the basic infrastructure. More later.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2206,"value":"md`\nHere we have a series of __${maxEpisodes}__ random walks around a __${gridDimensions.x} x ${gridDimensions.y}__ grid, where walks are limited to a horizon of __${horizon}__ steps.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2549,"value":"viewof maxEpisodes = labeledSlider(\n '', 'number of walks')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2084,"value":"viewof horizon = labeledSlider(\n '',\n 'horizon (maximum number of steps per walk)');","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":763,"value":"viewof gridDimensions = {\n const defaultX = 15;\n const defaultY = 10\n const maxValue = Math.min(displayDimensions.x, displayDimensions.y)/minGranularity;\n \n const form = html`\n`;\n \n form.value = { x: form.x.valueAsNumber, y: form.y.valueAsNumber };\n form.x.oninput = () => form.value.x = form.x.valueAsNumber;\n form.y.oninput = () => form.value.y = form.y.valueAsNumber;\n // ${this ? this.value : 30}\n \n return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":69,"value":"gridWorldDisplay = displayGridWorld(displayDimensions, rewardGrid, displayedEpisodes, mutable memory.Q)\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":958,"value":"viewof numberOfEpisodesToDisplay = labeledSlider(\n '',\n 'number of walks to display')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1788,"value":"viewof displayUpdateInterval = labeledSlider(\n ``, \n 'pause between display updates (in ms)')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2182,"value":"viewof displayStepwise = inlineSpinner(\n `update display after every \n `,\n (val) => val === 'step')\n ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2213,"value":"md`\nThe display currently shows the most recent __${Math.min(numberOfEpisodesToDisplay, memory.stats.length)}__ of __${memory.stats.length}__ walks, updating after every new __${displayStepwise ? 'step' : 'walk'}__, with a __${displayUpdateInterval}__ millisecond pause in between.\n\nThe green circle represents the start state, and the red circle represents the end state, which if filled indicates a successful outcome. ${showPathCircles ? `The grey circles mark the steps along the path of the most recent walk.` : ''}\n\n${showQ ? \n `\n\nThe small arrows in each cell indicate the action-value (Q) for each state/action pair.\n` : ''}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2663,"value":"md`Here are some plots for tracking progress:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1907,"value":"md`Episode length (number of steps per walk) for the _n_-th walk:` ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1812,"value":"plotStats(displayDimensions, memory.stats, d => d.episodeLength)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1913,"value":"md`Episode total (discounted) return vs walk number:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1883,"value":"plotStats(displayDimensions, memory.stats, d => d.totalReturn)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2681,"value":"md`Click this to make the agent start walking again from scratch.` ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2601,"value":"viewof go = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2338,"value":"currentEpisode = {\n return go, \n (displayUpdateInterval > 0) \n ? slowly(() => genExperiment(env, currentAgent, displayStepwise), displayUpdateInterval)\n : genExperiment(env, currentAgent, displayStepwise);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2341,"value":"$48","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2356,"value":"md`# Agent`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2683,"value":"md`i.e. the thing that's doing the walking.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2345,"value":"currentAgent = agentRandom","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":989,"value":"md`### Random Walker`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2385,"value":"md`The simplest agent: choose the next action randomly, and don't try to learn or remember anything.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":982,"value":"randomWalkPolicy = (env, observation) => Array(env.actions.length).fill(1.0/env.actions.length)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2362,"value":"noLearning = (episode, isDone, memory) => memory","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2359,"value":"agentRandom = ({\n initialConditions: memoryInitialValues,\n policyFn: randomWalkPolicy,\n learnFn: noLearning,\n stepwise: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1799,"value":"md`## Memory\n\nAgents might need to remember things based on their experience; that's what this part is for.\n\nBasically, anything mutable goes in here. The specifics of what is remembered here will depend on what algorithm the agent is following. \n\nAt a minimum, we will keep some tracking statistics per episode to help with evaluating the agent's progress: the total episode length, and each episode's discounted return.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1380,"value":"mutable memory = {}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2000,"value":"function memoryInitialValues() {\n return {\n Q: {},\n stats: []\n };\n}\n ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1759,"value":"function eraseMemory() {\n mutable memory = memoryInitialValues();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1748,"value":"function recordStats(epi) {\n mutable memory.stats = mutable memory.stats || [];\n mutable memory.stats.push({\n episodeLength: epi.length, \n totalReturn: discountedReturn(epi)\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2199,"value":"memory.stats","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2388,"value":"md`### Return\n\nThe _return G_t_ attained in a given episode is the total discounted reward received from time step _t_ onward, where ${tex`\\gamma`} is a _discount factor_ which attenuates the effects of future rewards.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2390,"value":"tex.block`\n\\begin{aligned}\nG_t &= R_{t+1} + \\gamma R_{t+2} + \\gamma^2 R_{t+3} + ... \t\\\\\n &= \\sum_{k=0}^{\\infty} \\gamma^k R_{t+k+1}\n\\end{aligned}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1163,"value":"gamma = 0.9","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2382,"value":"function discountedReturn(walk, _gamma = gamma) {\n return walk.reduce((acc, step) => acc * _gamma + (step.reward || 0), 0);\n}\n\n// discountedReturns = walks.map((walk) => round(discountedReturn(walk, 0.9), 5))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":480,"value":"md`# Environment\n\nGeneric functions for characterizing and interacting with any environment, loosely based on the [environment API](https://gym.openai.com/docs/#environments) from [OpenAI Gym](https://gym.openai.com/).`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":607,"value":"/** reset the environment to its initial state **/\nreset = (env) => {\n const initState = env.initFn();\n \n return {\n state: initState, \n observation: env.observationFn(initState)\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":610,"value":"step = (env, state, action) => {\n const [nextState, reward] = env.transitionFn(state, action);\n const observation = env.observationFn(nextState);\n const done = env.isEndOfEpisodeFn(nextState);\n\n return {\n state: nextState,\n observation: observation, \n reward: reward, \n done: done\n };\t// TODO: info\t\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1445,"value":"chooseAction = (env, observation, policyFn) => {\n const actionProbabilities = policyFn(env, observation);\n return env.actions[chooseRandomly(actionProbabilities)];\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":644,"value":"$49","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1017,"value":"function* genEpisode(env, policy, limit) {\n let epi = [];\n for (const aStep of genEpisodeStep(env, policy, epi.length, limit)) {\n epi.push(aStep);\n yield epi;\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1718,"value":"function* slowGenEpisode(env, policyFn, limit=100, interval=1000) {\n yield* slowly(() => genEpisode(env, policyFn, limit), interval);\n \n // for (const aStep of genEpisode(env, policyFn, limit))\n // yield Promises.tick(interval, aStep);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":618,"value":"md`### The Grid World environment\n\nThis captures the dynamics of a specific environment instance -- in this case, the Grid World where the reward surface is determined by a 2D difference of Gaussians function (described in greater detail below). The DoG generates a surface where the positive peak values are surrounded by a shallower ring of negative values.\n\nEach episode terminates when the Manhattan distance between the current state and the Gaussian peak is at most __${gwMaxDistanceFromPeak}__, or when reaching the specified maximum episode length of __${horizon}__. There is an additional __${extraSuccessBonus}__ point reward for reaching the goal (i.e. the Gaussian peak).\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":472,"value":"env = makeGridWorld(rewardGrid, extraSuccessBonus, mu, gwMaxDistanceFromPeak)\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":557,"value":"function gwActionFns(rewardGrid) {\n \n const gridDimensions = {\n x: rewardGrid.length,\n y: rewardGrid[0].length\n };\n \n function dontMoveIfWall([xold, yold], [xnew, ynew]) {\n return Number.isNaN(rewardGrid[xnew][ynew]) ? [xold, yold] : [xnew, ynew];\n }\n \n return {\n N: ([x,y]) => dontMoveIfWall([x, y], [x, Math.min(gridDimensions.y-1, y+1)]),\n E: ([x,y]) => dontMoveIfWall([x, y], [Math.min(gridDimensions.x-1, x+1), y]),\n W: ([x,y]) => dontMoveIfWall([x, y], [Math.max(0, x-1), y]),\n S: ([x,y]) => dontMoveIfWall([x, y], [x, Math.max(0, y-1)]),\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":500,"value":"function gwTransition(rewardGrid, extraSuccessBonus, gwActionFns, gwSuccess) {\n // transitionFn(env.start, 'S')\n \n return ([x,y], action) => {\n const actionFn = gwActionFns[action];\n const nextState = actionFn([x,y]);\n const success = gwSuccess(nextState); \n const reward = rewardGrid[x][y] + (success ? extraSuccessBonus : 0);\n\n return [nextState, reward];\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3006,"value":"function gwSuccess(peak, maxDistanceFromPeak) {\n return ([x,y]) => {\n const dx = Math.abs(x-peak[0]);\n const dy = Math.abs(y-peak[1]);\n\n return dx + dy <= maxDistanceFromPeak;\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2877,"value":"gwMaxDistanceFromPeak = 0.5","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1968,"value":"extraSuccessBonus = 10\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3010,"value":"makeGridWorld = (rewardGrid, extraSuccessBonus, peak, maxDistanceFromPeak) => {\n const actionFns = gwActionFns(rewardGrid);\n const successFn = gwSuccess(peak, gwMaxDistanceFromPeak);\n \n return {\n initFn: () => [0,0], // [gridDimensions.x, gridDimensions.y].map((v) => Math.floor(v * Math.random())),\n actions: Object.keys(actionFns),\n transitionFn: gwTransition(rewardGrid, extraSuccessBonus, actionFns, successFn),\n observationFn: (x) => x,\n isEndOfEpisodeFn: successFn\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":469,"value":"md`### The reward surface`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":733,"value":"md`These parameters determine the shape of the 2D difference of Gaussians function that generates the reward surface. \n\nThis first set parameterizes the \"positive\" Gaussian:` ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2894,"value":"tex.block`\n\\begin{aligned}\n\n\\boldsymbol{\\mu}\n &= [ \\mu_x, \\mu_y ] \n &= [ ${mu} ]\n\\\\\n\n\\rho &= & ${round(sigma.p)} \\\\\n\n\\boldsymbol{\\Sigma_+} &= \n \\begin{bmatrix}\n \\sigma_x^2 & \\rho\\sigma_x\\sigma_y \\\\ \n \\rho\\sigma_x\\sigma_y & \\sigma_y^2\n \\end{bmatrix}\n&=\n \\begin{bmatrix}\n ${sigma[0][0]} & ${sigma[0][1]} \\\\ \n ${sigma[1][0]} & ${sigma[1][1]}\n \\end{bmatrix} \n\\\\\n\\end{aligned}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":802,"value":"viewof mu = {\n const form = html`\n`;\n \n form.value = [ form.mu_x.valueAsNumber, form.mu_y.valueAsNumber ].map(round);\n form.mu_x.oninput = () => form.value[0] = round(form.mu_x.valueAsNumber);\n form.mu_y.oninput = () => form.value[1] = round(form.mu_y.valueAsNumber);\n \n return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":129,"value":"$4a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":734,"value":"md`where ρ is the Pearson correlation coefficient between x and y.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":737,"value":"md`These next parameters characterize the negative Gaussian relative to the shape of the positive Gaussian:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2308,"value":"tex.block`\n\\begin{aligned}\n\n\\boldsymbol{\\Sigma_-} &= \n k \\boldsymbol{\\Sigma_+}\n&=\n ${sigmaRatio}\n \\begin{bmatrix}\n ${sigma[0][0]} & ${sigma[0][1]} \\\\ \n ${sigma[1][0]} & ${sigma[1][1]}\n \\end{bmatrix} \n\\\\\n\nDoG(x,y) &= &\n \\mathcal{N}(\\boldsymbol{\\mu}, \\boldsymbol{\\Sigma_+}) - \n ${magnitudeRatio} \n \\mathcal{N}(\\boldsymbol{\\mu}, \\boldsymbol{\\Sigma_-}) \\\\\n\n\\end{aligned}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2914,"value":"md`Ratio of sigmas between the negative and positive Gaussians:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":730,"value":"viewof sigmaRatio = html`\t\n \n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":744,"value":"md`Ratio of magnitude coefficients between the negative and positive Gaussians:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":739,"value":"viewof magnitudeRatio = html`\t\n \n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":182,"value":"mode = rewardSurfaceFn(mu)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":164,"value":"function differenceOfGaussians2d([x,y], { mu: mu, sigma: sigma, sigmaRatio: sigmaRatio, magnitudeRatio: magnitudeRatio }) {\n const positive = new MultivariateGaussian({mu: mu, sigma: sigma});\n const negative = new MultivariateGaussian({mu: mu, sigma: matrixScalarMult(sigma, sigmaRatio)});\n return positive.density([x,y]) - magnitudeRatio * negative.density([x,y]);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3001,"value":"rewardSurfaceFn = xy => differenceOfGaussians2d(xy, {\n mu: mu, \n sigma: sigma, \n sigmaRatio: sigmaRatio, \n magnitudeRatio: magnitudeRatio \n })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2969,"value":"md`This function returns true if the grid position contains a wall, i.e. somewhere the agent can't go.\n\nWe mark wall positions in the reward grid as \\`NaN\\` values.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2951,"value":"wallFn = ([x,y]) => {\n return (x === 5 && (y % 2 === 0 || y % 5 === 0))\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":159,"value":"rewardGrid = makeGrid(gridDimensions.x, gridDimensions.y, rewardSurfaceFn, wallFn)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1796,"value":"md`# Experiment`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2091,"value":"md`An experiment puts an agent in an environment and lets it run for a set number of episodes. This is the top-level entry point for this notebook.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1771,"value":"function* genExperiment(env, agent, stepwise=false) {\n \n let mem = agent.initialConditions(env); // (env); \n\n mutable memory = mem;\n\n for (let i = 0; i < maxEpisodes; ++i) {\n for (const episode of genEpisode(env, agent.policyFn, horizon)) {\n const latestStep = episode[episode.length-1];\n const isComplete = latestStep.done || episode.length >= horizon;\n\n mem = agent.learnFn(episode, isComplete, mem);\n mutable memory = mem;\n\n if (stepwise)\n yield [i, episode];\n \n if (isComplete) {\n recordStats(episode);\n if (!stepwise)\n yield [i, episode];\n }\n }\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":460,"value":"md`### Appendix\n\nMiscellaneous display setup and configuration of no scientific interest.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2979,"value":"$4b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":986,"value":"showPathCircles = true","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2675,"value":"showQ = false","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2613,"value":"episodeToPath = (episode) => \n episode.map((step) => step.current.observation)\n .concat([episode[episode.length-1].next.observation])\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":251,"value":"minGranularity = 12","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":64,"value":"displayDimensions = ({ \n x: 900, y: 600, \n margin: {\n left: 20, \n bottom: 20,\n \tright: 10,\n \ttop: 10 }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"function makeGrid(xrange, yrange, rewardFn, wallFn) {\n\treturn d3.range(xrange)\n \t\t .map(x => d3.range(yrange)\n \t\t .map(y => wallFn([x,y]) ? NaN : rewardFn([x,y])));\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3077,"value":"$4c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2521,"value":"$4d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2203,"value":"function* slowly(generatorFn, interval=1000) {\n for (const thing of generatorFn())\n yield Promises.tick(interval, thing);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1540,"value":"function normalize(vec) {\n const denom = d3.sum(vec);\n return vec.map((v) => v/denom);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1335,"value":"function chooseRandomly(pvec) {\n const r = Math.random();\n const cumvec = pvec.reduce((acc, p, i) => acc.concat(1+p).map((v) => v-p), []);\n \n return cumvec.findIndex((x) => r < x);\n}\n\n// Array(100).fill(0).map(() => chooseRandomly([0.1,0.1,0.1,0.7])).filter((x) => x === 3)\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1329,"value":"function argmax(vv) {\n return vv.reduce(([mx, mi], n, i) => (mx > n) ? [mx,mi] : [n,i], [Number.NEGATIVE_INFINITY, -1])[1];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":284,"value":"matrixSum = (mx) => [].concat(...mx).filter(x => !Number.isNaN(x)).reduce((acc,v) => acc + v, 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":512,"value":"matrixScalarMult = (mx, coeff) => mx.map((vv) => vv.map((v) => v * coeff))\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":532,"value":"matrixMin = mx => Math.min(...[].concat(...mx).filter(x => !Number.isNaN(x)))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1478,"value":"matrixMax = mx => Math.max(...[].concat(...mx).filter(x => !Number.isNaN(x)))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":258,"value":"function round(x, precision = 3) { \n const n = 10**precision;\n return Math.round(x*n)/n;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":99,"value":"MultivariateGaussian = require('https://bundle.run/multivariate-gaussian@3.3.1')\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3147,"value":"inlineSpinner = (selectHtml, transformFn) => {\n const form = html``;\n \n form[0].oninput = () => form.value = transformFn(form[0].value);\n form[0].oninput();\n\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3132,"value":"labeledSlider = (inputHtml, labelHtml) => {\n const form = html``;\n \n form[0].oninput = () => form.value = form[0].valueAsNumber;\n form[0].oninput();\n\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":3160,"value":"labeledCheckbox = (inputHtml) => {\n const form = html``;\n \n form[0].oninput = () => form.value = form[0].checked;\n form[0].oninput();\n\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"d3 = require(\"https://d3js.org/d3.v5.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W4e","stdlib":"1"}} 47:"$46:body"