3e:null 3f:["$","title",null,{"children":"PG Code Base | Farzad Abdolhosseini | Observable"}] 45:T417,viewof policyName = select({ onchange: (e) => { // if( typeof(runInfo.resetTrain) == 'function') // runInfo.resetTrain(); //e.stopPropagation(); // Object.values(policyDict).forEach(p => { // p.getTFParams.forEach(x => x.dispose()) // p.getTFConsts.forEach(x => x.dispose()) // }); // stopTrain(); }, options: [ {value: "gaussianFixed", label: 'Fixed Variance Gaussian'}, {value: "gaussian", label: 'Fully Parametrized Gaussian'}, {value: "hat", label: 'Triangular Distribution'}, {value: "uniform", label: 'Uniform Distribution'}, {value: "uniformFixed", label: 'Fixed Uniform Distribution'}, {value: "unhat", label: 'Inverted Triangular Distribution'}, {value: "unhatFixed", label: 'Fixed Inverted Triangular Distribution'}, {value: "twoGaussiansFixed", label: 'Two Fixed Gaussians'}, {value: "twoGaussiansFixedDist", label: 'Two Gaussians with Fixed Distance'}, {value: "twoGaussians", label: 'Two Gaussians'}, {value: "fake", label: 'Fake Policy'}, ] })46:T8f2,trainer = { const runEnv = (policy, reward, particles) => { let s = policy.sample(particles); let deviation = s.sub(policy.mean); // the amount of deviation from the mean // return [s, deviation, tf.tensor(s.dataSync().map(s => reward.js(s)))]; return [s, deviation, reward.tf(s)]; } var trainer = async(runInfo, callback) => { const optimizer = runInfo.optimizer(runInfo.learningRate); const policy = runInfo.policy; const reward = runInfo.reward; const base = tf.variable(tf.tensor1d([0])); const beta = tf.scalar(runInfo.baselineDecayRate); // how fast does the baseline change for (let i = 0; i < runInfo.nbOptSteps; i++) { var [states, rewards] = tf.tidy(() => { /// sample actions from the environment which is just a simple 1D function in our case /// ** note that the actions are taken w.r.t. current policy ** const [states, deviation, rewards] = runEnv(policy, runInfo.reward, runInfo.nbParticles); /// train the policy optimizer.minimize( () => runInfo.loss.loss(policy, reward.tf, states, rewards, deviation, base), false, /// returnCost? no, we're not using it policy.getTFParams /// varList? only the policy parameters and not the baseline param(s) ); if( runInfo.useBaseline ) { /// train the baseline base.assign(base.mul(beta.sub(1).neg()).add(rewards.mean().mul(beta))); } return [states, rewards]; }); /// train loss (if it has function approximators) if( runInfo.loss.fit ) { await runInfo.loss.fit(states, rewards); } /// a (sort of hacky) way to stop the training before it has finished (e.g. the stop button) if( runInfo.runId != currentRunId.value ) break; /// call the `callback` function if present: can be used to slow down training or step it manually if( typeof(callback) == 'function' ) { await callback(i, states, rewards, policy, base); } /// freeing-up memory //states.dispose(); //rewards.dispose(); } /// freeing-up memory base.dispose(); beta.dispose(); } return trainer }47:T22df,policyDict = { // const defaultMu = 5; const minSigma = 0.01; const gaussianPolicy = (fixedSigma) => { // Defining the parameters const mu = tf.variable(tf.tensor1d([defaultMu])); var _sigma, sigma; if( fixedSigma ) { _sigma = tf.scalar(defaultSigma); sigma = () => _sigma; } else { _sigma = tf.variable(tf.tensor1d([defaultSigma])); sigma = () => tf.softplus(_sigma).add(minSigma); } return { getTFParams: [mu].concat(fixedSigma? [] : [_sigma]), getTFConsts: (fixedSigma? [_sigma] : []), mean: mu, sample: nbParticles => tf.randomNormal([nbParticles]).mul(sigma()).add(mu), prob: (x, params) => stats.dnorm(x, params[0], params[1]), logProb: states => states.sub(mu).square().div(-2).div(sigma().square()), getParams: async() => [(await mu.data())[0], (await sigma().data())[0]], reset: (params) => { console.log('params', params); mu.assign(tf.tensor1d([params.defaultMu])); if( !fixedSigma ) _sigma.assign(tf.tensor1d([params.defaultSigma])); }, } }; /// Based on Triangular Distribution from: https://github.com/jstat/jstat/blob/master/src/distribution.js#L1000 const triDist = { sample: (a, b, c, nbParticles) => { let u = tf.randomUniform([nbParticles]); return tf.where(u.less(c.sub(a).div(b.sub(a))), u.mul(b.sub(a).mul(c.sub(a))).sqrt().add(a), u.sub(1).neg().mul(b.sub(a).mul(b.sub(c))).sqrt().neg().add(b) ); }, pdf: (x, a, b, c) => { if (x < a || x > b) { return 0; } else if (x < c) { return (2 * (x - a)) / ((b - a) * (c - a)); } else if (x === c) { return (2 / (b - a)); } else { // x > c return (2 * (b - x)) / ((b - a) * (b - c)); } }, logPdf: (states, a, b, c) => tf.where( // if s >= b --> 0 states.greaterEqual(b), states.mul(0), tf.where( // if c <= s < b --> (2 * (b - x)) / ((b - a) * (b - c)); states.greaterEqual(c), b.sub(states).mul(2).div(b.sub(a).mul(b.sub(c))), tf.where( // if a <= s < c --> (2 * (x - a)) / ((b - a) * (c - a)); states.greaterEqual(a), states.sub(a).mul(2).div(b.sub(a).mul(c.sub(a))), // else (s < mu-support / 2) states.mul(0) ))).log(), } const hatPolicy = () => { // Defining the parameters const mu = tf.variable(tf.tensor1d([defaultMu])); const support = tf.scalar(2); return { getTFParams: [mu], getTFConsts: [support], mean: mu, sample: nbParticles => tf.tidy(() => triDist.sample(mu.sub(support.div(2)), mu.add(support.div(2)), mu, nbParticles) ), prob: (x, params) => triDist.pdf(x, params[0]-params[1]/2, params[0]+params[1]/2, params[0]), logProb: states => tf.tidy(() => triDist.logPdf(states, mu.sub(support.div(2)), mu.add(support.div(2)), mu) ), getParams: async() => [(await mu.data())[0], (await support.data())[0]], reset: (params) => {mu.assign(tf.tensor1d([params.defaultMu]));}, } }; const unhatRampPolicy = (rampLength) => { // Defining the parameters const mu = tf.variable(tf.tensor1d([defaultMu])); const support = tf.scalar(2); const ramp = tf.scalar(rampLength); return { getTFParams: [mu], getTFConsts: [support, ramp], mean: mu, sample: nbParticles => tf.tidy(() => // if second half: μ + s // else: μ - s mu.add( triDist.sample(0, support.div(2), support.div(2).sub(rampLength), nbParticles) .mul(tf.where( tf.randomUniform([nbParticles]).less(0.5), tf.ones([nbParticles]).mul(-1), tf.ones([nbParticles]) )) ) ), prob: (x, p) => triDist.pdf(Math.abs(x-p[0]), 0, p[1]/2, p[1]/2-p[2]) / 2, logProb: states => triDist.logPdf( // if s >= μ => second half is like a triangular where x = s-μ // elif s < μ => first half is like a triangular where x = μ-s // therefore it's always like a triangular where x = abs(μ-s) states.sub(mu).abs(), 0, // a support.div(2), // b support.div(2).sub(ramp) // c ).sub(tf.log(2)), // P = (P1 or P2) / 2 --> log P = (log P1 or log P2) - log(2) // It shouldn't matter in the end, can get rid of it by using a baseline getParams: async() => [(await mu.data())[0], (await support.data())[0], (await ramp.data())[0]], reset: (params) => {mu.assign(tf.tensor1d([params.defaultMu]));}, } }; const uniformPolicy = (fixedSupport) => { // Defining the parameters const mu = tf.variable(tf.tensor1d([defaultMu])); let _support; if( fixedSupport ) { _support = tf.scalar(2); } else { _support = tf.variable(tf.tensor1d([2])); } let support = () => tf.softplus(_support).add(0.2); return { getTFParams: [mu].concat(fixedSupport ? [] : [_support]), getTFConsts: (fixedSupport ? [_support]: []), mean: mu, sample: nbParticles => tf.randomUniform([nbParticles]).sub(0.5).mul(support()).add(mu), prob: (x, params) => { if( x < params[0]-params[1]/2 || x > params[0]+params[1]/2 ) return 0; return 1 / params[1]; }, logProb: states => tf.onesLike(states).div(support()), getParams: async() => [(await mu.data())[0], (await support().data())[0]], reset: (params) => { mu.assign(tf.tensor1d([params.defaultMu])); if( !fixedSupport ) _support.assign(tf.tensor1d([2])); }, } }; const twoGaussians = (fixedSigma, fixedDistance) => { // Defining the parameters const mu = tf.variable(tf.tensor1d([defaultMu])); let _sigma, sigma, distance; if( fixedSigma ) { _sigma = tf.scalar(defaultSigma); sigma = () => _sigma; } else { _sigma = tf.variable(tf.tensor1d([defaultSigma])); sigma = () => tf.softplus(_sigma).add(minSigma); } if( fixedDistance ) { distance = tf.scalar(defaultDist); } else { distance = tf.variable(tf.tensor1d([defaultDist])); } return { getTFParams: [mu].concat(fixedSigma? [] : [_sigma]).concat(fixedDistance ? [] : [distance]), getTFConsts: (fixedSigma? [_sigma] : []).concat(fixedDistance ? [distance] : []), mean: mu, sample: nbParticles => { const u = tf.randomUniform([nbParticles]).less(tf.scalar(0.5)).mul(2).sub(1); return tf.randomNormal([nbParticles]).mul(sigma()).add(mu.add(u.mul(distance))); }, prob: (x, params) => (stats.dnorm(x, params[0]+params[2], params[1]) + stats.dnorm(x, params[0]-params[2], params[1])) / 2, logProb: states => tf.add(states.sub(mu.add(distance)).square().div(-2).div(sigma().square()).exp().div(2), states.sub(mu.sub(distance)).square().div(-2).div(sigma().square()).exp().div(2)).log(), getParams: async() => [(await mu.data())[0], (await sigma().data())[0], (await distance.data())[0]], reset: (params) => { mu.assign(tf.tensor1d([params.defaultMu])); if( ! fixedSigma ) _sigma.assign(tf.tensor1d([params.defaultSigma])); if( ! fixedDistance ) distance.assign(tf.tensor1d([params.defaultDist])); }, } }; const fakePolicy = () => { // Defining the parameters const mu = tf.variable(tf.tensor1d([defaultMu])); return { getTFParams: [mu], getTFConsts: [], mean: mu, sample: nbParticles => tf.ones([nbParticles]).mul(mu), prob: (x, params) => 0, logProb: states => states.mul(0), getParams: async() => (await mu.data()), reset: (params) => { mu.assign(tf.tensor1d([params.defaultMu])); }, } }; const policies = { gaussian: gaussianPolicy(false), gaussianFixed: gaussianPolicy(true), hat: hatPolicy(), uniform: uniformPolicy(false), uniformFixed: uniformPolicy(true), unhat: unhatRampPolicy(-1e-5), // adding 𝜖 to make sure it won't go into the infinite ramp region unhatFixed: unhatRampPolicy(0.1), twoGaussiansFixed: twoGaussians(true, true), twoGaussiansFixedDist: twoGaussians(false, true), twoGaussians: twoGaussians(false, false), fake: fakePolicy(), } yield policies; try { yield invalidation; } finally { console.log('disposed!!!'); Object.values(policies).forEach(p => { p.getTFParams.forEach(x => x.dispose()) p.getTFConsts.forEach(x => x.dispose()) }); } }48:Tb5c,lossDict = { /// surrogate loss for the policy gradient /// it is called "surrogate" since its derivative (dLoss/dTheta) is equal to the derivative of the "true" loss const surrogateLoss = (logProb, reward) => logProb.mul(reward).neg().mean(); // policy gradient loss const pgLoss = (policy, reward, states, rewards, deviation, baseline) => surrogateLoss(policy.logProb(states), rewards.sub(baseline)); // reward-gradient loss const rewardGradLoss = (policy, rewardFunc, states, rewards, deviation, baseline) => rewardFunc(deviation.add(policy.mean)).neg().mean(); // policy+reward gradient loss const rpLoss = (policy, rewardFunc, states, rewards, deviation, baseline) => surrogateLoss(policy.logProb(states), rewards.sub(baseline)).add(rewardFunc(deviation.add(policy.mean)).neg().mean()); const rewardGradFALoss = () => { const h = 16; let model = null; let replay_mem = null; let reset = () => { model = tf.sequential(); // Add a single hidden layer model.add(tf.layers.dense({inputShape: [1], units: h, useBias: true})); model.add(tf.layers.reLU()); model.add(tf.layers.dense({inputShape: [h], units: h, useBias: true})); model.add(tf.layers.reLU()); model.add(tf.layers.dense({inputShape: [h], units: h, useBias: true})); model.add(tf.layers.reLU()); model.add(tf.layers.dense({inputShape: [h], units: h, useBias: true})); model.add(tf.layers.reLU()); model.add(tf.layers.dense({inputShape: [h], units: 1, useBias: true})); model.compile({ optimizer: tf.train.adam(), loss: tf.losses.meanSquaredError, metrics: ['mse'], }); replay_mem = { states: tf.tensor([]), rewards: tf.tensor([]), } }; let loss = (policy, rewardFunc, states, rewards, deviation, baseline) => { // TODO: can use `rewardGradLoss` return model.predict(deviation.add(policy.mean)).neg().mean(); }; let fit = async(states, rewards) => { replay_mem.states = tf.concat([replay_mem.states, states]); replay_mem.rewards = tf.concat([replay_mem.rewards, rewards]); if( replay_mem.states.shape[0] > replayMemSize ) { let extra = replay_mem.states.shape[0] - replayMemSize; replay_mem.states = replay_mem.states.slice(extra, replayMemSize); replay_mem.rewards = replay_mem.rewards.slice(extra, replayMemSize); } await model.fit(replay_mem.states, replay_mem.rewards, { batchSize: 16, epochs: 1, shuffle: true, }); }; let predict = (states) => { return model.predict(states); }; reset(); return {loss, fit, predict, reset}; }; return { pg: () => ({loss: pgLoss}), rg: () => ({loss: rewardGradLoss}), rpg: () => ({loss: rpLoss}), rgfa: rewardGradFALoss, } }49:T679,rewardsDict = { let points = [0].concat(d3.range(20).map(x => 2*(Math.random()-0.5))).concat([0]); let rew = d3interp.interpolateBasis(points); let width = defaults.xrange[1] - defaults.xrange[0]; let start = defaults.xrange[0]; const binLen = 0.25; let br = [6.5, 7]; let brlen = br[1] - br[0]; return { /* Sinusoid reward */ sin: {tf: tf.sin, js: Math.sin, jsGrad: Math.cos}, /* Rough Sinusoid reward */ roughSin: {tf: (x) => tf.sin(x).add(tf.sin(x.mul(10)).div(8)), js: x => Math.sin(x) + Math.sin(x*10) / 8, jsGrad: x => Math.cos(x) + 10*Math.cos(x*10)/8}, /* Sinusoid with a flat region reward */ flatRegionSin: { tf: x => tf.sin( tf.where( x.greaterEqual(br[1]), x.sub(brlen), tf.where( x.greaterEqual(br[0]), tf.ones(x.shape).mul(br[0]), x ) ) ), js: x => Math.sin(x < br[0] ? x : (x > br[1] ? x - brlen : br[0])), jsGrad: x => (x < br[0] ? Math.cos(x) : (x > br[1] ? Math.cos(x-brlen) : 0))}, /* Step Sinusoid reward */ stepSin: {tf: x => tf.sin(tf.floor(x.div(binLen)).mul(binLen)), js: x => Math.sin(Math.floor(x / binLen) * binLen), jsGrad: x => 0}, /* Sinusoid reward plus 0.5 just to throw off PG without a baseline */ sinAdd: {tf: (x) => tf.sin(x).add(0.5), js: (x) => Math.sin(x) + 0.5, jsGrad: Math.cos}, // {tf: sinReward, js: x => rew((x-start) / width)} /* Step function reward */ step: {tf: x => tf.greaterEqual(x, 8).mul(0.45), js: x => x > 8 ? 0.45 : 0, jsGrad: x => 0}, //d3interp.interpolateBasis([1,2,10]) }; } 4a:T8a9,objectivePlot = (runInfo) => { const height = 100; const svg = d3.select(DOM.svg(width, height)); const objectiveColor = "#5E35B1"; const nObjectiveExtraPoints = 100; const disp = (defaults.xrange[1] - defaults.xrange[0]) / defaults.nPoints * nObjectiveExtraPoints; const objective = (d3utils.linspace(defaults.xrange[0], defaults.xrange[1], defaults.nPoints) .map(x => ({ x, px: calculateReward(runInfo.policy, [x, defaultSigma, defaultDist], runInfo.reward.js, false) }))); /// update function: this is called everytime we need to redraw the plot const update = () => { svg.selectAll("*").remove(); // removing everything .. not the fast way for sure //const yrange = [Math.min(-1, d3.min(objective, d => d.px)), Math.max(1, d3.max(objective, d => d.px))]; const yrange = d3.extent(objective, d => d.px); const x = d3.scaleLinear() .domain(defaults.xrange) .range([defaults.margin.left, width - defaults.margin.right]); const y = d3.scaleLinear() .domain(yrange) .range([height - defaults.margin.bottom, defaults.margin.top]); const area = d3.area() .x(d => x(d.x)) .y1(y(0)) .y0(d => y(d.px)); const meanIndex = Math.max(0, Math.min(objective.length-1, Math.round((runInfo.jsParams[0]-defaults.xrange[0]) / (defaults.xrange[1] - defaults.xrange[0]) * objective.length))); d3utils.drawCoordinates(svg, x, y, defaults.xrange, yrange, 'μ'); d3utils.drawLegend(svg, x, y, yrange, [{text: "𝐽 (objective)", color: objectiveColor}]); svg.append("circle") // .style("stroke", "gray") .style("fill", defaults.policyColor) .attr("r", 4) .attr("cx", x(runInfo.jsParams[0])) .attr("cy", y(objective[meanIndex].px)); svg.append("path") .datum(objective) .attr("stroke", objectiveColor) .attr("stroke-width", 1.5) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("fill", objectiveColor) .attr("fill-opacity", 0.2) .attr("d", area); }; /// end of the update function return {element: svg.node(), update}; }4b:T104b,rewardAtTimePlot = (runInfo) => { // just to make sure it gets updated at every step var margin = ({top: 5, right: 30, bottom: 5, left: 40}) var height = 150; const rewardColor = "#F57C00"; const baselineColor = "#1B5E20"; const empiricalColor = "#283593"; const xrange = [0, runInfo.nbOptSteps]; const svg = d3.select(DOM.svg(width, height)); const drawBaseline = (runInfo.lossName == 'pg' && runInfo.useBaseline); /// update function: this is called everytime we need to redraw the plot const update = () => { svg.selectAll("*").remove(); // removing everything .. not the fastest way for sure const tRew = runInfo.trueReward.map((x, i) => ({x: (i+1) * runInfo.skipFrames, px: x})); const bRew = runInfo.baselinePred.map((x, i) => ({x: (i+1) * runInfo.skipFrames, px: x})); const eRew = runInfo.rewardEmp.map((x, i) => ({x: (i+1) * runInfo.skipFrames, px: x})); const allValues = runInfo.trueReward.concat(runInfo.baselinePred).concat(runInfo.rewardEmp); const yrange = [ Math.min.apply(Math, [-1].concat(allValues)), Math.max.apply(Math, [1].concat(allValues))]; const x = d3.scaleLinear() .domain(xrange) .range([margin.left, width - margin.right]); const y = d3.scaleLinear() .domain(yrange) .range([height - margin.bottom, margin.top]); const area = d3.area() .x(d => x(d.x)) .y1(y(0)) .y0(d => y(d.px)); //const mx = stats.dnorm(mu, mu, sigma); //const policy = x => stats.dnorm(mu, x, sigma) svg.append("path") .datum(tRew) .attr("stroke", rewardColor) .attr("stroke-width", 1.5) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("fill", rewardColor) .attr("fill-opacity", 0.04) .attr("d", area); if( drawBaseline ) { svg.append("path") .datum(bRew) .attr("stroke", baselineColor) .attr("stroke-width", 1) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("fill", baselineColor) .attr("fill-opacity", 0.04) .attr("d", area); } svg.append("path") .datum(eRew) .attr("stroke", empiricalColor) .attr("stroke-width", 1) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("fill", empiricalColor) .attr("fill-opacity", 0.04) .attr("d", area); // svg.append("path") // .datum(pdfV) // .attr("stroke", policyColor) // .attr("stroke-width", 1.5) // .attr("stroke-linejoin", "round") // .attr("stroke-linecap", "round") // .attr("fill", policyColor) // .attr("fill-opacity", 0.2) // .attr("d", area.curve(policyCurve[policyName])); // svg.append('circle') // .attr('cx', x(0)) // .attr('cy', y(0)) // .attr('r', 5) // .attr('fill', 'steelblue'); const drawLineWithText = (svg, x, y, pos, text, func, color) => { // line indicating the mean μ svg.append("line") .attr("fill", "none") .attr("stroke", color) .attr("stroke-width", 1.5) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("x1", x(pos)) .attr("x2", x(pos)) .attr("y1", y(func(pos))-5) .attr("y2", y(0)) // svg.append("text") .html(text) .attr('font-size', 15) .attr("x", x(pos)-3*text.length) .attr("y", y(0)+29) } d3utils.drawCoordinates(svg, x, y, xrange, yrange, 'Step'); /// draw the legends let legends = [{text: "Empirical Reward Estimate", color: empiricalColor}, {text: "True Reward", color: rewardColor}]; if( drawBaseline ) legends.push({text: "Baseline Predication", color: baselineColor}); d3utils.drawLegend(svg, x, y, yrange, legends); // drawLineWithText(svg, x, y, runInfo.jsParams[0], '𝜇', pdf, policyColor); }; /// end of the update function return {element: svg.node(), update}; }4c:T1f7f,rewardPolicyPlot = (runInfo) => { let height = 300; const policyCurve = { gaussian: d3.curveNatural, gaussianFixed: d3.curveNatural, hat: d3.curveBasis, unhat: d3.curveBasis, unhatFixed: d3.curveBasis, uniform: d3.curveStep, uniformFixed: d3.curveStep, twoGaussiansFixed: d3.curveNatural, twoGaussiansFixedDist: d3.curveNatural, twoGaussians: d3.curveNatural, fake: d3.curveNatural, }; const svg = d3.select(DOM.svg(width, height)); const rewardColor = "steelblue"; const policyColor = defaults.policyColor; const approxRewColor = "green"; const increaseColor = "#FF6F00"; const decreaseColor = "#6A1B9A"; const constantColor = "#263238"; let new_point = 0.16; const diff = runInfo.jsParams[0] - Math.floor(runInfo.jsParams[0] * 40) / 40; // runInfo.jsParams[0] += diff; // const xrange = [0, 15]; --> defined elsewhere const pdf = x => runInfo.policy.prob(x, runInfo.jsParams); const nPoints = 1000; /// update function: this is called everytime we need to redraw the plot const update = () => { svg.selectAll("*").remove(); // removing everything .. not the fast way for sure const pdfV = d3utils.linspace(defaults.xrange[0], defaults.xrange[1], nPoints).map(x => ({x, px: pdf(x)})); // console.log(pdfV); const rew = d3utils.linspace(defaults.xrange[0], defaults.xrange[1], nPoints).map(x => ({x, px: runInfo.reward.js(x)})); let legend = [{text: "𝑟 (reward Function)", color: rewardColor}, {text: "𝜋 (policy)", color: policyColor}]; let points = []; if( runInfo.lossName == 'pg' ) { let pointsRews = runInfo.sampledPoints.map(x => x.r); let stdRews = Math.max(0.6, Math.sqrt(d3.variance(pointsRews))); let baseline = 0; if( runInfo.useBaseline && runInfo.baselinePred.length > 0 ) { baseline = runInfo.baselinePred[runInfo.baselinePred.length-1]; } points = runInfo.sampledPoints.map(x => ({x: x.s, px: pdf(x.s), r: (x.r-baseline) / stdRews})); } else { points = runInfo.sampledPoints.map(x => ({x: x.s, px: x.r, r: runInfo.reward.jsGrad(x.s)})); } const yrange = [Math.min(0, d3.min(rew, d => d.px)), Math.max(d3.max(pdfV, d => d.px), d3.max(rew, d => d.px))]; const x = d3.scaleLinear() .domain(d3.extent(pdfV, d => d.x)) .range([defaults.margin.left, width - defaults.margin.right]); const y = d3.scaleLinear() .domain(yrange) .range([height - defaults.margin.bottom, defaults.margin.top]); const area = d3.area() .x(d => x(d.x)) .y1(y(0)) .y0(d => y(d.px)); // console.log(pdfV, rew, trueReward); //const mx = stats.dnorm(mu, mu, sigma); //const policy = x => stats.dnorm(mu, x, sigma) svg.append("path") .datum(rew) .attr("stroke", rewardColor) .attr("stroke-width", 1.5) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("fill", rewardColor) .attr("fill-opacity", 0.2) .attr("d", area); svg.append("path") .datum(pdfV) .attr("stroke", policyColor) .attr("stroke-width", 1.5) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("fill", policyColor) .attr("fill-opacity", 0.2) .attr("d", area.curve(policyCurve[runInfo.policyName])); // console.log('points', points[0].r, runInfo.sampledPoints[0].r); // console.log(points[1].r, runInfo.sampledPoints[1].r, runInfo.sampledPoints[1].b); // console.log(points[2].r, runInfo.sampledPoints[2].r, runInfo.sampledPoints[2].b); d3utils.makeArrow(svg, increaseColor, 'arrow-inc'); d3utils.makeArrow(svg, decreaseColor, 'arrow-dec'); // d3utils.makeArrow(svg, constantColor, 'arrow-con'); if( runInfo.supervised ) { if( points.length > 0 ) { let eps = 1e-3; let slope = runInfo.reward.jsGrad(points[0].x) let len = (slope >= 0 ? 1 : -1) / 4; console.log(slope, len) svg.selectAll(".arrow") .data(points) .enter().append("line") // .attr("stroke", pChangeColor) .attr("stroke-width", 2.5) .attr("class", "arrow") .attr("marker-end", d => ("url(#arrow-" + (slope > eps ? 'inc' : (slope < -eps ? 'dec' : '')) + ")")) .attr("stroke", d => (slope > eps ? increaseColor : (slope < -eps ? decreaseColor : constantColor)) ) .attr("x1", d => x(d.x - len)) .attr("x2", d => x(d.x + len)) .attr("y1", d => y(d.px - len * slope)) .attr("y2", d => y(d.px + len * slope)); } } else if( showSamples ) { const vert = runInfo.lossName == 'pg'; // uses `increaseColor` for positive and `decreaseColor` for negative values // does not plot the constant/zero values svg.selectAll(".arrow") .data(points.filter(d => Math.abs(d.r) > 1e-4)) .enter().append("line") // .attr("stroke", pChangeColor) .attr("stroke-width", 1.3) .attr("class", "arrow") .attr("marker-end", d => ("url(#arrow-" + (d.r > 0 ? 'inc' : 'dec') + ")")) .attr("stroke", d => (d.r > 0 ? increaseColor : decreaseColor) ) .attr("x1", d => x(d.x)) .attr("x2", d => x(d.x+(1-vert)*d.r/8)) .attr("y1", d => y(d.px)) .attr("y2", d => y(d.px+vert*d.r/20)); // plots circles for constant/zero values svg.selectAll(".constarrow") .data(points.filter(d => Math.abs(d.r) <= 1e-4)) .enter().append("circle") .attr("class", "constarrow") .attr("fill", constantColor) .attr("cx", d => x(d.x)) .attr("cy", d => y(d.px)) .attr("r", 2); // .attr("r", 1.5) // .attr("cx", d => x(d.x)) // .attr("cy", d => y(d.px)); // .style("fill", function(d) { return color(cValue(d));}) } if( runInfo.lossName == 'rgfa' ) { legend.push({text: "Q (reward estimate)", color: approxRewColor}); let x = d3utils.linspace(defaults.xrange[0], defaults.xrange[1], nPoints); const approxRew = ( Array.from( runInfo.loss.predict(tf.tensor(x).reshape([-1, 1])) .dataSync() ).map( (r, i) => ({x: x[i], px: r}) ) ); svg.append("path") .datum(approxRew) .attr("stroke", approxRewColor) .attr("stroke-width", 1.5) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("fill", approxRewColor) .attr("fill-opacity", 0.2) .attr("d", area); } // svg.append('circle') // .attr('cx', x(0)) // .attr('cy', y(0)) // .attr('r', 5) // .attr('fill', 'steelblue'); const drawLineWithText = (svg, x, y, pos, text, func, color) => { // line indicating the mean μ let value = func(pos); svg.append("line") .attr("fill", "none") .attr("stroke", color) .attr("stroke-width", 1.5) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("x1", x(pos)) .attr("x2", x(pos)) .attr("y1", y(value) - 5 * Math.sign(value)) .attr("y2", y(0)) // svg.append("text") .html(text) .attr('font-size', 15) .attr("x", x(pos)-3*text.length) .attr("y", y(0)+29) } d3utils.drawCoordinates(svg, x, y, defaults.xrange, yrange); d3utils.drawLegend(svg, x, y, yrange, legend); let positionFunc = runInfo.supervised ? runInfo.reward.js: pdf; let paramChar = runInfo.supervised ? '𝜃' : '𝜇'; drawLineWithText(svg, x, y, runInfo.jsParams[0], paramChar, positionFunc, policyColor); // drawLineWithText(svg, x, y, mu-sigma, '𝜇-𝜎', policy, 'grey'); // drawLineWithText(svg, x, y, mu+sigma, '𝜇+𝜎', policy, 'grey'); }; /// end of the update function return {element: svg.node(), update}; }4d:T948,showTrainButtons = (runInfo, update) => { const stopTrain = async() => { runInfo.policy.reset(runInfo); runUtils.pauseRun(); if( runInfo.loss.reset ) runInfo.loss.reset(runInfo); } const resetTrain = async() => { stopTrain(); // sleep for 100ms await new Promise(resolve => setTimeout(resolve, 100)); runUtils.clearRun(runInfo); runInfo.jsParams = await runInfo.policy.getParams(); }; runInfo.resetTrain = resetTrain; const tButton = html`