34:["$","title",null,{"children":"Possible Interfaces for Mixed-Initiative Reinforcement Learning for Learning from Humans | glassmanlab | Observable"}] 3c:Taf2,import * as tf from '@tensorflow/tfjs'; export class CartPole { /** * Constructor of CartPole. */ constructor() { // Constants that characterize the system. this.gravity = 9.8; this.massCart = 1.0; this.massPole = 0.1; this.totalMass = this.massCart + this.massPole; this.cartWidth = 0.2; this.cartHeight = 0.1; this.length = 0.5; this.poleMoment = this.massPole * this.length; this.forceMag = 10.0; this.tau = 0.02; // Seconds between state updates. // Threshold values, beyond which a simulation will be marked as failed. this.xThreshold = 2.4; this.thetaThreshold = 12 / 360 * 2 * Math.PI; this.setRandomState(); } /** * Set the state of the cart-pole system randomly. */ setRandomState() { // The control-theory state variables of the cart-pole system. // Cart position, meters. this.x = Math.random() - 0.5; // Cart velocity. this.xDot = (Math.random() - 0.5) * 1; // Pole angle, radians. this.theta = (Math.random() - 0.5) * 2 * (6 / 360 * 2 * Math.PI); // Pole angle velocity. this.thetaDot = (Math.random() - 0.5) * 0.5; } /** * Get current state as a tf.Tensor of shape [1, 4]. */ getStateTensor() { return tf.tensor2d([[this.x, this.xDot, this.theta, this.thetaDot]]); } /** * Update the cart-pole system using an action. * @param {number} action Only the sign of `action` matters. * A value > 0 leads to a rightward force of a fixed magnitude. * A value <= 0 leads to a leftward force of the same fixed magnitude. */ update(action) { const force = action > 0 ? this.forceMag : -this.forceMag; const cosTheta = Math.cos(this.theta); const sinTheta = Math.sin(this.theta); const temp = (force + this.poleMoment * this.thetaDot * this.thetaDot * sinTheta) / this.totalMass; const thetaAcc = (this.gravity * sinTheta - cosTheta * temp) / (this.length * (4 / 3 - this.massPole * cosTheta * cosTheta / this.totalMass)); const xAcc = temp - this.poleMoment * thetaAcc * cosTheta / this.totalMass; // Update the four state variables, using Euler's metohd. this.x += this.tau * this.xDot; this.xDot += this.tau * xAcc; this.theta += this.tau * this.thetaDot; this.thetaDot += this.tau * thetaAcc; return this.isDone(); } /** * Determine whether this simulation is done. * * A simulation is done when `x` (position of the cart) goes out of bound * or when `theta` (angle of the pole) goes out of bound. * * @returns {bool} Whether the simulation is done. */ isDone() { return this.x < -this.xThreshold || this.x > this.xThreshold || this.theta < -this.thetaThreshold || this.theta > this.thetaThreshold; } }3d:[] 38:{"id":"ca1a70339b10c3c8","slug":"possible-interfaces-for-mixed-initiative-reinforcement-l","title":"Possible Interfaces for Mixed-Initiative Reinforcement Learning for Learning from Humans","thumbnail":"ed924744c599998b0e5640eab989ab712c3431a5ec4ba30171440977ddb97251","access_level":"public","create_time":"$D2020-01-13T20:26:53.230Z","update_time":"$D2020-04-03T13:20:29.276Z","view_time":null,"first_public_time":"$D2020-01-13T20:42:26.732Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"f62b7bdd156efacc","login":"glassmanlab","name":"","avatar_url":null,"tier":"pro","bio":"","followed_by_you":false,"follows_you":false},"version":40,"publish_version":40,"role":null,"authored":false,"authors":[{"type":"individual","id":"66c773b7068139cc","login":"eglassman","name":"Elena Glassman","avatar_url":"https://avatars-next.observableusercontent.com/8505936ae4f7a9cb6b1993876dc61afa278fececf789fdaf7e40160161f6ba93","tier":null,"bio":"","followed_by_you":false,"follows_you":false}],"members":[{"type":"team","id":"f62b7bdd156efacc","login":"glassmanlab","name":"","avatar_url":null,"tier":"pro","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Possible Interfaces for Mixed-Initiative Reinforcement Learning for Learning from Humans","cells":[{"id":0,"value":"md`# Possible Interfaces for Mixed-Initiative Reinforcement Learning for Learning from Humans`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":37,"value":"tf = require('@tensorflow/tfjs@0.11')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":39,"value":"md`Note that this version of tfjs may be old and not be the one we want`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":26,"value":"md`## Cart-pole renderings`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":28,"value":"md`JS rendering of cart-pole for tensorflow example`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":31,"value":"md`

`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"md`## Cart-pole dynamics`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":24,"value":"md`Cartpole dynamics in Python`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":14,"value":"md`Cart-pole system simulator implemented usign tensorflow`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"md`Details about this simulator: \n\nIn the control-theory sense, there are four state variables in this system:\n\n - x: The 1D location of the cart.\n - xDot: The velocity of the cart.\n - theta: The angle of the pole (in radians). A value of 0 corresponds to\n a vertical position.\n - thetaDot: The angular velocity of the pole.\n\n The system is controlled through a single action:\n\n - leftward or rightward force.`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":20,"value":"$3c","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"md`## Other RL implementations for cart-pole`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"md`Tensorflow.js for the Cart-Pole`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W3d","stdlib":"1"}} 33:null