29:null 2a:["$","title",null,{"children":"RL: Policy Gradient Algorithm | Ioannis Tzolas | Observable"}] 2f:T47e,// Visualization of doors ( = treasures they hide) along with the probabilities of being chosen { let probability_data = [] probabilities.forEach((p, i) => probability_data.push({probability : p, index: `Door ${i + 1}`})) let treasure_data = [] treasuresBehindDoors.dataSync().forEach((t, i) => treasure_data.push({treasure : t, index: `Treasure ${i + 1}`})) console.log(treasure_data) const treasure_pane = vl.markBar() .data(treasure_data) .encode( vl.y().fieldQ("treasure"), vl.x().fieldO("index").axis({grid: false, title: null, ticks : false, labelAngle : 0, labelFontSize: 20, labelColor: "green"}), vl.color().value("gold") ) .width(500) .height(200) const probability_pane = vl.markBar({stroke: "gray"}) .data(probability_data) .encode( vl.y().fieldQ("probability").scale({domain: [0, 1]}), vl.x().fieldO("index").axis({grid: false, title: null, ticks : false, labelAngle : 0, labelFontSize: 20, labelColor: "green"}), vl.color().value("#C46200") ) .width(500) .height(200) return vl.vconcat([treasure_pane, probability_pane]).render() }30:[] 2d:{"id":"7f0a3de182ee82a3","slug":"rl-policy-gradient-algorithm","title":"RL: Policy Gradient Algorithm","thumbnail":"9aa92f93a55b12ab33930a6aea74062b777643b280b0b38547b71050646b1a91","access_level":"public","create_time":"$D2021-09-11T19:34:32.075Z","update_time":"$D2024-11-13T15:19:13.964Z","view_time":null,"first_public_time":"$D2022-05-02T20:11:29.654Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"d6c7a67e66ff4d09","login":"igtzolas","name":"Ioannis Tzolas","avatar_url":"https://avatars-next.observableusercontent.com/45dace6229dc8914709cf833fbe8e7ddc9f996925bf158b909e24ed7fbb2e5ba","tier":"starter_2024"},"version":778,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"f371e71cbe248195","login":"igtzolas","name":"Ioannis Tzolas","avatar_url":"https://avatars-next.observableusercontent.com/45dace6229dc8914709cf833fbe8e7ddc9f996925bf158b909e24ed7fbb2e5ba","tier":null}],"members":[{"type":"individual","id":"f371e71cbe248195","login":"igtzolas","name":"Ioannis Tzolas","avatar_url":"https://avatars-next.observableusercontent.com/45dace6229dc8914709cf833fbe8e7ddc9f996925bf158b909e24ed7fbb2e5ba","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"title":"RL: Policy Gradient Algorithm","cells":[{"id":0,"value":"# RL: Policy Gradient Algorithm","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":500,"value":"md`Every algorithm requires an accompanying narrative...
\nInspired by https://towardsdatascience.com/an-intuitive-explanation-of-policy-gradient-part-1-reinforce-aa4392cbfd3c`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"Imagine there are 3 doors.
\nBehind each door there is a treasure of different value.
\n
\nA wise man is sitting in front of the doors advising visitors which door to open.
\nVisitors have no prior tendency towards a specific door.
\n
\nVisitors reach the doors and ask the wise man which door to open to get the largest treasure.
\nOnce the visitors have opened the door based on the advice, they let the wise man know of the treasure found.
\n
\nHere comes the big question:
\nHow should the wise man update the proposals made based on the feedback/reward reported by visitors ?
\n
","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":579,"value":"md`The first idea of how the wise man thinks is the following:
\n
\nI provide the visitor with a suggestion to visit a particular door.
\nWhen the visitor informs me of the treasure found, i update my advice policy as follows :
\nIf the treasure is positive (i.e. something good is found) then i add to my opinion that the door visited is good.
\nif the treasure is negative, i add to my opinion that the door visited is bad.
`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":583,"value":"md`Here is how the advice of the wise man changes by following the above strategy -> initial policy`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":723,"value":"As can be seen, the original advice is not updated properly. We are missing something...\n\nWhat can we do ?\nImagine that we knew which door was hiding the largest treasure and we told the wise man.\nThen the wise man could do the following : Whenever the action to follow the door was chosen, the parameters governing the wise man's behavior would change by a big amount towards favoring this action (in the code we multiply the reward as if it was not only 90 but 90 * 10000). Does this have an effect ? ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":751,"value":"Here is how the advice of the wise man changes by following the above strategy -> tweaked policy","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":759,"value":"As we can see the tweaked policy has the desired effect. But we have cheated in the sense that we cannot a priori\nknow which is the best action to push by a multiplicative factor.\n\nWhat can we do ?\nLet's think about it ! The problem is that the best action is not assigned more probability because of the fact that the initial policy is such that the best door is not chosen many times (we accounted for this in the tweaked policy by multiplying with a large number the change in the policy favoring the best action). We can have the same effect if we divide the reward (= treasure) behind each door by the probability of visiting this door to further boost the wise man towards updating his believes. Say hello to the very essence of the Policy Gradient Algorithm ! ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":766,"value":"Here is how the advice of the wise man changes by following the above strategy -> final policy","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":676,"value":"viewof ae = Inputs.radio([\"initial policy\", \"tweaked policy\", \"final policy\"], {label: \"Algorithm Evolution\", value: \"initial policy\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":610,"value":"viewof iterations = Inputs.range([1000, 5000], {label: \"Training iterations\", step: 100, value: 3000})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":635,"value":"// Initial Conditions \nmutable adviceStrength = tf.variable(tf.tensor1d([1., 1., 5.]))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":606,"value":"viewof control_training = Inputs.button([\n [\"Start\", value => iterations],\n [\"Continue\", value => iterations],\n [\"Stop / Reset\", value => { mutable adviceStrength = tf.variable(tf.tensor1d([1., 1., 5.])); return 0}]\n], {value: 0, label: \"Training Control\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":71,"value":"$2f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":69,"value":"probabilities = { \n \n if (control_training === 0)\n yield model(priorTendency).dataSync()\n \n for (let iteration = 0; iteration <= control_training ; iteration++){\n yield model(priorTendency).dataSync()\n optimizer.minimize(() => {\n \n let probabilities = model(priorTendency)\n let action = tf.multinomial(probabilities, 1, undefined, true).dataSync()[0]\n \n if (ae === \"initial policy\"){\n return tf.sum(probabilities.slice(action, 1).neg().mul(treasuresBehindDoors.slice(action, 1)))\n }\n else if (ae === \"tweaked policy\"){\n if (action === 0){\n return tf.sum(probabilities.slice(action, 1).neg().mul(tf.scalar(10000).mul(treasuresBehindDoors.slice(action, 1))))\n }\n else {\n return tf.sum(probabilities.slice(action, 1).neg().mul(treasuresBehindDoors.slice(action, 1)))\n }\n }\n else\n return tf.sum(tf.log(probabilities.slice(action, 1)).neg().mul(treasuresBehindDoors.slice(action, 1))) \n \n })\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":73,"value":"t = tf.tensor([[1., 1., 5.]]) // rank, shape, dtypedata","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":57,"value":"// Setting up the optimizer \noptimizer = tf.train.adam(0.005)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":35,"value":"// We will use the Core API to define the model as in : https://www.tensorflow.org/js/guide/models_and_layers\nmodel = function model(input){ \n return adviceStrength.softmax()\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":124,"value":"// adviceStrength = tf.variable(tf.tensor1d([1., 1., 5.])); // Initial values ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":113,"value":"priorTendency = tf.tensor1d([1., 1., 1.]) // There is no prior tendency to choose one door over the others","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":29,"value":"treasuresBehindDoors = tf.tensor([90.0, 40.0, 30.0])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"# Imports","pinned":true,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"tf = require(\"@tensorflow/tfjs\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":406,"value":"import {vl} from \"@vega/vega-lite-api\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W30","stdlib":"1","titling":"auto"}} 2e:"$2d:body"