2c:null 2d:["$","title",null,{"children":"Using Slit Squares with Vizdoom | Théo Jaunet | Observable"}] 32:Ta11, function brushed3() { const selection = d3.event.selection; const [[x0, y0], [x1, y1]] = selection; const context_main = mean_scan.getContext('2d'); const context = temp_can.getContext('2d') const getmax = object => { return Object.keys(object).filter(x => { return object[x] == Math.max.apply(null, Object.values(object)); }); } for(let w =0;w<67;w++) { context.beginPath() context.filter = "brightness(190%)"; context.drawImage(observations, 240 * w, 0, 240, 162, 0, 0, 274, 176); context.save() context.closePath() let p = context.getImageData(x0, y0, x1-x0, y1-y0).data; let rgb = {r:0,g:0,b:0} const pixels = p.length; for (let i = 0; i < pixels; i+=4) { rgb.r += p[i]; rgb.g += p[i + 1]; rgb.b += p[i + 2]; } rgb.r = Math.min(~~(rgb.r/pixels)*5.5,255); rgb.g = Math.min(~~(rgb.g/pixels)*5.5,255); rgb.b = Math.min(~~(rgb.b/pixels)*5.5,255) console.log(rgb) // console.log(b); context_main.beginPath(); context_main.moveTo(stripe_width*w, 0); context_main.lineTo(stripe_width*w, 60); context_main.lineWidth = stripe_width; console.log('rgb('+(rgb.r )+', '+(rgb.g)+', '+(rgb.b )+')') // set line color context_main.strokeStyle = 'rgb('+(rgb.r)+', '+(rgb.g)+', '+(rgb.b)+')'; context_main.stroke(); context_main.beginPath(); context_main.moveTo(stripe_width*w, 62); context_main.lineTo(stripe_width*w, 122); context_main.lineWidth = stripe_width; let cat = colors[closest_col(colors,rgb)] context_main.strokeStyle = 'rgb('+(cat.r)+', '+(cat.g)+', '+(cat.b)+')'; context_main.stroke(); context_main.beginPath(); context_main.moveTo(stripe_width*w, 62); context_main.lineTo(stripe_width*w, 122); context_main.lineWidth = stripe_width; context_main.strokeStyle = 'rgb('+(cat.r)+', '+(cat.g)+', '+(cat.b)+')'; context_main.stroke(); // let max_col = getmax(rgb) // console.log(max_col) // cat = {r:20,g:20,b:20} // cat[max_col] =200; // context_main.beginPath(); // context_main.moveTo(stripe_width*w, 124); // context_main.lineTo(stripe_width*w, 184); // context_main.lineWidth = stripe_width; // context_main.strokeStyle = 'rgb('+(cat.r)+', '+(cat.g)+', '+(cat.b)+')'; // context_main.stroke(); // context_main.save(); } } 33:[] 30:{"id":"f5aea472b6fe05ef","slug":"using-slit-squares-with-vizdoom","title":"Using Slit Squares with Vizdoom","thumbnail":"ccd7a213022c11ed8f782743d635c0d8389c09f87b8ff0805fa5fd1b7bf2b0fd","access_level":"public","create_time":"$D2019-12-18T14:16:57.179Z","update_time":"$D2022-03-02T09:01:11.824Z","view_time":null,"first_public_time":"$D2020-01-13T14:54:04.892Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"d44620d109ce89eb","login":"theo-jaunet","name":"Théo Jaunet","avatar_url":"https://avatars-next.observableusercontent.com/bfe312248d299ea4d71fe7e752b785c2251399fb64f5b6d4ea4a10a3e71fc525","tier":"starter_2024"},"version":1233,"publish_version":1233,"role":null,"authored":false,"authors":[{"type":"individual","id":"b99dc983809e178e","login":"theo-jaunet","name":"Théo Jaunet","avatar_url":"https://avatars-next.observableusercontent.com/bfe312248d299ea4d71fe7e752b785c2251399fb64f5b6d4ea4a10a3e71fc525","tier":null}],"members":[{"type":"individual","id":"b99dc983809e178e","login":"theo-jaunet","name":"Théo Jaunet","avatar_url":"https://avatars-next.observableusercontent.com/bfe312248d299ea4d71fe7e752b785c2251399fb64f5b6d4ea4a10a3e71fc525","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Using Slit Squares with Vizdoom","cells":[{"id":0,"value":"md`# Using Slit Squares with Vizdoom`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":386,"value":"md` Tang et al. explored the possibilities of slit tears applied to video analysis \\[[paper\\]](https://dl.acm.org/citation.cfm?id=1520516). Here, we will explore another approach: slit squares and see how it can help overviewing Deep Reinforcement Learning episodes.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":42,"value":"md`First, let's load the data. The data is a single image (16080px;135px) which concatenates 67 observations (images) from [Memory Reduction](https://theo-jaunet.github.io/MemoryReduction) in a single line. Memory reduction is a project in which we explore how we can change the memory of an AI. Here, the AI must collect 4 items in a specific order (green, red, gray, and blue). ` ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":28,"value":"observations = new Promise((resolve, reject) => {\n const image = new Image;\n image.style.imageRendering = \"pixelated\";\n image.crossOrigin = \"anonymous\";\n image.src = \"https://raw.githubusercontent.com/Theo-Jaunet/MemoryReduction/master/data/main/images/main.jpg\";\n image.onload = () => resolve(image);\n image.onerror = reject;\n})\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":54,"value":"md`This image contains 67 images of 240x135 pixels from one episode of the game.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":47,"value":"md`In order to verify them, let's make a function to extract and visualize one observation at the time. `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":68,"value":"function stepIm(con,timest) {\n if (observations) {\n con.filter = \"brightness(160%)\"; // Make it look brighter\n con.drawImage(observations, 240 * timest, 0, 240, 162, 0, 0, 274, 176);\n con.save()\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":50,"value":"observation1 = {\n const context = DOM.context2d(240,146)\n stepIm(context,timestep)\n return context.canvas\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":79,"value":"md`With the slider below we can select a timestep and thus see all observations. While using the replay button, we can see the that the AI collected all items, as it should.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":404,"value":"viewof timestep = Scrubber(Array.from({length: 67}, (_, i) => i), {loop: false, delay:90})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":125,"value":"md`### Let's Slit Scan this Episode`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":531,"value":"md` A slit scan consists in sampling a single 1px width column at each time step of a video (images in our case), and then concatenate them in a single stripe. Without any indication, this column is usually vertical and extracted from the middle of the image. `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":133,"value":"slit_scan = {\n \n const context = DOM.context2d(960,60);\n const stripe_width = 960 / 67;\n\n for (let i = 0; i < 67; i++) {\n context.filter = \"brightness(160%)\";\n context.drawImage(observations,240 *i +120, 0, 1, 146, stripe_width*i, 0, stripe_width, 60);\n }\n\n context.save();\n return context.canvas\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":218,"value":"md`Here the width of slices (columns) is computed to fit the screen, i.e. 14px per slice. But we can make it smaller to further compress the stripe (with the following slider).`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":576,"value":"viewof stripe_width = Scrubber(Array.from({length: 28}, (_, i) => i/2), {loop: false, delay:120})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":220,"value":"{\nconst context = slit_scan.getContext('2d');\n context.clearRect(0,0,960,60);\n for (let i = 0; i < 67; i++) {\n context.drawImage(observations,240 *i +120, 0, 1, 162, stripe_width*i, 0, stripe_width, 60);\n }\n\n context.save();\n return stripe_width\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":252,"value":"md` ## Slit Square`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":281,"value":"md`Despite having relevant information, the slit scan stripe may be hard to analyse. This is due to the fact that desired information (i.e. items) are only taking a small part of the stripe. As slit tears, our approach only uses a small portion (brushed rectangle) of the observation as a column. `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":297,"value":"slit_square= {\nconst context = DOM.context2d(960,60)\n context.filter = \"brightness(160%)\";\nreturn context.canvas\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":538,"value":"md`Sadly, canvas cannot properly handle brush events, hence we need to use a SVG in which we display observations.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":369,"value":" function brushed() {\n const selection = d3.event.selection;\n const [[x0, y0], [x1, y1]] = selection;\n const context = slit_square.getContext('2d');\n \n context.clearRect(0,0,960,60);\n \n for (let i = 0; i < 67; i++) {\n context.drawImage(observations,240 *i +x0, y0, x1-x0, y1-y0, stripe_width*i, 0, stripe_width, 60);\n }\n \n context.save();\n }\n ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":287,"value":"{ \n const node = DOM.svg(240, 135)\n const svg = d3.select(node)\n svg.attr('style', 'border: 1px solid gray')\n \n svg.append(\"svg:image\")\n .attr('x', 0)\n .attr('y', 0)\n .attr('width', 240)\n .attr('height', 146)\n .attr(\"xlink:href\", observation1.toDataURL());\n\n const brush = d3.brush()\n .on(\"brush end\", brushed);\n\n svg.append(\"g\")\n .attr(\"class\", \"brush\")\n .call(brush)\n .call(brush.move, [[88, 70], [174, 133]])\n .call(g => g.select(\".overlay\").style(\"cursor\", \"default\"));\n\n return svg.node();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":359,"value":"md`### How can this be useful?\n\nWell, if we select specific parts of the observation, we can have a compact timeline of points of interest. As an example with the following slit square we can clearly see when the AI picked up items.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":375,"value":"\nviewof count = {\n const button = html`