2d:null 2e:["$","title",null,{"children":"Three Formats to Export Videos: webm, sprite, or base64 in CSV | Sylvain Lesage | Observable"}] 33:[] 31:{"id":"d01972454cf54b2d","slug":"load-process-and-export-video","title":"Three Formats to Export Videos: webm, sprite, or base64 in CSV","thumbnail":"ffdd4068e5572b7091102a6ef85815525435fad557ff5a1e6d6e5b7af3a8ae8e","access_level":"public","create_time":"$D2019-11-28T09:59:31.952Z","update_time":"$D2021-03-13T14:46:10.998Z","view_time":null,"first_public_time":"$D2019-11-29T17:51:56.858Z","has_unpublished_edits":null,"fork":{"id":"78c9e04c83a58610","slug":"naive-face-tracking-and-extraction","title":"Naive face tracking and extraction","owner":{"id":"5c7b4ebed88da11a","login":"severo","name":"Sylvain Lesage"}},"stars":5,"starred":false,"owner":{"type":"team","id":"5c7b4ebed88da11a","login":"severo","name":"Sylvain Lesage","avatar_url":"https://avatars-next.observableusercontent.com/7713195c83e5bfd8090ffe70c871c23e743c711e9e712675c99a8960ca8c7944","tier":"starter_2024"},"version":1003,"publish_version":1003,"role":null,"authored":false,"authors":[{"type":"individual","id":"bf1872cca880d345","login":"severo","name":"Sylvain Lesage","avatar_url":"https://avatars-next.observableusercontent.com/7713195c83e5bfd8090ffe70c871c23e743c711e9e712675c99a8960ca8c7944","tier":null}],"members":[{"type":"individual","id":"bf1872cca880d345","login":"severo","name":"Sylvain Lesage","avatar_url":"https://avatars-next.observableusercontent.com/7713195c83e5bfd8090ffe70c871c23e743c711e9e712675c99a8960ca8c7944","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":7,"collection":{"id":"80decc12bea1b8a1","slug":"video","title":"Video","owner":{"id":"5c7b4ebed88da11a","login":"severo","name":"Sylvain Lesage"}},"type":"notebook","body":{"title":"Three Formats to Export Videos: webm, sprite, or base64 in CSV","cells":[{"id":0,"value":"md`# Three Formats to Export Videos: webm, sprite, or base64 in CSV\n\nIn this tutorial, we load a video, loop over some of its frames, process the frames, and then propose three formats to export the frames: as a new webm video (Chromium-only), as a sprite image, or as base64 values in a CSV or JSON file.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":107,"value":"inputVideo = html`

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":452,"value":"md`There is currently (as far as I understand) no way to know which is the \"native\" framerate of a video. We will have to fix its value (current: ${printN(\n framerate\n)} fps, ie. one frame every ${printN(1000 / framerate)} ms)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":461,"value":"viewof framerate = slider({\n min: 0.1,\n max: 60,\n step: 0.1,\n value: 4,\n description: \"Frame rate, fps\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":988,"value":"info = vidvid.info(src, { crossOrigin })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":478,"value":"md`As the video duration is ${printN(\n info.duration\n)} s, we will manage ${frames} frames.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":510,"value":"frames = Math.floor(info.duration * framerate)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":504,"value":"md`## Example: show a video frame\n\nSelect the frame index (between 0 and ${frames - 1}):`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":515,"value":"viewof frameIdx = slider({\n min: 0,\n max: frames - 1,\n step: 1,\n value: 65,\n description: \"Current frame index\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":520,"value":"frameTime = frameIdx / framerate","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":383,"value":"md`The video is positionned to the current video frame (frame index ${frameIdx}, corresponding to time ${printN(\n frameTime\n)} s). The frame is then copied in an intermediate canvas that will serve as a basis for the video processing.\n\nNote: it doesn't work on Android Firefox v68.2.1. See https://bugzilla.mozilla.org/show_bug.cgi?id=1481406#c4`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":539,"value":"w = info.videoWidth","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":40,"value":"orig = frameAt(src, frameTime, { crossOrigin })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":541,"value":"h = info.videoHeight","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":386,"value":"md`## Process the frame\n\nAll the green pixels are then made transparent (\"chroma-keying effect\", see https://observablehq.com/@severo/manipulating-video-using-canvas-mdn-tutorial).\n\nThe center of mass of non-transparent pixels is marked with a red point.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":365,"value":"{\n const context = DOM.context2d(w, h, dpr);\n\n const frame = removeGreenFromCanvas(orig);\n context.putImageData(frame, 0, 0);\n\n // add center of mass\n const [x, y] = getCenterOfMass(frame);\n context.arc(x, y, 3, 0, 360);\n context.fillStyle = 'red';\n context.fill();\n\n // and extract zone\n context.strokeStyle = 'red';\n context.strokeRect(x - radius, y - radius, 2 * radius, 2 * radius);\n\n return context.canvas;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":404,"value":"md`Finally, the area marked by the red rectangle is extracted and placed in a new smaller canvas.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":188,"value":"extracted = {\n const frame = removeGreenFromCanvas(orig);\n const [x, y] = getCenterOfMass(frame).map(d => d - radius);\n const context = DOM.context2d(2 * radius, 2 * radius, dpr);\n context.putImageData(frame, -x, -y, x, y, 2 * radius, 2 * radius);\n return context.canvas;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":412,"value":"md`*Note: this technique is obviously awfully bad to extract faces, and only remotely kind of works because the video is centered on the face. See [pico.js](https://tehnokv.com/posts/picojs-intro/demo/) or [face-api.js](https://justadudewhohacks.github.io/face-api.js/webcam_face_tracking) for face detection.*`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":565,"value":"md`## Loop on frames to generate a new video\n\nWe loop over the ${frames} frames, extract them from the video, process them, and then format them for download. Three flavours: webm video (Chromium-only), PNG/JPG sprite, or data URI in JSON/CSV.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":856,"value":"md`### Format as a webm video (Chromium only)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":868,"value":"html`

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":860,"value":"DOM.download(\n webm_blob,\n \"face.webm\", // optional file name\n \"Click to Download\" // optional button label\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":864,"value":"webm_blob = {\n const videoWriter = new WebMWriter({\n quality: 0.95, // WebM image quality from 0.0 (worst) to 1.0 (best)\n frameRate: framerate // Number of frames per second\n });\n\n const interval = 1 / framerate;\n const numFrames = Math.floor(info.duration / interval) + 1;\n\n const w = 2 * radius;\n const h = 2 * radius;\n\n const frames = [];\n for (let i = 0; i < numFrames; i++) {\n const t = i * interval;\n const canvas = extractCenterOfMass(await frameAt(src, t, { crossOrigin }));\n await videoWriter.addFrame(canvas);\n }\n return await videoWriter.complete();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":800,"value":"md`### Format as a sprite`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":773,"value":"sprite = {\n const interval = 1 / framerate;\n const numFrames = Math.floor(info.duration / interval) + 1;\n\n const w = 2 * radius;\n const h = 2 * radius;\n const cols = Math.floor(Math.sqrt(numFrames));\n const rows = Math.ceil(numFrames / cols);\n const canvas = DOM.canvas(2 * radius * cols, 2 * radius * rows);\n canvas.style.maxWidth = `${width}px`;\n const context = canvas.getContext('2d');\n\n const currentFrame = undefined;\n for (let i = 0; i < numFrames; i++) {\n const t = i * interval;\n const x = (i % cols) * w;\n const y = Math.floor(i / cols) * h;\n context.drawImage(\n extractCenterOfMass(await frameAt(src, t, { crossOrigin })),\n x,\n y\n );\n context.fillStyle = 'white';\n context.fillText(`t=${printN(t)}`, x + 5, y + h - 5);\n }\n return canvas;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":876,"value":"DOM.download(\n await new Promise(resolve => sprite.toBlob(resolve)),\n \"face.png\", // optional file name\n \"Click to Download as PNG\" // optional button label\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":893,"value":"DOM.download(\n await new Promise(resolve => sprite.toBlob(resolve, 'image/jpeg', 1)),\n \"face.jpg\", // optional file name\n \"Click to Download as JPG\" // optional button label\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":970,"value":"md`*comment: see also https://movieprint.fakob.com/.*`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":815,"value":"md`### Format as base64 JPEG data URIs (and store in JSON)\n\nYou can also use the three dots on the left to [download the data](https://observablehq.com/@observablehq/introduction-to-data#downloads) as JSON or CSV.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":819,"value":"data = {\n const interval = 1 / framerate;\n const numFrames = Math.floor(info.duration / interval) + 1;\n\n const w = 2 * radius;\n const h = 2 * radius;\n\n const frames = [];\n for (let i = 0; i < numFrames; i++) {\n const t = i * interval;\n const canvas = extractCenterOfMass(await frameAt(src, t, { crossOrigin }));\n frames.push({\n i,\n x: canvas.x,\n y: canvas.y,\n t,\n w,\n h,\n image: canvas.toDataURL('image/jpeg', 1.0)\n });\n }\n return frames;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":898,"value":"DOM.download(\n new Blob([dsv.csvFormat(data)], { type: \"text/csv\" }),\n \"face.csv\", // optional file name\n \"Click to Download as CSV\" // optional button label\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":907,"value":"DOM.download(\n new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' }),\n \"face.json\", // optional file name\n \"Click to Download as JSON\" // optional button label\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":430,"value":"md`---\n\n## Credits\n\nDeveloped for the [LIRIS M2i project](https://projet.liris.cnrs.fr/mi2/) by Sylvain Lesage.\n\n---\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"src = \"https://raw.githubusercontent.com/mdn/dom-examples/master/canvas/chroma-keying/media/video.mp4\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":354,"value":"radius = 40","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":231,"value":"dpr = 1 // window.devicePixelRatio","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":687,"value":"extractCenterOfMass = canvas => {\n const im = removeGreenFromCanvas(canvas);\n // get extraction point\n const [x, y] = getCenterOfMass(im).map(d => d - radius);\n // generate output canvas\n const output = DOM.context2d(2 * radius, 2 * radius, dpr);\n output.putImageData(im, -x, -y, x, y, 2 * radius, 2 * radius);\n const c = output.canvas;\n c.x = x;\n c.y = y;\n return c;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"removeGreenFromCanvas = canvas => {\n const w = canvas.width;\n const h = canvas.height;\n\n let frame = canvas.getContext(\"2d\").getImageData(0, 0, w, h);\n\n // green pixels made transparent\n let l = frame.data.length / 4;\n for (let i = 0; i < l; i++) {\n let r = frame.data[i * 4 + 0];\n let g = frame.data[i * 4 + 1];\n let b = frame.data[i * 4 + 2];\n if (g > 100 && r > 100 && b < 43) frame.data[i * 4 + 3] = 0;\n }\n\n return frame;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":342,"value":"getCenterOfMass = frame => {\n let l = frame.data.length / 4;\n let mx = 0;\n let my = 0;\n let num = 0;\n for (let i = 0; i < l; i++) {\n // only count the non transparent pixels\n if (frame.data[i * 4 + 3] > 0) {\n mx += i % w;\n my += i / w;\n num += 1;\n }\n }\n const x = mx / num;\n const y = my / num;\n return [x, y];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":482,"value":"printN = n => {\n return n.toLocaleString('en-US', {\n minimumFractionDigits: 2,\n maximumFractionDigits: 2\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":973,"value":"import { frameAt } from \"@severo/frameat\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":981,"value":"vidvid = import('https://unpkg.com/vidvid-info?module')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":464,"value":"import { slider, button, select } from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":900,"value":"dsv = require('d3-dsv')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":692,"value":"WebMWriter = require('https://bundle.run/webm-writer@0.2.4')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":984,"value":"crossOrigin = 'anonymous'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":597,"value":"// isWebPSupported = document\n// .createElement('canvas')\n// .toDataURL(\"image/webp\")\n// .slice(5, 15) === 'image/webp'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}} 32:"$31:body"