45:["$","title",null,{"children":"Audio Sample and FFT Prep | Zach Pino | Observable"}] 4d:T52b,--- ### Configure FFT The Fast Fourier Transform is often called [the most important algorithm every written](https://www.youtube.com/watch?v=nmgFG7PUHfo). Take a look at this interactive explainer for an [intuitive understanding](https://www.jezzamon.com/fourier/). We often want to see the qualities of sounds that humans perceive — pitch and volume over time — rather than the super noisy amplitude wave that defines a physical soundwave. This is what FFT allows us to do... - Given a **time window** (for example, from second 3 to second 5) of *sampled amplitudes* like those generated above... - Construct a series of pure sine waves which, when combined, **harmonically approximate** the sampled wave. - The lowest frequency generated is 0 Hz (silence), and the highest frequency generated will be half the sample rate (for 48k Hz files, the highest pitch will be 24k Hz, which is right at the upper-end of human hearing). - Return how prevalent each set of nearby frequencies, called a *frequency bin*, is within the specific time window. - Optionally, convert the samples from [PCM instantaneous amplitude](https://en.wikipedia.org/wiki/Pulse-code_modulation), numbers between -1 to 1, and the [logarithmic decibel scale](https://en.wikipedia.org/wiki/Decibel) which better approximates human hearing. 4e:T5b4,--- ### Download Data At last, what we came for. To download the data below for use in other software like Rhino/Grasshopper.. - (**Recommended** for amplitude visualization) Right-click on the vertical ellipses to the left of `simplifiedSampleData` and click "Download CSV" to grab a simplified stream of PCM audio samples. - (**Recommended** for frequency visualization) Right-click on the vertical ellipses to the left of `simplifiedBins` and click "Download CSV" to grab the frequency bins of audio file in a two-dimensional array, averaged by the two sliders below. The first dimension is the *time windows*, and the second dimensions is the *frequency bins*. - (**Not recommended**, this will likely crash Grasshopper!) Right-click on the vertical ellipses to the left of `windowBins` and click "Download CSV" to grab the frequency bins of audio file in a two-dimensional array. The first dimension is the *time windows*, and the second dimensions is the *frequency bins*. - (**Not recommended**, this will likely crash Grasshopper!) Right-click on the vertical ellipses to the left of `totalDurationBins` and click "Download JSON" to grab the frequency bins of the entire audio file in a one-dimensional array. - (**Not recommended**, this will likely crash Grasshopper!) Right-click on the vertical ellipses to the left of `allSamples` and click "Download JSON" to grab the samples in decibels of the entire audio file in a one-dimensional array.4f:Tc7e,Waveform = { // For an explanation, see: https://observablehq.com/@skybrian/how-to-plot-long-audio-samples-at-any-zoom-level function findPeaks(audioData, offset, higher) { const a = Array(); let y = audioData[0]; let next = audioData[1]; if (higher(y, next)) { a.push({ t: offset, y }); } let prev; for (let i = 1; i < audioData.length - 1; i++) { prev = y; y = next; next = audioData[i + 1]; if (higher(y, prev) && higher(y, next)) { a.push({ t: offset + i, y }); } } if (higher(next, y)) { a.push({ t: offset + audioData.length - 1, y: next }); } return a; } function filterPeaks(peaks, nearby, higher) { let kept = null; let candidate = null; const a = Array(); for (let d of peaks) { if (!candidate) { candidate = d; if (!kept) { a.push(candidate); kept = candidate; candidate = null; } continue; } // invariant: kept and candidate are non-null and different points. if (d.t - kept.t > nearby) { // no more nearby candidates to check a.push(candidate); kept = candidate; candidate = d; } else if (higher(d.y, candidate.y)) { candidate = d; } } return a; } function chooseNearby(sampleLength, plotWidth) { return Math.ceil(0.5 * (sampleLength / plotWidth)); } function merge(a, b) { const out = Array(); let i = 0; let j = 0; while (i < a.length && j < b.length) { if (a[i].t < b[j].t) { out.push(a[i]); i++; } else { out.push(b[j]); j++; } } out.push(...a.slice(i)); out.push(...b.slice(j)); return out; } function enrich(points, audioData, start, end, nearby) { start = Math.max(0, start); const a = Array(); let kept = null; for (let t = start; t < points[0].t; t += nearby) { a.push({ t, y: audioData[t] }); } for (let d of points) { if (kept) { for (let t = kept.t + nearby; t < d.t; t += nearby) { a.push({ t, y: audioData[t] }); } } a.push(d); kept = d; } let t = end - 1; while (t - kept.t > nearby) { t -= nearby; } while (t < end) { a.push({ t, y: audioData[t] }); t += nearby; } return a; } class Waveform { constructor(audioData) { this.audioData = audioData; this.maxes = findPeaks(audioData, 0, (a, b) => a > b); this.mins = findPeaks(audioData, 0, (a, b) => a < b); } selectedData(start, length) { start = Math.floor(start); length = Math.floor(length); const nearby = chooseNearby(length, 0.5 * width); const end = start + length; const maxes = filterPeaks( this.maxes.filter((d) => d.t >= start && d.t < end), nearby, (a, b) => a > b ); const mins = filterPeaks( this.mins.filter((d) => d.t >= start && d.t < end), nearby, (a, b) => a < b ); return enrich(merge(maxes, mins), this.audioData, start, end, nearby); } } return Waveform; }50:[] 49:{"id":"54f7a388de82ccc4","slug":"audio-sample-and-fft-prep","title":"Audio Sample and FFT Prep","thumbnail":"8c5bee2320008df5960d0b1b1230792de0309499aade33ca3d0d77f3a67394c9","access_level":"public","create_time":"$D2022-11-13T17:43:39.160Z","update_time":"$D2024-01-06T00:33:44.227Z","view_time":null,"first_public_time":"$D2022-11-18T02:31:10.525Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"e13322fbaa35cc24","login":"zachpino","name":"Zach Pino","avatar_url":"https://avatars-next.observableusercontent.com/f70bf2a8b5c385b3d78c000b2635e8f37388d805fbc2986968496928347bc0e1","tier":"starter_2024","bio":"","following":false},"version":934,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"90c59c5f3ef39e0a","login":"zachpino","name":"Zach Pino","avatar_url":"https://avatars-next.observableusercontent.com/f70bf2a8b5c385b3d78c000b2635e8f37388d805fbc2986968496928347bc0e1","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"90c59c5f3ef39e0a","login":"zachpino","name":"Zach Pino","avatar_url":"https://avatars-next.observableusercontent.com/f70bf2a8b5c385b3d78c000b2635e8f37388d805fbc2986968496928347bc0e1","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Audio Sample and FFT Prep","cells":[{"id":0,"value":"# Audio Sample and FFT Prep\nAudio data is difficult to work with, and often inverted from how we tend to understand it intuitively. Check out this [Pudding interactive explainer](https://pudding.cool/2018/02/waveforms/) for more details and definitions of any unfamiliar terms below. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":507,"value":"-----\n### Upload an Audio File\nUpload a file to analyze, simplify, and visualize. The \"Choose File\" button below only loads the file into the local web browser, it does not upload data to any servers. Reloading the page will wipe everything.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":128,"value":"viewof audioFile = Inputs.file()\n//FileAttachment(\"09 - Daft Punk - Technologic 1.mp3\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":510,"value":"-----\n### Setup Javascript Audio Analysis\nBoring boilerplate code to ensure the browser can interact with the uploaded audio file, and we can access browser built-in methods for audio analysis.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":109,"value":"ctx = new window.AudioContext()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":461,"value":"buffer = await ctx.decodeAudioData(await audioFile.arrayBuffer())","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":512,"value":"-----\n### Extract Amplitude Samples\nGet each audio sample (usually 44.8K or 48K per second of audio - the *sample rate*) from the file and store in a buffer. Each PCM sample ranges from -1 to 1, reflecting the oscillating nature of the soundwave.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":257,"value":"audioData = buffer.getChannelData(0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":295,"value":"sampleCount = audioData.length","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":465,"value":"sampleRate = buffer.sampleRate","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":374,"value":"sampleExtent = d3.extent(audioData.map((d) => db.fromGain(d)))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":386,"value":"sampleScale = d3.scaleLinear().domain(sampleExtent)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":516,"value":"$4d","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":631,"value":"The *Time Sensitivity* slider below changes which dimension of sound that FFT prioritizes. \n\n- Smaller values shrink the duration of each window, so FFT is run more times (on a smaller amount of samples each time). But as a result, only a small number of frequency bins are able to be identified.\n- Larger values include more samples in each time window, so FFT is run fewer times overall (on a larger amount of samples each time). But as a result, a larger number of frequency bins are able to be identified.\n\nA good number for a short audio clip is probably 8-12.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":486,"value":"viewof sensitivity = Inputs.range([2, 16], {\n step: 1,\n value: 10,\n label: \"Time Sensitivity\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":643,"value":"----\n### FFT Calculations\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":289,"value":"windowSize = 2 ** sensitivity","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":259,"value":"windowCount = Math.floor(sampleCount / windowSize)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":440,"value":"durationSeconds = sampleCount / sampleRate","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":604,"value":"secondsPerWindow = durationSeconds / windowCount","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":459,"value":"maxFFTFrequency = sampleRate / 2","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":654,"value":"frequencyBins = windowSize / 2","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":337,"value":"windows = d3\n .range(windowCount)\n .map((i) => audioData.slice(i * windowSize, (i + 1) * windowSize))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":268,"value":"ffts = windows.map((d) => ft(d).map((e) => db.fromGain(e)))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":910,"value":"d3.max(ffts)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":645,"value":"----\n### Visualize Audio Samples and Frequency Bins\n\nThe visualization below plots time on the x-axis, frequency bins on the y-axis (the bottom of the graph is low frequencies and the top is high frequencies), and proportional volume with color.\n\nThere might be perfectly uniform, vertical 'stripes' towards the top of the visualization below, which is caused by consumer audio recording devices being unable to capture high frequencies of sound reliably.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":350,"value":"{\n const canvas = d3.create(\"canvas\").node();\n const ctx = canvas.getContext(\"2d\");\n canvas.width = ffts.length;\n canvas.height = ffts[0].length;\n\n ffts.map((fft, x) => {\n fft.map((sample, y) => {\n ctx.fillStyle = d3.interpolateMagma(sampleScale(sample));\n ctx.fillRect(x, windowSize / 2 - y, 1, 1);\n });\n });\n\n return canvas;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":679,"value":"We can also visualize a *massively* simplified version of the audio wave, after tracking which samples can be removed because they are part of an ongoing rising or falling wave, or represent sound frequencies faster than the resolution of the visualization.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":502,"value":"Plot.plot({\n width: width,\n height: 200,\n y: { domain: [-1, 1], axis: null },\n x: { axis: null },\n marginLeft: 0,\n marginRight: 0,\n marks: [\n Plot.areaY(new Waveform(audioData).selectedData(0, audioData.length), {\n x: \"t\",\n y: \"y\",\n stroke: waveColor\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":925,"value":"viewof waveColor = Inputs.color({\n label: \"Waveform Color\",\n value: d3.interpolateMagma(0.5)\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":684,"value":"$4e","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":544,"value":"simplifiedSampleData = new Waveform(audioData).selectedData(0, audioData.length)\n// .map((d) => d.t + \"|\" + d.y)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":792,"value":"viewof binsPerBandPower = Inputs.range([1, 8], {\n step: 1,\n value: 5,\n label: \"Bins per Frequency Band Power\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":845,"value":"viewof samplesPerSample = Inputs.range([1, 50], {\n step: 1,\n value: 24,\n label: \"Samples per Simplified Sample\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":780,"value":"simplifiedBins = ffts\n .map((fft) => {\n return fft\n .reduce((all, one, i) => {\n const ch = Math.floor(i / Math.pow(2, binsPerBandPower));\n all[ch] = [].concat(all[ch] || [], one);\n return all;\n }, [])\n .map((d) =>\n d.reduce((acc, val, i, arr) => {\n return acc + val / arr.length;\n }, 0)\n );\n })\n .reduce((all, one, i) => {\n const ch = Math.floor(i / samplesPerSample);\n all[ch] = [].concat(all[ch] || [], [one]);\n return all;\n }, [])\n .map((matrix) =>\n matrix[0]\n .map((col, i) => matrix.map((row) => row[i]))\n .map((d) =>\n d.reduce((acc, val, i, arr) => {\n return acc + val / arr.length;\n }, 0)\n )\n )","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":871,"value":"{\n const canvas = d3.create(\"canvas\").node();\n const ctx = canvas.getContext(\"2d\");\n canvas.width = simplifiedBins.length;\n canvas.height = simplifiedBins[0].length;\n\n const sScale = d3.scaleLinear().domain(simplifiedBins.flat(Infinity));\n\n simplifiedBins.map((fft, x) => {\n fft.map((sample, y) => {\n ctx.fillStyle = d3.interpolateMagma(1 - sScale(sample));\n ctx.fillRect(x, simplifiedBins[0].length - y, 1, 1);\n });\n });\n\n return canvas;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":869,"value":"-----","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":745,"value":"windowBins = ffts","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":704,"value":"totalDurationBins = ft(\n audioData.slice(\n 0,\n Math.pow(2, Math.floor(Math.log(sampleCount) / Math.log(2)))\n )\n).map((e) => db.fromGain(e))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":892,"value":"allSamples = audioData.map((d) => db.toGain(d))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":529,"value":"-----\n### Libraries\nUseful utilities and classes for audio processing and visualization, particularly indebted to [SkyBrian's waveform class](https://observablehq.com/@skybrian/working-with-audio-samples)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":265,"value":"ft = require(\"https://bundle.run/fourier-transform@1.1.2\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":497,"value":"$4f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":410,"value":"db = require(\"https://bundle.run/decibels@2.0.0\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W50","stdlib":"1"}} 44:null