2b:null 2c:["$","title",null,{"children":"Extracting Image Data | Observable | Observable"}] 31:T848,extractImageData = (image) => { const width = image.naturalWidth; // retrieves the image's width const height = image.naturalHeight; // retrieves the image's height const context = DOM.context2d(width, height, 1); // create a canvas context.drawImage(image, 0, 0); // draw the image const {data} = context.getImageData(0, 0, width, height); // extract pixel data. this will return an array with a length = width * height (pixels) * 4 (it's four because by default, it's providing the red, green, blue and alpha value for each pixel) // This is for the bins that we will use to group pixel colors together const range = d3.range(0, 1, 1 / nBins); // The Quantize Scale allows us to bin values. The domain is set to the [0, 255], the extent of values for RGBA (red, green, blue, alpha) const bins = ({ r: d3.scaleQuantize().domain([0, 255]).range(range), g: d3.scaleQuantize().domain([0, 255]).range(range), b: d3.scaleQuantize().domain([0, 255]).range(range), }) const pixelData = []; // create an empty array let r = 0, g = 0, b = 0, a = 0; for (let i = 0; i < data.length; i += 4) { // For every four datapoints = RGBA r = data[i + 0]; g = data[i + 1]; b = data[i + 2]; a = data[i + 3]; const rgba = `rgba(${r}, ${g}, ${b}, ${a})`; // Get RGBA value const hsl = d3.hsl(rgba); // Get HSL value const rBin = bins.r(r); // Bin R const gBin = bins.g(g); // Bin G const bBin = bins.b(b); // Bin B const rgbBinColor = d3.rgb(bins.r.invertExtent(rBin)[0],bins.g.invertExtent(gBin)[0], bins.b.invertExtent(bBin)[0]) // Create new color from the binned values pixelData.push({ // Push all the data r: r, g: g, b: b, a: a, h: hsl.h, s: hsl.s, l: hsl.l, rgba: rgba, hex: d3.color(rgba).hex(), hsl: hsl, rBin: rBin, gBin: gBin, bBin: bBin, rgbBinR: rgbBinColor.r, rgbBinG: rgbBinColor.g, rgbBinB: rgbBinColor.b, rgbBinColor: rgbBinColor.toString() }) } return { width: width, height: height, pixelData: pixelData } }32:Tcfb,{ const ObjectIDToYear = {}; const sortedData = picassoArtEnriched.filter(d => d.Classification == selectedClassification) .sort((a,b) => +a.parsedDate - +b.parsedDate); sortedData.forEach(artwork => { ObjectIDToYear[artwork.ObjectID] = artwork.parsedDate }) const width = window.innerWidth * .85; const height = 500; const margin = {top: 20, bottom: 0, left: 10, right: 10}; const div = html`

` const divSel = d3.select(div) .style('width', `${width + margin.left + margin.right}px`) .style('height', `${height + margin.top + margin.bottom}px`); const svg = divSel.append('svg') .style('position', 'absolute') .style('top', '0px') .style('left', '0px') .attr('width', width + margin.left + margin.right) .attr('height', height + margin.top + margin.bottom); const hoverImg = divSel.append('img') .style('position', 'absolute') .style('top', '50px') .style('left', '0px') .style('opacity', 0) .style('pointer-events', 'none') .style('border', '3px solid #fff') const g = svg.append('g') .attr('transform', `translate(${margin.left}, ${margin.top})`); const xScale = d3.scaleBand() .domain(sortedData.map(d => d.ObjectID)) .paddingInner(0) .range([0, width]); const xAxis = g.append('g') .attr('class', 'x axis') .call(d3.axisTop(xScale).tickFormat(d => ObjectIDToYear[d])) xAxis.selectAll('text') .filter((d,i) => i % 10 != 0) .style("opacity", 0); const artworks = g.selectAll('.artwork') .data(sortedData) .join('g') .attr('class', 'artwork') .attr('transform', d => `translate(${xScale(d.ObjectID)}, 0)`) artworks.each(function(thisArt) { const reformatImageData = {}; thisArt.imageData.filter((d,i) => i <= (nColors - 1)).forEach((d,i) => { reformatImageData[d3.color(d.color).hex()] = d.length; }) const series = d3.stack() .order(d3.stackOrderAscending) .keys(Object.keys(reformatImageData))([reformatImageData]); const yScale = d3.scaleLinear() .domain([0,d3.max(series, d => d3.max(d, d=> d[1]))]) .range([height, 0]); d3.select(this) .selectAll("rect") .data(series) .join("rect") .attr("x", 0) .attr("y", d => yScale(d[0][1])) .attr("height", d => yScale(d[0][0]) - yScale(d[0][1])) .attr("width", xScale.bandwidth()) .style('fill', d => d.key) .style('stroke', d => d.key) }).on('mouseenter', function(event, data) { xAxis.selectAll('text') .style('opacity', 0) .filter(d => d == data.ObjectID) .style("font-weight", '900') .style('opacity', 1) d3.select(this) .selectAll('rect') .style("stroke", d => d3.color(d.key).brighter()); hoverImg.attr("src", data.ThumbnailURL) .style('left', event.x > width / 2 ? `${event.x - 400}px` : `${event.x + xScale.bandwidth()}px`) .style('opacity', 1); }).on('mouseleave', function(event) { xAxis.selectAll('text') .style("font-weight", '500') .style('opacity', 1) .filter((d,i) => i % 10 != 0) .style("opacity", 0); d3.select(this) .selectAll('rect') .style("stroke", d => d.key); hoverImg.style('opacity', 0) }) return div }33:[{"id":"8b47656181116e0c0785033405bd438246d187ebb2b4596026aa4d08d4d263a779d1da32234ec6455c384d5934a785466d1b8a043de265aec038c765f256e412","href":"https://static.observableusercontent.com/files/8b47656181116e0c0785033405bd438246d187ebb2b4596026aa4d08d4d263a779d1da32234ec6455c384d5934a785466d1b8a043de265aec038c765f256e412","name":"leger.jpeg","type":"image/jpeg","lastModified":1649884420005.614,"size":55975},{"id":"a1a0ebcee0bd4c5bb2f2339dd72d898578269809c50927cd0f89bdf6fc3f6424fefe026886b17499e088d2d6cc72382fc67b5577176f37e4405aefcf259c3603","href":"https://static.observableusercontent.com/files/a1a0ebcee0bd4c5bb2f2339dd72d898578269809c50927cd0f89bdf6fc3f6424fefe026886b17499e088d2d6cc72382fc67b5577176f37e4405aefcf259c3603","name":"picassoArtEnriched.json","type":"application/json","lastModified":1650408023128.632,"size":2230207}] 2f:{"id":"a961bbd67525b896","slug":"extracting-image-data","title":"Extracting Image Data","thumbnail":"4dffac795e4dbab47ea44cdf598ff040853932f80f557643cdacfc48cee43140","access_level":"public","create_time":"$D2020-05-21T17:39:43.379Z","update_time":"$D2022-04-26T16:53:14.254Z","view_time":null,"first_public_time":"$D2022-04-26T19:48:08.112Z","fork":{"id":"bd3e1ef9cc484be5","slug":"image-average","title":"Image Average","owner":{"id":"8c2e47252fd4995c","login":"mbostock","name":"Mike Bostock"}},"stars":33,"starred":false,"owner":{"type":"team","id":"f35c755083683fe5","login":"observablehq","name":"Observable","avatar_url":"https://avatars-next.observableusercontent.com/424c6c1f7794371c9915a160519bdfb34cf34e50069fcd6def38a1a04d4e9702","tier":"enterprise_2024"},"version":920,"publish_version":920,"role":null,"authored":false,"authors":[{"type":"individual","id":"42520cea6c99d3c8","login":"pstuffa","name":"Paul Buffa","avatar_url":"https://avatars-next.observableusercontent.com/32c09bb97a2bb742225b949dcc4cf81a03dcd2b0caae97325970223a48a14b94","tier":null}],"members":[{"type":"team","id":"f35c755083683fe5","login":"observablehq","name":"Observable","avatar_url":"https://avatars-next.observableusercontent.com/424c6c1f7794371c9915a160519bdfb34cf34e50069fcd6def38a1a04d4e9702","tier":"enterprise_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":14,"collection":null,"type":"notebook","body":{"title":"Extracting Image Data","cells":[{"id":0,"value":"md`# Extracting Image Data\n\nThis notebook walks through how to extract pixel data from any image. We'll examine a few ways to look at image data and what it could be used for.\n\nHere's a beauty from Fernand Leger`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":16,"value":"viewof image = imageInput({\n crossOrigin: \"anonymous\",\n initialUrl: await FileAttachment(\"leger.jpeg\").url()\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":20,"value":"image","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":397,"value":"I have a function here for extracting the image data, and to make the rest of the walkthrough simple, all the values we'll want are already extracted. I've added comments to the code so you can see what each section does.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":348,"value":"$31","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":403,"value":"Now that we have the image data, we can redraw the image pixel by pixel. I'm using canvas (vs. SVG) because of how large the data is.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":35,"value":"imagePixelData = extractImageData(image)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":186,"value":"{\n const width = imagePixelData.width;\n const height = imagePixelData.height;\n const context = DOM.context2d(width, height, 1);\n\n const pixelData = imagePixelData.pixelData\n let row = 0\n let column = 0;\n let i = 0\n let size = 1;\n for (let i = 0; i < pixelData.length; i++) {\n const pixel = pixelData[i];\n context.beginPath();\n context.fillStyle = pixel.hex\n context.rect(column, row, size, size);\n context.fill(); \n row = Math.floor(i / width) \n column = i - (row * width)\n }\n\n return context.canvas \n \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":181,"value":"What can we do with this data? We could group the results by hex code and redraw the image with grouped colors.\n\nThis literally groups by hex code (e.g. #FFFFFF), then sorts by the most common hex codes, and merges the results back into a single array.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":192,"value":"groupedPixelData = d3.merge(d3.groups(imagePixelData.pixelData, d => d.hex).sort((a,b) => b[1].length - a[1].length).map(d => d[1]))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":211,"value":"{\n const width = imagePixelData.width;\n const height = imagePixelData.height;\n const context = DOM.context2d(width, height, 1);\n\n const pixelData = groupedPixelData\n let row = 0\n let column = 0;\n let i = 0\n let size = 1;\n for (let i = 0; i < pixelData.length; i++) {\n const pixel = pixelData[i];\n context.beginPath();\n context.fillStyle = pixel.hex\n context.rect(column, row, size, size);\n context.fill(); \n row = Math.floor(i / width) \n column = i - (row * width)\n }\n\n return context.canvas \n \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":223,"value":"Funky? Yes. Meaningful? Hmmm...\n\nWe could sort the groupings so that its easier to distinguish any trends. You can use this dropdown to try sorting it by r (red) values, g (green) values or b (blue) values.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":255,"value":"viewof rgbSort = Inputs.select([\"r\", \"g\", \"b\"], {label: \"Select a value to sort by\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":232,"value":"sortedGroupedPixelData = d3.merge(d3.groups(imagePixelData.pixelData, d => d.hex)\n .sort((a,b) => b[1][0][rgbSort]- a[1][0][rgbSort]).map(d => d[1]))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":237,"value":"{\n const width = imagePixelData.width;\n const height = imagePixelData.height;\n const context = DOM.context2d(width, height, 1);\n\n const pixelData = sortedGroupedPixelData\n let row = 0\n let column = 0;\n let i = 0\n let size = 1;\n for (let i = 0; i < pixelData.length; i++) {\n const pixel = pixelData[i];\n context.beginPath();\n context.fillStyle = pixel.hex\n context.rect(column, row, size, size);\n context.fill(); \n row = Math.floor(i / width) \n column = i - (row * width)\n }\n\n return context.canvas \n \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":246,"value":"I love how this looks like a sunset over an ocean. I'm surprised after sorting how much of the image is white(ish). From the original artwork, I thought this would be a larger percent.\n\nAnother way to quantify color (other than rgba) is HSL (hue, saturation and lightness). D3 has built in HSL functions, which makes this much easier. I find that lightness works quite well as a means to sort the color.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":318,"value":"viewof hslSort = Inputs.select([\"h\", \"s\", \"l\"], {label: \"Select a value to sort by\", value: \"l\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":326,"value":"sortedHSLPixelData = d3.merge(d3.groups(imagePixelData.pixelData, d => d.hex)\n .sort((a,b) => b[1][0][hslSort]- a[1][0][hslSort]).map(d => d[1]))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":292,"value":"{\n const width = imagePixelData.width;\n const height = imagePixelData.height;\n const context = DOM.context2d(width, height, 1);\n\n const pixelData = sortedHSLPixelData\n let row = 0\n let column = 0;\n let i = 0\n let size = 1;\n for (let i = 0; i < pixelData.length; i++) {\n const pixel = pixelData[i];\n context.beginPath();\n context.fillStyle = pixel.hsl\n context.rect(column, row, size, size);\n context.fill(); \n row = Math.floor(i / width) \n column = i - (row * width)\n }\n\n return context.canvas \n \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":329,"value":"There's still a good amount of noise here, so we could try using binning methods to group color even better. The idea here is to group colors that are very similar, so we're not splitting hairs looking at minute differences. We could bin the RGB values or the HCL values. Here I've done RGB, but its a similar process for HCL.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":320,"value":"viewof rgbBinSort = Inputs.select(['rgbBinR', \"rgbBinG\", 'rgbBinB'], {label: \"Select a value to sort by\", value: 'rgbBinG'})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":276,"value":"nBins = 10 // This can be slow","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":362,"value":"sortedRGBBinPixelData = d3.merge(d3.groups(imagePixelData.pixelData, d => d.rgbBinColor)\n .sort((a,b) => b[1][0][rgbBinSort] - a[1][0][rgbBinSort]).map(d => d[1]))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":342,"value":"{\n const width = imagePixelData.width;\n const height = imagePixelData.height;\n const context = DOM.context2d(width, height, 1);\n\n const pixelData = sortedRGBBinPixelData\n let row = 0\n let column = 0;\n let i = 0\n let size = 1;\n for (let i = 0; i < pixelData.length; i++) {\n const pixel = pixelData[i];\n context.beginPath();\n context.fillStyle = pixel.rgbBinColor\n context.rect(column, row, size, size);\n context.fill(); \n row = Math.floor(i / width) \n column = i - (row * width)\n }\n\n return context.canvas \n \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":419,"value":"The green that was very prevalent in the original image looks to be grouping quite well. Let's use this to try to find what the top 20 colors are.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":429,"value":"topColorData = d3.groups(imagePixelData.pixelData, d => d.rgbBinColor).sort((a,b) => b[1].length - a[1].length).map(d => {\n return {\n color: d[0],\n length: d[1].length\n }\n}).filter((d,i) => i < 20)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":425,"value":"Plot.plot({\n marks: [\n Plot.barX(topColorData, {y: \"color\", x: \"length\", fill: \"color\", sort: {y: 'x', reverse: true}})\n ],\n marginLeft: 150\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":464,"value":"Comparing this to our bar chart, it looks pretty accurate. I only wish that the dark blue section in the middle of the artwork were grouped better, but it does look, upon closer inspection, that there is a gradient in that dark blue.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":455,"value":"{\n const width = imagePixelData.width;\n const height = imagePixelData.height;\n const context = DOM.context2d(width, height, 1);\n\n const pixelData = imagePixelData.pixelData\n let row = 0\n let column = 0;\n let i = 0\n let size = 1;\n for (let i = 0; i < pixelData.length; i++) {\n const pixel = pixelData[i];\n context.beginPath();\n context.fillStyle = pixel.hex\n context.rect(column, row, size, size);\n context.fill(); \n row = Math.floor(i / width) \n column = i - (row * width)\n }\n\n return context.canvas \n \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":468,"value":"Ok, so we've found a method for extracting multiple colors from an image. Pretty neat! We can apply this analysis to a dataset of artworks to look at trends in an artist's color palette. For that, I'll use data from the [MoMA Collection](https://observablehq.com/@observablehq/the-moma-collection-data-exploration) to get all the [Picasso](https://en.wikipedia.org/wiki/Pablo_Picasso) artworks in their collection.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":516,"value":"Getting the image data can take a while and is a bit tricky, as you have to work around CORS policies. I've just added a file attachment here of the already processed data. If you're interested in seeing how it's done, check out this notebook I made on [ImageQuilts](https://observablehq.com/@pstuffa/imagequilts).","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":503,"value":"picassoArtEnriched = FileAttachment(\"picassoArtEnriched.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":736,"value":"viewof selectedClassification = Inputs.select(new Set(picassoArtEnriched.map(d => d.Classification)), {label: \"Select one\", value: \"Painting\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":821,"value":"viewof nColors = Inputs.range([1, 20], {label: \"Top N Colors\", step: 1, value: 10})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":903,"value":"Hover over a column to see the artwork. Picasso had several periods in his career. See if you can spot some of them in the data:\n* Early Work (pre 1901)\n* Blue Period (1901-1904)\n* Rose Period (1904-1906)\n* African Period (1906-1909)\n* Cubism (1908-1914)\n* Neoclassicism (1917-1925)\n* Surrealism (1925-1932)\n* Late Work (1932 - 1973)\n\nIf you want to read more about Picasso's periods, [check out this article](https://mymodernmet.com/pablo-picasso-periods/).","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":669,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":478,"value":"### Imported Modules","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":475,"value":"import { getArtworks } from 'fb26bbff9402c08f'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":7,"value":"import {imageInput} from \"@mbostock/file-input\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}} 30:"$2f:body"