24:null 25:["$","title",null,{"children":"Youtube Demonetized Words Similarity Study | Radamés Ajna | Observable"}] 2a:T5bd,md` Very recently I came across the conversation about [youtube demonetization words](https://www.google.com/search?q=Youtube+Demonetization+Words&source=lnms&tbm=nws). Some folks made a massive study on terms that might disable the monetization. If your video title contain any ${html`purple`} terms, then youtube might not enable ads for your video. The ${html`light gray`} terms seems to be passing the automated filter. Some **\*** in the terms means that the result is inconsistent. A more detailed exaplanation on how/what they found is in this [video](https://www.youtube.com/watch?v=oFyHpBsvcK0) and the spreasheet with the data [here](https://docs.google.com/spreadsheets/d/1ozg1Cnm6SdtM4M5rATkANAi07xAzYWaKL7HKxyvoHzk/edit#gid=674179785) While I know nothing more than what I've read on the articles, I've thought the dataset with ~16k terms could lead to an interesting visualization. Terms located close to one another are similar and further away dissimilar. The annotations labels are an example of clusters. Please explore more! * Mouseover / tap on points to see the term * Scroll to Zoom In/Out * Click and drag to pan around after zoom * Double click to reset It's a responsive but better view on the desktop. warning: there are many inappropriate words in this dataset `2b:T158c,function renderSVGGl() { const div = DOM.element("div"); div.style.position = "relative"; const svg = d3.select(DOM.svg(width, height)); const tooltip = d3 .select(div) .append("div") .attr("class", "tooltip") .style("display", "none"); svg.style("position", "relative"); svg.style("z-index", "1000"); svg.style("display", "block"); svg.style("width", "100%"); pixiApp.view.style.position = "absolute"; pixiApp.view.style.width = "100%"; pixiApp.view.style.height = "auto"; div.appendChild(pixiApp.view); div.appendChild(svg.node()); document.body.appendChild(div); const xScaleZoom = xScale.copy(); const yScaleZoom = yScale.copy(); svg .append("circle") .attr("class", "debug-dot") .attr("r", "5px") .attr("fill", "blue") .attr("opacity", "0"); svg .append("circle") .attr("class", "error-dot") .attr("r", "5px") .attr("fill", "red") .attr("opacity", "0"); const delaunay = d3.Delaunay.from( filterData.map((e) => [xScale(e.x) * maxZoom, yScale(e.y) * maxZoom]) ); const voronoi = delaunay.voronoi(extent.flat().map((e) => e * maxZoom)); let currScale = 1; const xAxis = (g) => g .attr( "transform", `translate(0, ${(margin.top + height - margin.bottom) / 2})` ) .call(d3.axisBottom(xScaleZoom)); const yAxis = (g) => g .attr( "transform", `translate(${(margin.left + width - margin.right) / 2}, 0)` ) .call(d3.axisLeft(yScaleZoom)); const redrawCtx = () => { pixiPoints.forEach((e, i) => { e.x = xScaleZoom(filterData[i].x); e.y = yScaleZoom(filterData[i].y); if (e.selected) { e.scale.x = 1.5 * dotScale(currScale); e.scale.y = 1.5 * dotScale(currScale); e.tint = filterData[i].Monetized ? 0xf2f0f7 : 0x54278f; e.alpha = 0.4; } else { e.scale.x = dotScale(currScale); e.scale.y = dotScale(currScale); e.tint = filterData[i].color; e.alpha = filterData[i].Monetized ? 0.6 : 1; } }); }; function zoomed(event, d) { currScale = event.transform.k; xScaleZoom.range( [margin.left, width - margin.right].map((d) => event.transform.applyX(d)) ); yScaleZoom.range( [margin.top, height - margin.bottom].map((d) => event.transform.applyY(d)) ); svg.selectAll(".x-axis").call(xAxis); svg.selectAll(".y-axis").call(yAxis); updateAnnotations(xScaleZoom, yScaleZoom, currScale); redrawCtx(); } const zoom = d3 .zoom() .scaleExtent([1, maxZoom]) .translateExtent([ [0, 0], [width, height] ]) .extent(extent) .on("zoom", zoomed); let tick = false; svg.on("mousemove", (event, d) => { if (!tick) { window.requestAnimationFrame(() => { mouseMoved(event); tick = false; }); tick = true; } }); function mouseMoved(event) { const [rmx, rmy] = [event.layerX, event.layerY]; const [mx, my] = [ xScale(xScaleZoom.invert(rmx)) * maxZoom, yScale(yScaleZoom.invert(rmy)) * maxZoom ]; const tooltip_width = parseInt(tooltip.style("width")); pointsContainer.children.forEach((d) => { d.selected = false; }); const voronoi_pt_idx = delaunay.find(mx, my); const points_container_last_idx = pointsContainer.children.length - 1; if (pointsContainer.children[points_container_last_idx].selected) { pointsContainer.children[points_container_last_idx].selected = false; // deselect last element tooltip.style("display", "none"); } if (voronoi_pt_idx !== null) { // if (voronoi_pt_idx !== null && // Math.sqrt((xScale(data[voronoi_pt_idx].x)*maxZoom - mx)**2 - // (yScale(data[voronoi_pt_idx].y)*maxZoom - my)**2) < maxZoom*width/500){ tooltip .style("display", "block") .attr("data-px", `${filterData[voronoi_pt_idx].x}`) .attr("data-py", filterData[voronoi_pt_idx].y) .style("left", `${xScaleZoom(filterData[voronoi_pt_idx].x)}px`) .style("top", `${yScaleZoom(filterData[voronoi_pt_idx].y)}px`) .html( `${filterData[voronoi_pt_idx].Word}${filterData[voronoi_pt_idx].Note}` ); d3.select(".debug-dot") .attr("cx", xScaleZoom(filterData[voronoi_pt_idx].x)) .attr("cy", yScaleZoom(filterData[voronoi_pt_idx].y)) .attr("opacity", "0.3"); const ind = pointsContainer.children.findIndex( (e) => e.orig_idx === voronoi_pt_idx ); pointsContainer.children[ind].selected = true; //select over element const temp = pointsContainer.children[points_container_last_idx]; pointsContainer.children[points_container_last_idx] = pointsContainer.children[ind]; pointsContainer.children[ind] = temp; redrawCtx(); } else { // svg.on('click', null) tooltip.style("display", "none"); } } svg .on("click", (e) => { const [rmx, rmy] = [d3.event.layerX, d3.event.layerY]; const [mx, my] = [xScale.invert(rmx), yScale.invert(rmy)]; }) .call(zoom) .call(zoom.scaleTo, 1) .on("dblclick.zoom", (e) => { svg.transition().duration(750).call(zoom.transform, d3.zoomIdentity); // svg.select('.annotation-group').style('visibility', 'visible'); }); svg .append("g") .attr("class", "annotation-group") .style("font-size", "1.2em") .call(makeAnnotations); return div; }2c:Ta54,makeAnnotations = { width; monete; const xs = d3 .scaleLinear() .range([margin.left, 954 - margin.right]) .domain(d3.extent(data, (d) => d.x)); const ys = d3 .scaleLinear() .range([margin.top, 536.625 - margin.bottom]) .domain([d3.max(data, (d) => d.y), d3.min(data, (d) => d.y)]); const annotations = [ { x: 615, y: 385.99999999999994, dx: -42, dy: 67, subject: { radius: 48.485281374238554 }, note: { title: "Profanity" } }, { x: 687, y: 156, dx: 57, dy: -52, subject: { radius: 50 }, note: { title: "Country Terms" } }, { x: 738, y: 334, dx: 62, dy: 52, subject: { radius: 50 }, note: { title: "Sex Terms", align: "left" } }, { x: 415, y: 92.00000000000006, dx: 24, dy: -29, subject: { radius: 20 }, note: { title: "Male Names" } }, { x: 270, y: 63.99999999999994, dx: -42, dy: 16, subject: { radius: 20 }, note: { title: "Female Names" } }, { x: 781.9999999999999, y: 245.00000000000003, dx: 32, dy: -34, subject: { radius: 20 }, note: { title: "Drugs" } }, { x: 871.7843307948311, y: 273, dx: -49, dy: 59, subject: { radius: 32.82842712474619 }, note: { title: "Diseases", align: "left" } }, { x: 804, y: 462, dx: -128, dy: 30, subject: { radius: 44.14213562373093 }, note: { title: "Insults", align: "left" } }, { x: 69, y: 310, dx: -50, dy: 66, subject: { radius: 20 }, note: { title: "Money Terms", align: "left" } }, { x: 265, y: 207, dx: -113, dy: -17, subject: { radius: 28.485281374238575 }, note: { title: "Gaming", align: "left" } }, { x: 190, y: 435.99999999999994, dx: 29, dy: 27, subject: { radius: 20 }, note: { title: "Time Terms" } }, { x: 97, y: 276, dx: -57, dy: -29, subject: { radius: 20 }, note: { title: "Politics", align: "left" } } ].map(function (d) { d.x = xsAnnotation.invert(d.x); d.y = ysAnnotation.invert(d.y); d.data = { x0: d.x, y0: d.y, dx0: d.dx, dy0: d.dy, r0: window.innerWidth > 500 ? d.subject.radius : d.subject.radius / 2 }; d.type = d3.annotationCalloutCircle; d.color = "#636363"; return d; }); return d3.annotation().type(d3.annotationLabel).annotations(annotations); }2d:T471,data = fetch( "https://gist.githubusercontent.com/radames/3471b27cbbb6f44fdc682a9743fb78b6/raw/7039d5ef6276838e2ebea378e37355eca183cf05/youtube-words-umap_embeds-50-hoods-cosine-10-repulsion-cluster.csv" ) .then((res) => res.text()) .then((text) => d3.csvParse(text)) // 'https://gist.githubusercontent.com/radames/9f66c27144032708c6c5814d1c94349d/raw/15792b2831bd4280d1b60cab2ddf195c661baab8/youtube-words-umap_embeds-50-hoods-cosine-10-repulsion.csv' // 'https://gist.githubusercontent.com/radames/cd6437459c36a771bf64e632623aeb58/raw/c7b80baeb4f9d8758df3257df3d95ede737324cb/youtube-words-umap_embeds-50-hoods-cosine.csv' .then((d) => { const color_scale = d3 .scaleOrdinal() .domain(d3.extent(d, (e) => e.cluster)) .range(d3.schemeAccent); return d.map((e, id) => Object.assign(e, { id: id, x: +e.x, y: +e.y, Note: +e.Note !== 0 ? e.Note : "", Monetized: +e.Monetized, color: +e.Monetized ? 0xf2f0f7 : 0x54278f, clustercolor: parseInt( color_scale(e.cluster).replace("#", "0x").concat("00") ) }) ); })2e:[] 28:{"id":"4d92c315348be99a","slug":"youtube-demonetized-words-similarity-study","title":"Youtube Demonetized Words Similarity Study","thumbnail":"0d40c6064c3ff46124bc48be3d6a09ad7dc892c7d5f2eb76ccbbb5b93b708575","access_level":"public","create_time":"$D2019-10-05T03:24:41.514Z","update_time":"$D2021-08-12T03:50:54.469Z","view_time":null,"first_public_time":"$D2019-10-07T18:32:48.061Z","fork":true,"stars":60,"starred":false,"owner":{"type":"team","id":"d6fb6dd3ddf05afb","login":"radames","name":"Radamés Ajna","avatar_url":"https://avatars-next.observableusercontent.com/7e61199ed2efe90aec8d897ac8b17a3fd0a129bff12a46a4048ddd0f73a2c43e","tier":"starter_2024"},"version":3739,"publish_version":3739,"role":null,"authored":false,"authors":[{"type":"individual","id":"06bc2aaa9d8c4a54","login":"radames","name":"Radamés Ajna","avatar_url":"https://avatars-next.observableusercontent.com/7e61199ed2efe90aec8d897ac8b17a3fd0a129bff12a46a4048ddd0f73a2c43e","tier":null}],"members":[{"type":"individual","id":"06bc2aaa9d8c4a54","login":"radames","name":"Radamés Ajna","avatar_url":"https://avatars-next.observableusercontent.com/7e61199ed2efe90aec8d897ac8b17a3fd0a129bff12a46a4048ddd0f73a2c43e","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"subscribed":null,"forks":34,"collection":{"id":"55e9900117038bfb","slug":"data-visualization","title":"Data Visualizations","owner":{"id":"d6fb6dd3ddf05afb","login":"radames","name":"Radamés Ajna"}},"type":"notebook","body":{"title":"Youtube Demonetized Words Similarity Study","cells":[{"id":0,"value":"md`# Youtube Demonetized Words Similarity Study`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2293,"value":"$2a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":506,"value":"chart = html`DemonetizedMonetized${renderSVGGl()}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2954,"value":"viewof monete = Inputs.toggle({ label: \"Filter Only Demonetized Words\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3321,"value":"md` ## Methods \n\n### Similarity analysis \n* Python + Tensorflow + [magical USE (Universal Sentence Encoder)](https://tfhub.dev/google/universal-sentence-encoder-large/3) + [UMAP](https://umap-learn.readthedocs.io/en/latest/) for dimenson reduction (I'll publish my notebook soon)\n* Annotations with labels were manually positioned\n\n\n### Data Visualization \n* d3.js + d3.delaunay + d3.annotation + Pixi.js (using webGL) \n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2027,"value":"md`## Pixi.js elements `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1664,"value":"pixiApp = {\n const app = new PIXI.Application({\n antialias: true, // default: false\n transparent: true, // default: false\n resolution: 2 // default: 1\n });\n invalidation.then(() => {\n app.destroy(true, true);\n });\n return app;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2688,"value":"function resizePixiCanvas(w, h) {\n pixiApp.renderer.resize(w, h);\n}\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2708,"value":"{\n width;\n resizePixiCanvas(width, height);\n return md`### pixijs canvas`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1671,"value":"pointsContainer = {\n const container = new PIXI.ParticleContainer(filterData.length, {\n scale: true,\n position: true,\n rotation: true,\n uvs: false,\n alpha: true\n });\n invalidation.then(() => container.destroy());\n return container;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1705,"value":"{\n while (pixiApp.stage.children[0]) {\n pixiApp.stage.removeChild(pixiApp.stage.children[0]);\n } // clean stage first\n pixiApp.stage.addChild(pointsContainer);\n return md`add points to stage`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1719,"value":"pixiPoints = {\n width;\n const circle = new PIXI.Graphics();\n circle.lineStyle(pointWidth / 5, 0x000000);\n circle.beginFill(0xffffff, 1);\n circle.drawCircle(0, 0, pointWidth);\n circle.endFill();\n\n const ptTexture = pixiApp.renderer.generateTexture(circle);\n ptTexture.mipmap = true;\n\n const points = filterData.map((e, id) => {\n const pt = PIXI.Sprite.from(ptTexture);\n pt.anchor.x = 0.5;\n pt.anchor.y = 0.5;\n pt.alpha = 0.5;\n pt.x = xScale(e.x);\n pt.y = yScale(e.y);\n pt.selected = false;\n pt.orig_idx = id;\n pt.money = e.Monetized;\n return pt;\n });\n\n invalidation.then(() => {\n circle.destroy();\n ptTexture.destroy();\n points.forEach((pt) => pt.destroy());\n });\n\n return points;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1781,"value":"//Add Points\naddPoints = {\n while (pointsContainer.children[0])\n pointsContainer.removeChild(pointsContainer.children[0]);\n pixiPoints.forEach(p => pointsContainer.addChild(p));\n return md`#### addPoints`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2035,"value":"md`### Constants`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1738,"value":"pointWidth = 20","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1743,"value":"maxZoom = 500","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1525,"value":"height = width\n// height = (width * 9) / 16","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1766,"value":"margin = ({ top: 50, right: 50, bottom: 50, left: 50 })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1764,"value":"extent = [\n [margin.left, margin.top],\n [width - margin.right, height - margin.top]\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2042,"value":"md`### d3 scales`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1975,"value":"dotScale = d3\n .scaleLinear()\n .range([0.1, 10])\n .domain([1, maxZoom])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1758,"value":"xScale = d3.scaleLinear()\n .range([margin.left, width - margin.right])\n .domain(d3.extent(data, d => d.x));","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1761,"value":"yScale = d3.scaleLinear()\n .range([margin.top, height - margin.bottom])\n .domain([d3.max(data, d => d.y),d3.min(data, d => d.y)]);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2059,"value":"md`## Render function`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":6,"value":"$2b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3210,"value":"md`## Annotations`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3143,"value":"xsAnnotation = d3\n .scaleLinear()\n .range([margin.left, 954 - margin.right]) // 954 - margin.right])\n .domain(d3.extent(data, d => d.x))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3142,"value":"ysAnnotation = d3\n .scaleLinear()\n .range([margin.top, 536.625 - margin.bottom]) //536.625 - margin.bottom])\n .domain([d3.max(data, d => d.y), d3.min(data, d => d.y)])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3055,"value":"$2c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3275,"value":"function updateAnnotations(xscale, yscale, currScale) {\n makeAnnotations.annotations().forEach((d, i) => {\n d.x = xscale(d.data.x0);\n d.y = yscale(d.data.y0);\n d.dx = currScale * d.data.dx0;\n d.dy = currScale * d.data.dy0;\n d.subject.radius = currScale * d.data.r0;\n });\n makeAnnotations.update();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2071,"value":"md`### Load Libraries `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3708,"value":"filterData = monete !== false ? data.filter((e) => e.Monetized === 0) : data\n// filterData = data.filter(e => e.Monetized === 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2868,"value":"$2d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2,"value":"d3 = require(\"d3@7\", \"d3-svg-annotation@2\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1539,"value":"PIXI = require(\"pixi.js@6/dist/browser/pixi.js\").catch(() => window.PIXI)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1261,"value":"CSS = html`