24:null 25:["$","title",null,{"children":"Youtube Demonetized Words Similarity Study | Nabil Kashyap | Observable"}] 2a:T5b7,md` Very recently I came across the conversation about [youtube demonetization words](https://www.google.com/search?q=Youtube+Demonetization+Words&source=lnms&tbm=nws). Some folks made a massive study on terms that might disable the monetization. If your video title contain any ${html`purple`} terms, then youtube might not enable ads for your video. The ${html`light gray`} terms seems to be passing the automated filter. Some **\*** in the terms means that the result is inconsistent. A more detailed exaplanation on how/what they found is in this [video](https://www.youtube.com/watch?v=oFyHpBsvcK0) and the spreasheet with the data [here](https://docs.google.com/spreadsheets/d/1ozg1Cnm6SdtM4M5rATkANAi07xAzYWaKL7HKxyvoHzk/edit#gid=674179785) While I know nothing more than what I've read on the articles, I've thought the dataset with ~16k terms could lead to same interesting visualization. All terms are organized using similiarity analysis. The annotations labels are just few examples of regions with similar terms. Please explore more. * Mouse over / tap on points to see term * Scroll to Zoom In/Out * Click and drag to pan around after zoom * Double click to reset It's responsive, but better view on desktop warning: there are many inappropriate words in this dataset `2b:T156d,function renderSVGGl() { const div = html`
`; const svg = d3.select(DOM.svg(width, height)); const tooltip = d3 .select(div) .append("div") .attr("class", "tooltip") .style("display", "none"); svg.style("position", "relative"); svg.style("z-index", "1000"); pixiApp.view.style.position = "absolute"; pixiApp.view.style.width = "100%"; pixiApp.view.style.height = "auto"; div.appendChild(pixiApp.view); div.appendChild(svg.node()); document.body.appendChild(div); const xScaleZoom = xScale.copy(); const yScaleZoom = yScale.copy(); svg .append('circle') .attr('class', 'debug-dot') .attr('r', '5px') .attr('fill', 'blue') .attr('opacity', '0'); svg .append('circle') .attr('class', 'error-dot') .attr('r', '5px') .attr('fill', 'red') .attr('opacity', '0'); const delaunay = new d3.Delaunay.from( filterData.map(e => [xScale(e.x) * maxZoom, yScale(e.y) * maxZoom]) ); const voronoi = delaunay.voronoi(extent.flat().map(e => e * maxZoom)); let currScale = 1; const xAxis = g => g .attr( "transform", `translate(0, ${(margin.top + height - margin.bottom) / 2})` ) .call(d3.axisBottom(xScaleZoom)); const yAxis = g => g .attr( "transform", `translate(${(margin.left + width - margin.right) / 2}, 0)` ) .call(d3.axisLeft(yScaleZoom)); const redrawCtx = () => { pixiPoints.forEach((e, i) => { e.x = xScaleZoom(filterData[i].x); e.y = yScaleZoom(filterData[i].y); if (e.selected) { e.scale.x = 1.5 * dotScale(currScale); e.scale.y = 1.5 * dotScale(currScale); e.tint = filterData[i].Monetized ? 0xf2f0f7 : 0x54278f; e.alpha = 0.4; } else { e.scale.x = dotScale(currScale); e.scale.y = dotScale(currScale); e.tint = filterData[i].color; e.alpha = filterData[i].Monetized ? 0.6 : 1; } }); }; function zoomed(e) { currScale = d3.event.transform.k; xScaleZoom.range( [margin.left, width - margin.right].map(d => d3.event.transform.applyX(d)) ); yScaleZoom.range( [margin.top, height - margin.bottom].map(d => d3.event.transform.applyY(d) ) ); svg.selectAll(".x-axis").call(xAxis); svg.selectAll(".y-axis").call(yAxis); updateAnnotations(xScaleZoom, yScaleZoom, currScale); redrawCtx(); } const zoom = d3 .zoom() .scaleExtent([1, maxZoom]) .translateExtent([[0, 0], [width, height]]) .extent(extent) .on("zoom", zoomed); let tick = false; svg.on('mousemove', evt => { const d3_event = d3.event; if (!tick) { window.requestAnimationFrame(() => { mouseMoved(d3_event); tick = false; }); tick = true; } }); function mouseMoved(d3_event) { const [rmx, rmy] = [d3_event.layerX, d3_event.layerY]; const [mx, my] = [ xScale(xScaleZoom.invert(rmx)) * maxZoom, yScale(yScaleZoom.invert(rmy)) * maxZoom ]; const tooltip_width = parseInt(tooltip.style('width')); pointsContainer.children.forEach(d => { d.selected = false; }); const voronoi_pt_idx = delaunay.find(mx, my); const points_container_last_idx = pointsContainer.children.length - 1; if (pointsContainer.children[points_container_last_idx].selected) { pointsContainer.children[points_container_last_idx].selected = false; // deselect last element tooltip.style("display", "none"); } if (voronoi_pt_idx !== null) { // if (voronoi_pt_idx !== null && // Math.sqrt((xScale(data[voronoi_pt_idx].x)*maxZoom - mx)**2 - // (yScale(data[voronoi_pt_idx].y)*maxZoom - my)**2) < maxZoom*width/500){ tooltip .style("display", "block") .attr('data-px', `${filterData[voronoi_pt_idx].x}`) .attr('data-py', filterData[voronoi_pt_idx].y) .style("left", `${xScaleZoom(filterData[voronoi_pt_idx].x)}px`) .style("top", `${yScaleZoom(filterData[voronoi_pt_idx].y)}px`) .html( `${filterData[voronoi_pt_idx].Word}${filterData[voronoi_pt_idx].Note}` ); d3.select('.debug-dot') .attr('cx', xScaleZoom(filterData[voronoi_pt_idx].x)) .attr('cy', yScaleZoom(filterData[voronoi_pt_idx].y)) .attr('opacity', '0.3'); const ind = pointsContainer.children.findIndex( e => e.orig_idx === voronoi_pt_idx ); pointsContainer.children[ind].selected = true; //select over element const temp = pointsContainer.children[points_container_last_idx]; pointsContainer.children[points_container_last_idx] = pointsContainer.children[ind]; pointsContainer.children[ind] = temp; redrawCtx(); } else { // svg.on('click', null) tooltip.style("display", "none"); } } svg .append("g") .attr("class", "annotation-group") .style('font-size', "1.2em") .call(makeAnnotations); svg .on('click', e => { const [rmx, rmy] = [d3.event.layerX, d3.event.layerY]; const [mx, my] = [xScale.invert(rmx), yScale.invert(rmy)]; }) .call(zoom) .call(zoom.scaleTo, 1) .on("dblclick.zoom", e => { svg .transition() .duration(750) .call(zoom.transform, d3.zoomIdentity); // svg.select('.annotation-group').style('visibility', 'visible'); }); return div; }2c:Ta62,makeAnnotations = { filterData; width; const xs = d3 .scaleLinear() .range([margin.left, 954 - margin.right]) .domain(d3.extent(data, d => d.x)); const ys = d3 .scaleLinear() .range([margin.top, 536.625 - margin.bottom]) .domain([d3.max(data, d => d.y), d3.min(data, d => d.y)]); const annotations = [ { x: 615, y: 385.99999999999994, dx: -42, dy: 67, subject: { radius: 48.485281374238554 }, note: { title: "Profanity" } }, { x: 687, y: 156, dx: 57, dy: -52, subject: { radius: 50 }, note: { title: "Country Terms" } }, { x: 738, y: 334, dx: 62, dy: 52, subject: { radius: 50 }, note: { title: "Sex Terms", align: "left" } }, { x: 415, y: 92.00000000000006, dx: 24, dy: -29, subject: { radius: 20 }, note: { title: "Male Names" } }, { x: 270, y: 63.99999999999994, dx: -42, dy: 16, subject: { radius: 20 }, note: { title: "Female Names" } }, { x: 781.9999999999999, y: 245.00000000000003, dx: 32, dy: -34, subject: { radius: 20 }, note: { title: "Drugs" } }, { x: 871.7843307948311, y: 273, dx: -49, dy: 59, subject: { radius: 32.82842712474619 }, note: { title: "Diseases", align: "left" } }, { x: 804, y: 462, dx: -128, dy: 30, subject: { radius: 44.14213562373093 }, note: { title: "Insults", align: "left" } }, { x: 69, y: 310, dx: -50, dy: 66, subject: { radius: 20 }, note: { title: "Money Terms", align: "left" } }, { x: 265, y: 207, dx: -113, dy: -17, subject: { radius: 28.485281374238575 }, note: { title: "Gaming", align: "left" } }, { x: 190, y: 435.99999999999994, dx: 29, dy: 27, subject: { radius: 20 }, note: { title: "Time Terms" } }, { x: 97, y: 276, dx: -57, dy: -29, subject: { radius: 20 }, note: { title: "Politics", align: "left" } } ].map(function(d) { d.x = xsAnnotation.invert(d.x); d.y = ysAnnotation.invert(d.y); d.data = { x0: d.x, y0: d.y, dx0: d.dx, dy0: d.dy, r0: window.innerWidth > 500 ? d.subject.radius : d.subject.radius / 2 }; d.type = d3.annotationCalloutCircle; d.color = "#636363"; return d; }); return d3 .annotation() .type(d3.annotationLabel) .annotations(annotations); }2d:T450,data = d3 .csv( 'https://gist.githubusercontent.com/radames/3471b27cbbb6f44fdc682a9743fb78b6/raw/7039d5ef6276838e2ebea378e37355eca183cf05/youtube-words-umap_embeds-50-hoods-cosine-10-repulsion-cluster.csv' // 'https://gist.githubusercontent.com/radames/9f66c27144032708c6c5814d1c94349d/raw/15792b2831bd4280d1b60cab2ddf195c661baab8/youtube-words-umap_embeds-50-hoods-cosine-10-repulsion.csv' // 'https://gist.githubusercontent.com/radames/cd6437459c36a771bf64e632623aeb58/raw/c7b80baeb4f9d8758df3257df3d95ede737324cb/youtube-words-umap_embeds-50-hoods-cosine.csv' ) .then(d => { const color_scale = d3 .scaleOrdinal() .domain(d3.extent(d, e => e.cluster)) .range(d3.schemeAccent); return d.map((e, id) => Object.assign(e, { id: id, x: +e.x, y: +e.y, Note: +e.Note !== 0 ? e.Note : '', Monetized: +e.Monetized, color: +e.Monetized ? 0xf2f0f7 : 0x54278f, clustercolor: parseInt( color_scale(e.cluster) .replace("#", "0x") .concat("00") ) }) ); })2e:[] 28:{"id":"c63e0d93d8f4a121","slug":"youtube-demonetized-words-similarity-study","title":"Youtube Demonetized Words Similarity Study","thumbnail":"b8529c58a13e6cbf3ab316a3dbb9635ffafd79ed7a5ee3937eed7045da6c1c88","access_level":"public","create_time":"$D2019-10-11T14:08:38.489Z","update_time":"$D2019-10-11T14:08:38.489Z","view_time":null,"first_public_time":"$D2019-10-11T14:08:52.638Z","fork":{"id":"4d92c315348be99a","slug":"youtube-demonetized-words-similarity-study","title":"Youtube Demonetized Words Similarity Study","owner":{"id":"d6fb6dd3ddf05afb","login":"radames","name":"Radamés Ajna"}},"stars":0,"starred":false,"owner":{"type":"team","id":"92032f850a4679ab","login":"bulbil","name":"Nabil Kashyap","avatar_url":"https://avatars-next.observableusercontent.com/d9e8d262f1076187594d379710a3a83a646393d004ed9faaffc35a85a0f4f9ce","tier":"starter_2024"},"version":3417,"publish_version":3417,"role":null,"authored":false,"authors":[{"type":"individual","id":"72aca3ddc200afbc","login":"bulbil","name":"Nabil Kashyap","avatar_url":"https://avatars-next.observableusercontent.com/d9e8d262f1076187594d379710a3a83a646393d004ed9faaffc35a85a0f4f9ce","tier":null}],"members":[{"type":"individual","id":"72aca3ddc200afbc","login":"bulbil","name":"Nabil Kashyap","avatar_url":"https://avatars-next.observableusercontent.com/d9e8d262f1076187594d379710a3a83a646393d004ed9faaffc35a85a0f4f9ce","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":0,"collection":{"id":"19933efcb70be790","slug":"lib-lab-2019","title":"Lib/Lab 2019","owner":{"id":"92032f850a4679ab","login":"bulbil","name":"Nabil Kashyap"}},"type":"notebook","body":{"title":"Youtube Demonetized Words Similarity Study","cells":[{"id":0,"value":"md`# Youtube Demonetized Words Similarity Study`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2293,"value":"$2a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":506,"value":"html`DemonetizedMonetized${renderSVGGl()}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2954,"value":"viewof monete = checkbox({\n options: [{ value: \"on\", label: \"Filter Only Demonetized Words\" }],\n value: false\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3321,"value":"md` ## Methods \n\n### Similarity analysis \n* Python + Tensorflow + [magical USE (Universal Sentence Encoder)](https://tfhub.dev/google/universal-sentence-encoder-large/3) + [UMAP](https://umap-learn.readthedocs.io/en/latest/) for dimenson reduction (I'll publish my notebook soon)\n* Annotations with labels were manually positioned\n\n\n### Data Visualization \n* d3.js + d3.delaunay + d3.annotation + Pixi.js (using webGL) \n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2027,"value":"md`## Pixi.js elements `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1664,"value":"pixiApp = {\n const app = new PIXI.Application({\n antialias: true, // default: false\n transparent: true, // default: false\n resolution: 2 // default: 1\n });\n invalidation.then(() => {\n app.destroy(true, true);\n });\n return app;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2688,"value":"function resizePixiCanvas(w, h){\n \tpixiApp.renderer.resize(w, h);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2708,"value":"{\n resizePixiCanvas(width, height);\n return md`### pixijs canvas`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1671,"value":"pointsContainer = new PIXI.ParticleContainer(filterData.length, {\n scale: true,\n position: true,\n rotation: true,\n uvs: false,\n alpha: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1708,"value":"circleGraphic = {\n const circle = new PIXI.Graphics();\n circle.lineStyle(pointWidth / 5, 0x000000);\n circle.beginFill(0xFFFFFF, 1);\n circle.drawCircle(0, 0, pointWidth);\n circle.endFill();\n\n return circle;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1705,"value":"{\n while (pixiApp.stage.children[0]) {\n pixiApp.stage.removeChild(pixiApp.stage.children[0]);\n } // clean stage first\n pixiApp.stage.addChild(pointsContainer);\n return md`add points to stage`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1710,"value":"ptTexture = {\n const txt = circleGraphic.generateCanvasTexture();\n txt.mipmap = true;\n return txt;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1719,"value":"pixiPoints = filterData.map((e, id) => {\n const pt = PIXI.Sprite.from(ptTexture);\n pt.anchor.x = 0.5;\n pt.anchor.y = 0.5;\n pt.alpha = 0.5;\n pt.x = xScale(e.x);\n pt.y = yScale(e.y);\n pt.selected = false;\n pt.orig_idx = id;\n pt.money = e.Monetized;\n return pt;\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1781,"value":"//Add Points\naddPoints = {\n while (pointsContainer.children[0])\n pointsContainer.removeChild(pointsContainer.children[0]);\n pixiPoints.forEach(p => pointsContainer.addChild(p));\n return md`#### addPoints`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2035,"value":"md`### Constants`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1738,"value":"pointWidth = 20","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1743,"value":"maxZoom = 500","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1525,"value":"height = width > 600 ? (width * 9) / 16 : width\n// height = (width * 9) / 16","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1766,"value":"margin = ({ top: 50, right: 50, bottom: 50, left: 50 })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1764,"value":"extent = [\n [margin.left, margin.top],\n [width - margin.right, height - margin.top]\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2042,"value":"md`### d3 scales`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1975,"value":"dotScale = d3\n .scaleLinear()\n .range([0.1, 10])\n .domain([1, maxZoom])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1758,"value":"xScale = d3.scaleLinear()\n .range([margin.left, width - margin.right])\n .domain(d3.extent(data, d => d.x));","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1761,"value":"yScale = d3.scaleLinear()\n .range([margin.top, height - margin.bottom])\n .domain([d3.max(data, d => d.y),d3.min(data, d => d.y)]);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2059,"value":"md`## Render function`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":6,"value":"$2b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3210,"value":"md`## Annotations`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3143,"value":"xsAnnotation = d3\n .scaleLinear()\n .range([margin.left, 954 - margin.right]) // 954 - margin.right])\n .domain(d3.extent(data, d => d.x))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3142,"value":"ysAnnotation = d3\n .scaleLinear()\n .range([margin.top, 536.625 - margin.bottom]) //536.625 - margin.bottom])\n .domain([d3.max(data, d => d.y), d3.min(data, d => d.y)])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3055,"value":"$2c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3275,"value":"function updateAnnotations(xscale, yscale, currScale) {\n makeAnnotations.annotations().forEach((d, i) => {\n d.x = xscale(d.data.x0);\n d.y = yscale(d.data.y0);\n d.dx = currScale * d.data.dx0;\n d.dy = currScale * d.data.dy0;\n d.subject.radius = currScale * d.data.r0;\n });\n makeAnnotations.update();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2071,"value":"md`### Load Libraries `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":3081,"value":"filterData = monete !== false ? data.filter(e => e.Monetized === 0) : data\n// filterData = data.filter(e => e.Monetized === 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2868,"value":"$2d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2,"value":"d3 = require(\"d3-delaunay@5\", \"d3@5\", \"d3-svg-annotation@2\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1539,"value":"// PIXI = require(\"https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.8.7/pixi.js\")\n\nPIXI = require('pixi.js-legacy@5.1.2/dist/pixi-legacy.min.js').catch(\n () => window.PIXI\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2728,"value":"import {checkbox} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1261,"value":"CSS = html`