2c:null 2d:["$","title",null,{"children":"Idea mining: the Mueller report | Diderot rebooted | Observable"}] 32:T40b, html`

This notebook lets you play with the notion that ideas are word sets, and see its application to textual analysis.

You'll use frequent item-set mining, a form of unsupervised learning more commonly seen in suggestion engines, to find words that appear often in the same document, in this case on the same page. The more often one word suggests another, the greater the association perceived. Read more..

By stripping the Mueller Report of common words and noticing coincidences in the remainder we can expose the associations at play. These are displayed below; darker nodes indicate greater frequency, and thicker connections indicate stronger associations.

You can explore the space by altering either the number of nodes, or the association strength required. Please select from the menus below, and enjoy!

Object.create(d)); const nodes = graph_data.nodes.map(d => Object.create(d)); const height = 500; const simulation = d3.forceSimulation(nodes) .force("link", d3.forceLink(links).id(d => d.id).distance(100)) .force("charge", d3.forceManyBody(100)) .force("center", d3.forceCenter(width / 2, height / 2)) .on("tick", ticked); //const svg = d3.select(DOM.svg(width, height)); const svg = d3.select(DOM.svg(1024, height)) .style('max-width', '100%') .style('height', 'auto'); const link = svg.append("g") .attr("stroke","gray") .attr("stroke-opacity", 0.95) //.attr("marker-end", "url(#arrow)") .selectAll("line") .data(links) .enter().append("line") .attr("stroke-width", d => 0.5+Math.sqrt(d.score)); const node = svg.append("g") .attr("stroke", "gray") .attr("stroke-width", 1.0) .selectAll("circle") .data(nodes) .enter().append("circle") .attr("r", 8) .attr("fill", d => nodecolour(d.frequency)) .call(drag(simulation)); node.on("mouseover", (d) => { link.style('stroke-opacity', function(l) { if (d === l.source) return 1; else return 0.25; }) }) .on("mouseout", (d) => { link.style("stroke-opacity", 0.2) }); svg.append("defs").append("marker") .attr("id", "arrow") .attr("viewBox", "0 -5 10 10") .attr("refX", 20) .attr("refY", 0) .attr("opacity", 0.2) .attr("markerWidth", 8) .attr("markerHeight", 8) .attr("orient", "auto") .append("svg:path") .attr("d", "M0,-5L10,0L0,5"); node.append("title") .text(d => d.id); const text = svg.append("g").attr("class", "labels").selectAll("g") .data(nodes) .enter().append("g"); text.append("text") .attr("x", 14) .attr("y", ".31em") .attr("opacity", 0.7) .attr("pointer-events", "none") .style("font-family", "sans-serif") .style("font-size", "0.7em") .text(function(d) { return d.id; }) function ticked() { link .attr("x1", d => d.source.x) .attr("y1", d => d.source.y) .attr("x2", d => d.target.x) .attr("y2", d => d.target.y); node .attr("cx", d => d.x) .attr("cy", d => d.y); text .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; }) } return svg.node(); }34:T89d,table_data ={ function netoff(fis1, fis2) { function subset(list1, list2) { var sub = true; list1.forEach(function (word) { if (!list2.includes(word)) { sub = false } }) return sub; } var nu = []; fis1.forEach(function (set) { var include = true; var iset = set.itemSet; fis2.forEach(function (set2) { var iset2 = set2.itemSet; if (subset(iset, iset2)) { include = false; } }) if (include) { nu.push(set) } }); return nu; }; const fis_sets = analysis_result.frequentItemSets; var list=[]; var fis_keys = Object.keys(fis_sets); var max_len = fis_keys.length - 1; var maximal_sets = []; fis_keys.forEach(function (fis_k, i) { var arity2 = i + 2; if (i < max_len ) { var maximals = netoff( fis_sets[fis_k], fis_sets[arity2.toString()]); if (i>1 && maximals.length > 0) { maximal_sets.push(maximals) } } }) var longest = fis_sets[fis_keys.length.toString()]; maximal_sets.push(longest); // maximal_sets.forEach(s => list.push({s:s.itemSet,sup:s.support})); //var no_singles = maximal_sets.shift(); var flattened = [].concat.apply([],maximal_sets); flattened.forEach(f =>{f.support=(f.support*1).toFixed(2)}) const map1 = flattened.map(x => x!== undefined ? JSON.stringify(x.itemSet)+ JSON.stringify(x.support): []); list.push(map1); return flattened}35:T51a, graph_data = {counter=1; var list=[]; var list2 =[]; var list3 =[]; const nodes2 ={}; var nodelist=[]; const linkset = {}; const links2 = []; var wordlist=[]; var counter=1; analysis_result.associationRules.forEach(rule => { links2.push(rule);//console.log(from, to, rule.confidence);nodes2[link.rhs[0]]+=1 }) ; links2.forEach(function (link) { var c = link.rhs[0]; if(c in nodes2){nodes2[c]+=1}else{nodes2[c]=1}; link.lhs.forEach(l => { if(l in nodes2){nodes2[l]+=1}else{nodes2[l]=1}; var todo = [l,c]; list.push(todo); }); }); list2=list.sort(); Object.keys(nodes2).sort().forEach(word => wordlist.push({'id':word,'group':nodes2[word]})); var oopd=d3.extent(wordlist, d => d.group); var wordlist2 = []; wordlist.map(w => {wordlist2.push({id:w.id,frequency: (100*w.group/oopd[1]).toFixed(2)})}); var i; for (i = 1; i < list2.length-1; i++) { if(list2[i].toString()!==list2[i-1].toString()){ list3.push({'source':list2[i][0],'target':list2[i][1],score:counter});counter=1; } else { counter+=1; } } //Object.keys(nodes2).forEach(n =>nodelist.push(n)); return {nodes:wordlist2,links:list3}; } 36:[] 30:{"id":"29f4babbbfc2b7c9","slug":"idea-mining-the-mueller-report","title":"Idea mining: the Mueller report","thumbnail":"aa96893d0393004e53a28cb94ac2d77e0b6e2ce4513a7761f24d284176b29e57","access_level":"public","create_time":"$D2019-06-06T15:14:52.932Z","update_time":"$D2019-07-16T07:58:55.105Z","view_time":null,"first_public_time":"$D2019-06-12T15:15:53.979Z","has_unpublished_edits":null,"fork":true,"stars":1,"starred":false,"owner":{"type":"team","id":"b1eda185b85db317","login":"meme-machines","name":"Diderot rebooted","avatar_url":"https://avatars-next.observableusercontent.com/04b5557199bf14bc03030f7ea88940069b57961198e8950cc349495e438681ef","tier":"starter_2024","bio":""},"version":1436,"publish_version":1436,"role":null,"authored":false,"authors":[{"type":"individual","id":"cf6309128d7d142b","login":"cjfpainter","name":"Chris PAINTER","avatar_url":"https://avatars-next.observableusercontent.com/b17254380dc7ccf1c5f46545b84ab92c25193c6e17454bb8f9a9509d7ec21253","tier":null,"bio":" I make software that finds associations in texts, or numbers. Just as you would do, given the time, but faster and better."}],"members":[{"type":"team","id":"b1eda185b85db317","login":"meme-machines","name":"Diderot rebooted","avatar_url":"https://avatars-next.observableusercontent.com/04b5557199bf14bc03030f7ea88940069b57961198e8950cc349495e438681ef","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Idea mining: the Mueller report","cells":[{"id":543,"value":"md`# Idea mining: the Mueller report`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":0,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":847,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1333,"value":"viewof conf = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":639,"value":"viewof target = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1273,"value":"datatable2 = table(\n \n table_data,{\n \n nully: () => 'No data',\n limit: 1500,\n enableFilter: false,\n enableCSVDownload: true\n \n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1387,"value":"html`

Data: Analyses, Sets, and Rules

Internals: Mouse action, node colours, and libraries

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":849,"value":"drag = simulation => {\n \n function dragstarted(d) {\n if (!d3.event.active) simulation.alphaTarget(0.3).restart();\n d.fx = d.x;\n d.fy = d.y;\n }\n \n function dragged(d) {\n d.fx = d3.event.x;\n d.fy = d3.event.y;\n }\n \n function dragended(d) {\n if (!d3.event.active) simulation.alphaTarget(0);\n d.fx = null;\n d.fy = null;\n }\n \n return d3.drag()\n .on(\"start\", dragstarted)\n .on(\"drag\", dragged)\n .on(\"end\", dragended);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1148,"value":"nodecolour = d3.scaleSequential(d3.extent(graph_data.nodes, d => d.frequency), d3.interpolateYlOrBr);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"width = 975","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"height = 600","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"d3 = require(\"d3@5\", \"d3-hexbin@0.2\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":659,"value":"import { table } from \"@gampleman/table\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W36","stdlib":"1"}} 31:"$30:body"