2c:null 2d:["$","title",null,{"children":"Idea mining: the Remixatron 0.1 | Diderot rebooted | Observable"}] 32:T6ca, html`

This notebook lets you play with the notion of ideas as word sets, and see its application to the analysis of two texts central to David Gunkel's ground-breaking work on cultural evolution, On Remixology.

A key finding of this cooperation was that ideas can emerge from the overlap between texts, in the form of patterns that only show when texts combine. In this case you will see a connection in the mashup, albeit faint, between 'Logos' and 'History' that is found in neither component.

You'll use frequent item-set mining, a form of unsupervised learning more commonly seen in suggestion engines, to find words that appear often in the same text, in this case on the same sentence. The more often one word suggests another, the greater the association perceived. Read more..

By stripping Jacques Derrida's Writing and Difference and Plato's Phaedrus of common words, and by noticing coincidences in the remainder, we can expose the associations at play. These are displayed below; darker nodes indicate greater frequency, and thicker connections indicate stronger associations.

You can explore the space by selecting either text, or a mashup of both. In order to balance the samples in the mashup the Derrida sample is reduced to contain the same number of sentences as are in the Phaedrus. Please select from the menu below, and enjoy!

Object.create(d)); const nodes = graph_data.nodes.map(d => Object.create(d)); const height = 500; const simulation = d3.forceSimulation(nodes) .force("link", d3.forceLink(links).id(d => d.id).distance(100)) .force("charge", d3.forceManyBody(100)) .force("center", d3.forceCenter(width / 2, height / 2)) .on("tick", ticked); //const svg = d3.select(DOM.svg(width, height)); const svg = d3.select(DOM.svg(1024, height)) .style('max-width', '100%') .style('height', 'auto'); const link = svg.append("g") .attr("stroke","gray") .attr("stroke-opacity", 0.95) //.attr("marker-end", "url(#arrow)") .selectAll("line") .data(links) .enter().append("line") .attr("stroke-width", d => 0.5+Math.sqrt(d.score)); const node = svg.append("g") .attr("stroke", "gray") .attr("stroke-width", 1.0) .selectAll("circle") .data(nodes) .enter().append("circle") .attr("r", 8) .attr("fill", d => nodecolour(d.frequency)) .call(drag(simulation)); node.on("mouseover", (d) => { link.style('stroke-opacity', function(l) { if (d === l.source) return 1; else return 0.25; }) }) .on("mouseout", (d) => { link.style("stroke-opacity", 0.2) }); svg.append("defs").append("marker") .attr("id", "arrow") .attr("viewBox", "0 -5 10 10") .attr("refX", 20) .attr("refY", 0) .attr("opacity", 0.2) .attr("markerWidth", 8) .attr("markerHeight", 8) .attr("orient", "auto") .append("svg:path") .attr("d", "M0,-5L10,0L0,5"); node.append("title") .text(d => d.id); const text = svg.append("g").attr("class", "labels").selectAll("g") .data(nodes) .enter().append("g"); text.append("text") .attr("x", 14) .attr("y", ".31em") .attr("opacity", 0.7) .attr("pointer-events", "none") .style("font-family", "sans-serif") .style("font-size", "0.7em") .text(function(d) { return d.id; }) function ticked() { link .attr("x1", d => d.source.x) .attr("y1", d => d.source.y) .attr("x2", d => d.target.x) .attr("y2", d => d.target.y); node .attr("cx", d => d.x) .attr("cy", d => d.y); text .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; }) } return svg.node(); }34:T89c,table_data ={ function netoff(fis1, fis2) { function subset(list1, list2) { var sub = true; list1.forEach(function (word) { if (!list2.includes(word)) { sub = false } }) return sub; } var nu = []; fis1.forEach(function (set) { var include = true; var iset = set.itemSet; fis2.forEach(function (set2) { var iset2 = set2.itemSet; if (subset(iset, iset2)) { include = false; } }) if (include) { nu.push(set) } }); return nu; }; const fis_sets = analysis_result.results.frequentItemSets; var list=[]; var fis_keys = Object.keys(fis_sets); var max_len = fis_keys.length - 1; var maximal_sets = []; fis_keys.forEach(function (fis_k, i) { var arity2 = i + 2; if (i < max_len ) { var maximals = netoff( fis_sets[fis_k], fis_sets[arity2.toString()]); if (i>1 && maximals.length > 0) { maximal_sets.push(maximals) } } }) var longest = fis_sets[fis_keys.length.toString()]; maximal_sets.push(longest); // maximal_sets.forEach(s => list.push({s:s.itemSet,sup:s.support})); //var no_singles = maximal_sets.shift(); var flattened = [].concat.apply([],maximal_sets); // flattened.forEach(f =>{f.support=(f.support*1)}) const map1 = flattened.map(x => x!== undefined ? JSON.stringify(x.itemSet)+ JSON.stringify(x.support): []); list.push(map1); return flattened}35:T522, graph_data = {counter=1; var list=[]; var list2 =[]; var list3 =[]; const nodes2 ={}; var nodelist=[]; const linkset = {}; const links2 = []; var wordlist=[]; var counter=1; analysis_result.results.associationRules.forEach(rule => { links2.push(rule);//console.log(from, to, rule.confidence);nodes2[link.rhs[0]]+=1 }) ; links2.forEach(function (link) { var c = link.rhs[0]; if(c in nodes2){nodes2[c]+=1}else{nodes2[c]=1}; link.lhs.forEach(l => { if(l in nodes2){nodes2[l]+=1}else{nodes2[l]=1}; var todo = [l,c]; list.push(todo); }); }); list2=list.sort(); Object.keys(nodes2).sort().forEach(word => wordlist.push({'id':word,'group':nodes2[word]})); var oopd=d3.extent(wordlist, d => d.group); var wordlist2 = []; wordlist.map(w => {wordlist2.push({id:w.id,frequency: (100*w.group/oopd[1]).toFixed(2)})}); var i; for (i = 1; i < list2.length-1; i++) { if(list2[i].toString()!==list2[i-1].toString()){ list3.push({'source':list2[i][0],'target':list2[i][1],score:counter});counter=1; } else { counter+=1; } } //Object.keys(nodes2).forEach(n =>nodelist.push(n)); return {nodes:wordlist2,links:list3}; } 36:[] 30:{"id":"9b3bbf6efcd9c976","slug":"idea-mining-the-remixatron-0-1","title":"Idea mining: the Remixatron 0.1","thumbnail":"b3a978ac27165473ddf040a4a8d00d9e5a655f960b3c37ab1fe4edb1c237f830","access_level":"public","create_time":"$D2019-07-01T17:48:06.966Z","update_time":"$D2019-08-14T07:51:42.210Z","view_time":null,"first_public_time":"$D2019-07-04T11:25:42.430Z","has_unpublished_edits":null,"fork":true,"stars":0,"starred":false,"owner":{"type":"team","id":"b1eda185b85db317","login":"meme-machines","name":"Diderot rebooted","avatar_url":"https://avatars-next.observableusercontent.com/04b5557199bf14bc03030f7ea88940069b57961198e8950cc349495e438681ef","tier":"starter_2024","bio":""},"version":1510,"publish_version":1510,"role":null,"authored":false,"authors":[{"type":"individual","id":"cf6309128d7d142b","login":"cjfpainter","name":"Chris PAINTER","avatar_url":"https://avatars-next.observableusercontent.com/b17254380dc7ccf1c5f46545b84ab92c25193c6e17454bb8f9a9509d7ec21253","tier":null,"bio":" I make software that finds associations in texts, or numbers. Just as you would do, given the time, but faster and better."}],"members":[{"type":"team","id":"b1eda185b85db317","login":"meme-machines","name":"Diderot rebooted","avatar_url":"https://avatars-next.observableusercontent.com/04b5557199bf14bc03030f7ea88940069b57961198e8950cc349495e438681ef","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Idea mining: the Remixatron 0.1","cells":[{"id":543,"value":"md`# Idea mining: the Remixatron 0.1`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":0,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":847,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":639,"value":"viewof target = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1273,"value":"datatable2 = table(\n \n table_data,{\n \n nully: () => 'No data',\n limit: 1500,\n enableFilter: false,\n enableCSVDownload: true\n \n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1387,"value":"html`

Data: Analyses, Sets, and Rules

Internals: Mouse action, node colours, and libraries

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":849,"value":"drag = simulation => {\n \n function dragstarted(d) {\n if (!d3.event.active) simulation.alphaTarget(0.3).restart();\n d.fx = d.x;\n d.fy = d.y;\n }\n \n function dragged(d) {\n d.fx = d3.event.x;\n d.fy = d3.event.y;\n }\n \n function dragended(d) {\n if (!d3.event.active) simulation.alphaTarget(0);\n d.fx = null;\n d.fy = null;\n }\n \n return d3.drag()\n .on(\"start\", dragstarted)\n .on(\"drag\", dragged)\n .on(\"end\", dragended);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1148,"value":"nodecolour = d3.scaleSequential(d3.extent(graph_data.nodes, d => d.frequency), d3.interpolateYlOrBr);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"width = 975","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"height = 600","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"d3 = require(\"d3@5\", \"d3-hexbin@0.2\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":659,"value":"import { table } from \"@gampleman/table\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W36","stdlib":"1"}} 31:"$30:body"