2c:null 2d:["$","title",null,{"children":"Idea mining: The Headstart Wikipedia index | Diderot rebooted | Observable"}] 32:Taa7,graph = { const links = graph_data.links.map(d => Object.create(d)); const nodes = graph_data.nodes.map(d => Object.create(d)); const height = 500; const simulation = d3.forceSimulation(nodes) .force("link", d3.forceLink(links).id(d => d.id).distance(graphcontrols.linkdistance)) .force("charge", d3.forceManyBody().strength(d => graphcontrols.mbstrength) .distanceMin(graphcontrols.mbmindist) .distanceMax(graphcontrols.mbmaxdist)) .force("center", d3.forceCenter(width / 2+graphcontrols.centerx, height / 2 + graphcontrols.centery)) .force("collide", d3.forceCollide(content.spaceman).strength(0.5)) .on("tick", ticked); //const svg = d3.select(DOM.svg(width, height)); const svg = d3.select(DOM.svg(1024, height)) .style('max-width', '100%') .style('height', 'auto'); const link = svg.append("g") .attr("stroke","gray") .attr("stroke-opacity", 0.95) .attr("marker-end", "url(#arrow)") .selectAll("line") .data(links) .enter().append("line") .attr("stroke-width", d => 2.0+Math.sqrt(d.score)); const node = svg.append("g") .attr("stroke", "gray") .attr("stroke-width", 1.0) .selectAll("circle") .data(nodes) .enter().append("circle") .attr("r", 8) .attr("fill", d => nodecolour(d.frequency)) .call(drag(simulation)); node.on("mouseover", (d) => { link.style('stroke-opacity', function(l) { if (d === l.source) return 1; else return 0.6; }) }) .on("mouseout", (d) => { link.style("stroke-opacity", 0.6) }); svg.append("defs").append("marker") .attr("id", "arrow") .attr("viewBox", "0 -5 10 10") .attr("refX", 20) .attr("refY", 0) .attr("opacity", 0.2) .attr("markerWidth", 8) .attr("markerHeight", 8) .attr("orient", "auto") .append("svg:path") .attr("d", "M0,-5L10,0L0,5"); node.append("title") .text(d => d.id); const text = svg.append("g").attr("class", "labels").selectAll("g") .data(nodes) .enter().append("g"); text.append("text") .attr("x", 14) .attr("y", ".31em") .attr("opacity", 0.7) .attr("pointer-events", "none") .style("font-family", "sans-serif") .style("font-size", "0.7em") .text(function(d) { return d.id; }) function ticked() { link .attr("x1", d => d.source.x) .attr("y1", d => d.source.y) .attr("x2", d => d.target.x) .attr("y2", d => d.target.y); node .attr("cx", d => d.x) .attr("cy", d => d.y); text .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; }) } return svg.node(); }33:Ta5a,graph_data = {counter=1; var list=[]; var list2 =[]; var list3 =[]; let nodes2 ={}; var nodelist=[]; const linkset = {}; const links2 = []; var wordlist=[]; var counter=1; if(patterns) { let end = Math.ceil(patterns.length * content.zoom2 / 100); console.log('end', end); let patternslice= patterns.slice(0,end); patternslice.forEach(example => { let score=example.score; let words=example.pattern.split(":"); let x = words.length; words.forEach((word,i) => { //console.log(word,i) if(word in nodes2){nodes2[word]+=1}else{nodes2[word]=1}; for( var y = i+1; y < x;y++){ links2.push({source:word,target:words[y],score:score}); console.log(example.pattern,word,'+',words[y], score); let linkname = word+':'+words[y]; if(linkname in linkset){linkset[linkname]+=1}else{linkset[linkname]=1}; } }); //console.log(from, to, rule.confidence);nodes2[link.rhs[0]]+=1 }) ;}; Object.keys(nodes2).sort().forEach(word => wordlist.push({'id':word,'frequency':nodes2[word]})); Object.keys(linkset).sort().forEach(link => { let names = link.split(':'); let source= names[0]; let target= names[1]; let count = linkset[link] list3.push({'source':source,'target':target,'frequency':linkset[link]}) }); return {nodes:wordlist,links:list3}; } /*links2.forEach(function (link) { var c = link.rhs[0]; if(c in nodes2){nodes2[c]+=1}else{nodes2[c]=1}; link.lhs.forEach(l => { if(l in nodes2){nodes2[l]+=1}else{nodes2[l]=1} links2.forEach(function (link) { var c = link.rhs[0]; if(c in nodes2){nodes2[c]+=1}else{nodes2[c]=1}; link.lhs.forEach(l => { if(l in nodes2){nodes2[l]+=1}else{nodes2[l]=1}; var todo = [l,c]; list.push(todo); }); }); list2=list.sort(); Object.keys(nodes2).sort().forEach(word => wordlist.push({'id':word,'group':nodes2[word]})); var oopd=d3.extent(wordlist, d => d.group); var wordlist2 = []; wordlist.map(w => {wordlist2.push({id:w.id,frequency: (100*w.group/oopd[1]).toFixed(2)})}); var i; for (i = 1; i < list2.length-1; i++) { if(list2[i].toString()!==list2[i-1].toString()){ list3.push({'source':list2[i][0],'target':list2[i][1],score:counter});counter=1; } else { counter+=1; } }*/ //Object.keys(nodes2).forEach(n =>nodelist.push(n)); 34:[] 30:{"id":"842030e37ba8f01f","slug":"idea-mining-the-headstart-wikipedia-index","title":"Idea mining: The Headstart Wikipedia index","thumbnail":"396fcf85f4f78f4d32fb3e52dbcadfe7ce16d96ebe48481bedcf0ba532e39d27","access_level":"public","create_time":"$D2019-08-31T14:44:39.744Z","update_time":"$D2019-12-16T11:50:46.187Z","view_time":null,"first_public_time":"$D2019-08-31T14:46:00.247Z","has_unpublished_edits":null,"fork":true,"stars":0,"starred":false,"owner":{"type":"team","id":"b1eda185b85db317","login":"meme-machines","name":"Diderot rebooted","avatar_url":"https://avatars-next.observableusercontent.com/04b5557199bf14bc03030f7ea88940069b57961198e8950cc349495e438681ef","tier":"starter_2024","bio":""},"version":2290,"publish_version":2290,"role":null,"authored":false,"authors":[{"type":"individual","id":"cf6309128d7d142b","login":"cjfpainter","name":"Chris PAINTER","avatar_url":"https://avatars-next.observableusercontent.com/b17254380dc7ccf1c5f46545b84ab92c25193c6e17454bb8f9a9509d7ec21253","tier":null,"bio":" I make software that finds associations in texts, or numbers. Just as you would do, given the time, but faster and better."}],"members":[{"type":"team","id":"b1eda185b85db317","login":"meme-machines","name":"Diderot rebooted","avatar_url":"https://avatars-next.observableusercontent.com/04b5557199bf14bc03030f7ea88940069b57961198e8950cc349495e438681ef","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Idea mining: The Headstart Wikipedia index","cells":[{"id":1739,"value":"md`# Idea mining: The Headstart Wikipedia index`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2181,"value":"html`

\n
Words can be like X-rays, if you use them properly—they’ll go through anything. You read and you’re pierced.
\n
—Aldous Huxley, Brave New World
\n

This notebook is connected to the Headstart Index, a database of patterns identified in Wikipedia by item set mining. You can read more about the idea survey and method, and input your own selections here

Use it as a basic template for your own area of interest. Take a fork, change the default value in the pattern finder below, add your special stuff, and publish! If you get stuck, leave a comment to cry for help.

\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":461,"value":"html`

Pattern finder

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":639,"value":"viewof scan = form(html``)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2280,"value":"md`### Matching Patterns`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1381,"value":"viewof keystable = {\n return table(\n keys,\n {\n nully: () => 'No data',\n limit: 500,\n enableFilter: false,\n enableCSVDownload: true,\n columns: [{ \n key: 'pattern',\n name: `Patterns that match .. ${scan.search}`,\n render: val => `${val}`\n }]\n }\n)}\n ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2282,"value":"md`### Matching Pages`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1578,"value":"mutable pagestable = {if(pages.length==0)return \"nada\";\n return table(\n pages,\n {\n nully: () => 'No data',\n limit: 500,\n enableFilter: false,\n enableCSVDownload: true,\n columns: [{ \n key: 'url',\n name: `Showing: ${clickedPattern.pattern}`,\n render: val => `${val}`\n },\n \n \n {key:'score',\n name:'Score'}]\n }\n)}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2286,"value":"md`### The page in view`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2207,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2227,"value":"html`

The key words in the page, and how they connect

Layout

link distance 0/100

many body force strength -100/100

many body min distance 0/50

many body max distance 50/1000\n

center x -300/300\n

center y -300/300\n \n`)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1826,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1882,"value":"nodecolour = d3.scaleSequential(d3.extent(graph_data.nodes, d => d.frequency), d3.interpolateYlOrBr);","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1880,"value":"drag = simulation => {\n \n function dragstarted(d) {\n if (!d3.event.active) simulation.alphaTarget(0.3).restart();\n d.fx = d.x;\n d.fy = d.y;\n }\n \n function dragged(d) {\n d.fx = d3.event.x;\n d.fy = d3.event.y;\n }\n \n function dragended(d) {\n if (!d3.event.active) simulation.alphaTarget(0);\n d.fx = null;\n d.fy = null;\n }\n \n return d3.drag()\n .on(\"start\", dragstarted)\n .on(\"drag\", dragged)\n .on(\"end\", dragended);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1593,"value":"mutable clickedPage=\"none\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1552,"value":"mutable clickedPattern=\"none\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":642,"value":"keys = { const source = await \nd3.json(\"https://sherlock.scalingo.io/headstart_patterns?search=\"+scan.search,datas => { return datas;});\n let list=[];\n source.result[1].forEach(s=>{\n let a = s.split(\":\").slice(2).join(\":\");\n list.push({pattern:a})\n }); \n if (list.length==0){list.push({pattern:\"nought, nothing, etc.\"})}\n return list\n }\n\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1561,"value":"pages = { if (clickedPattern==\"none\") return [];\n let url = \"https://sherlock.scalingo.io/headstart_pattern_pages?search=\";\n let search=clickedPattern.pattern.split(\" \").join(\":\");\n console.log('refreshing ',url+search);\n const source=await \nd3.json(url+search,datas => { return datas;});\n let list=[];\n let k='';\n source.result.forEach((s,i)=>{\n if(i % 2==0)\n {k = s}\n else\n {list.push({url:k,score:s})}\n });\n \n if (list.length==0){list.push({reply:\"nought, nothing, etc.\"})}\n return list\n }","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1612,"value":"patterns = { if (clickedPage==\"none\") return [];\n let url = \"https://sherlock.scalingo.io/headstart_page_patterns?search=\";\n let search=encodeURIComponent(clickedPage.url);\n console.log(url+search);\n const source=await \nd3.json(url+search,datas => { return datas;});\n let list=[];\n let k='';\n source.result.forEach((s,i)=>{\n if(i % 2==0)\n {k = s}\n else\n {list.push({pattern:k,score:s})}\n });\n \n if (list.length==0){list.push({reply:\"nought, nothing, etc.\"})}\n let sorted = list.reverse();\n return list\n }","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1418,"value":"viewof matches = {\n //console.log(keystable);\n d3.selectAll('.nestor')\n .on('click', function(d,i) {\n mutable clickedPattern=keys[i];\n });\nreturn(keystable.length)};","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1588,"value":"viewof wikipages = {\n if(pagestable){\n d3.selectAll('.rotsen')\n .on('click', function(d,i) {\n mutable clickedPage=pages[i];\n });}\nreturn(!pagestable==undefined?pagestable.length:0)};","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1662,"value":"viewof tensor = {\n if(patternstable){\n d3.selectAll('.tensor')\n .on('click', function(d,i) {\n //This gets passed back to the top\n let nextsearch = patterns[i].pattern.split(\"+\").join(\" \");\n console.log(nextsearch);\n \n mutable clickedPattern = {pattern:nextsearch} ;\n // mutable clickedPattern=patterns[i].url.split(\"+\").join(\" \");\n });}\nreturn({nodes:[],links:[]})};","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1280,"value":"html`

Pattern finder

The key words in the page, and how they connect

Display settings

Layout

Libraries used ..