2c:null 2d:["$","title",null,{"children":"Idea mining: Currency Markets | Diderot rebooted | Observable"}] 32:T9e8,graph = { const links = graph_data.links.map(d => Object.create(d)); const nodes = graph_data.nodes.map(d => Object.create(d)); const height = 500; const simulation = d3.forceSimulation(nodes) .force("link", d3.forceLink(links).id(d => d.id).distance(100)) .force("charge", d3.forceManyBody(100)) .force("center", d3.forceCenter(width / 2, height / 2)) .force("collide", d3.forceCollide(content.spaceman).strength(0.5)) .on("tick", ticked); //const svg = d3.select(DOM.svg(width, height)); const svg = d3.select(DOM.svg(1024, height)) .style('max-width', '100%') .style('height', 'auto'); const link = svg.append("g") .attr("stroke","gray") .attr("stroke-opacity", 0.95) .attr("marker-end", "url(#arrow)") .selectAll("line") .data(links) .enter().append("line") .attr("stroke-width", d => 0.5+Math.sqrt(d.score)); const node = svg.append("g") .attr("stroke", "gray") .attr("stroke-width", 1.0) .selectAll("circle") .data(nodes) .enter().append("circle") .attr("r", 8) .attr("fill", d => nodecolour(d.frequency)) .call(drag(simulation)); node.on("mouseover", (d) => { link.style('stroke-opacity', function(l) { if (d === l.source) return 1; else return 0.25; }) }) .on("mouseout", (d) => { link.style("stroke-opacity", 0.2) }); svg.append("defs").append("marker") .attr("id", "arrow") .attr("viewBox", "0 -5 10 10") .attr("refX", 20) .attr("refY", 0) .attr("opacity", 0.2) .attr("markerWidth", 8) .attr("markerHeight", 8) .attr("orient", "auto") .append("svg:path") .attr("d", "M0,-5L10,0L0,5"); node.append("title") .text(d => d.id); const text = svg.append("g").attr("class", "labels").selectAll("g") .data(nodes) .enter().append("g"); text.append("text") .attr("x", 14) .attr("y", ".31em") .attr("opacity", 0.7) .attr("pointer-events", "none") .style("font-family", "sans-serif") .style("font-size", "0.7em") .text(function(d) { return d.id; }) function ticked() { link .attr("x1", d => d.source.x) .attr("y1", d => d.source.y) .attr("x2", d => d.target.x) .attr("y2", d => d.target.y); node .attr("cx", d => d.x) .attr("cy", d => d.y); text .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; }) } return svg.node(); }33:T8a5,table_data ={ function netoff(fis1, fis2) { function subset(list1, list2) { var sub = true; list1.forEach(function (word) { if (!list2.includes(word)) { sub = false } }) return sub; } var nu = []; fis1.forEach(function (set) { var include = true; var iset = set.itemSet; fis2.forEach(function (set2) { var iset2 = set2.itemSet; if (subset(iset, iset2)) { include = false; } }) if (include) { nu.push(set) } }); return nu; }; const fis_sets = analysis_result.results.frequentItemSets; var list=[]; var fis_keys = Object.keys(fis_sets); var max_len = fis_keys.length - 1; var maximal_sets = []; fis_keys.forEach(function (fis_k, i) { var arity2 = i + 2; if (i < max_len ) { var maximals = netoff( fis_sets[fis_k], fis_sets[arity2.toString()]); if (i>1 && maximals.length > 0) { maximal_sets.push(maximals) } } }) var longest = fis_sets[fis_keys.length.toString()]; maximal_sets.push(longest); // maximal_sets.forEach(s => list.push({s:s.itemSet,sup:s.support})); //var no_singles = maximal_sets.shift(); var flattened = [].concat.apply([],maximal_sets); flattened.forEach(f =>{f.support=(f.support*1).toFixed(2)}) const map1 = flattened.map(x => x!== undefined ? JSON.stringify(x.itemSet)+ JSON.stringify(x.support): []); list.push(map1); return flattened}34:T5d7, graph_data = {counter=1; let zoom = content.zoom2; console.log(zoom); var list=[]; var list2 =[]; var list3 =[]; const nodes2 ={}; var nodelist=[]; const linkset = {}; const links2 = []; var wordlist=[]; var counter=1; let start = analysis_result.results.associationRules.length * zoom/100; analysis_result.results.associationRules.slice(start).forEach(rule => { links2.push(rule);//console.log(from, to, rule.confidence);nodes2[link.rhs[0]]+=1 }) ; links2.forEach(function (link) { var c = link.rhs[0]; if(c in nodes2){nodes2[c]+=1}else{nodes2[c]=1}; link.lhs.forEach(l => { if(l in nodes2){nodes2[l]+=1}else{nodes2[l]=1}; var todo = [l,c]; list.push(todo); }); }); list2=list.sort(); console.log(list2); Object.keys(nodes2).sort().forEach(word => wordlist.push({'id':word,'group':nodes2[word]})); var oopd=d3.extent(wordlist, d => d.group); var wordlist2 = []; wordlist.map(w => {wordlist2.push({id:w.id,frequency: (100*w.group/oopd[1]).toFixed(2)})}); var i; for (i = 1; i < list2.length-1; i++) { if(list2[i].toString()!==list2[i-1].toString()){ list3.push({'source':list2[i][0],'target':list2[i][1],score:counter});counter=1; } else { counter+=1; } } //Object.keys(nodes2).forEach(n =>nodelist.push(n)); return {nodes:wordlist2,links:list3}; } 35:[] 30:{"id":"e32953dd93c2ee20","slug":"idea-mining-currency-markets","title":"Idea mining: Currency Markets","thumbnail":"955cd5ac1b0363e9a7795e108db230bf215d938877b94056edbbd2c4f6736fa3","access_level":"public","create_time":"$D2020-01-02T20:21:16.300Z","update_time":"$D2020-09-15T12:59:19.151Z","view_time":null,"first_public_time":"$D2020-01-05T18:19:55.387Z","has_unpublished_edits":null,"fork":true,"stars":1,"starred":false,"owner":{"type":"team","id":"b1eda185b85db317","login":"meme-machines","name":"Diderot rebooted","avatar_url":"https://avatars-next.observableusercontent.com/04b5557199bf14bc03030f7ea88940069b57961198e8950cc349495e438681ef","tier":"starter_2024","bio":""},"version":2455,"publish_version":2455,"role":null,"authored":false,"authors":[{"type":"individual","id":"cf6309128d7d142b","login":"cjfpainter","name":"Chris PAINTER","avatar_url":"https://avatars-next.observableusercontent.com/b17254380dc7ccf1c5f46545b84ab92c25193c6e17454bb8f9a9509d7ec21253","tier":null,"bio":" I make software that finds associations in texts, or numbers. Just as you would do, given the time, but faster and better."}],"members":[{"type":"team","id":"b1eda185b85db317","login":"meme-machines","name":"Diderot rebooted","avatar_url":"https://avatars-next.observableusercontent.com/04b5557199bf14bc03030f7ea88940069b57961198e8950cc349495e438681ef","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Idea mining: Currency Markets","cells":[{"id":543,"value":"md`# Idea mining: Currency Markets`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2226,"value":"html`

How to discover intermarket dynamics?

By reducing fiat and crypto currency prices from CryptoCompare to change indicators and noticing coincidences we can expose the associations at play, and extrapolate rules.

\n\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1506,"value":"html`

Step one: Standardise the data against the Dollar

Mark as true if movement against the reference currency exceeds +/- 0.1% in the minute concerned.

The graphic below synchronises the ${analysis_result.prices.length} minutes preceding the time of this hourly report`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2265,"value":"lastupdate= {\n let dateObj = new Date(analysis_result.prices.pop().time); \n return `${dateObj.toUTCString()}`;}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1672,"value":"chart = {\n const div = DOM.element('div');\n \n const svg = d3.select(div)\n .selectAll('svg')\n .data(grouped2)\n .enter()\n .append('svg')\n .attr('width', width + margin.left + margin.right)\n .attr('height', height + margin.top + margin.bottom)\n .attr('display', 'block')\n .append('g')\n .attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');\n \n svg\n .append('text')\n .attr('font-family', 'Arial, sans-serif')\n .attr('font-size', '12px')\n .attr('font-weight', 700)\n .style('fill', '#888')\n .attr('x', margin.left)\n .attr('y', 12)\n .text(d => d.values[0].name)\n \n svg\n .append('path')\n .classed('line', true)\n .attr('fill', '#7c90c1')\n .attr('stroke', 'none')\n .attr('d', d => area2(d.values))\n\n \n \n return div;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2191,"value":"html`

Step two: count the coincidences

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1273,"value":"datatable2 = table(\n \n table_data,{\n \n nully: () => 'No data',\n limit: 1500,\n enableFilter: true,\n enableCSVDownload: true,\n columns: [{ \n key: 'itemSet',\n name: 'Coincidences',\n render: val => `${val}`\n },\n { \n key: 'support',\n name: 'Frequency %',\n render: val => `${val*100}`\n }, \n ]\n \n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1387,"value":"html`

Step three: Extrapolate the rules

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1504,"value":"rules = table(\n \n analysis_result.results.associationRules,{\n \n nully: () => 'No data',\n limit: 1500,\n enableFilter: true,\n enableCSVDownload: true,\n columns: [{ \n key: 'lhs',\n name: 'If this happens',\n render: val => `${val}`\n },\n { \n key: 'rhs',\n name: 'Then so should this',\n render: val => `${val}`\n }, \n { \n key: 'confidence',\n name: 'with confidence %',\n render: val => `${val}`\n }, \n ]\n \n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2205,"value":"html`

Step four: Inspect the derived intermarket dynamics (mouse over for links and drag)

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":847,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2311,"value":"viewof content = form(html``)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2416,"value":"html`

* The number of currencies in the rule`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2294,"value":"html`

Raw data, changes, and analysis result (JSON)

Post-processing (JSON)

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2110,"value":"grouped2 = {\n let g=d3.nest().key(d => d.name).entries(data2)\n .map(d => { \n d.values = d.values.sort(sortByDate)\n return d;\n});\n g.sort(function(a, b){\n \n if(a.key < b.key) return -1;\n if(a.key > b.key) return 1;\n return 0;\n});\n return g}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2075,"value":"data2 ={\n let symbols= analysis_result.results.frequentItemSets[\"1\"].map(d => d.itemSet[0]).sort();\n let changes=[];\n analysis_result.symbols.forEach(d => { \n let done= d.body.split(\" \");\n symbols.forEach(s =>{if (done.indexOf(s)<0)\n {console.log(s);\n changes.push({date:new Date(d.time),name:s,total:0})};\n \n })\n \n done.forEach(dd => changes.push({date:new Date(d.time),name:dd,total:1}))\n \n });\n let filtered=changes.filter(x => {return x.name!==\"\"});\n return filtered; \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1278,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":907,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1717,"value":"html`

Utility functions

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1734,"value":"x2 = d3.scaleTime()\n .domain(d3.extent(data2, d => d.date))\n .range([margin.left, width - margin.right])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1737,"value":"y2 = d3.scaleLinear()\n .domain([0, d3.max(data2, d => d.total)])\n.range([height - margin.bottom, margin.top])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1763,"value":"area2 = d3.area()\n .x(d => x2(d.date))\n .y0(height - margin.bottom - margin.top)\n .y1(d => y2(d.total))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1695,"value":"sortByDate = (a, b) => { \n if (a.date < b.date) {\n return -1;\n } else if (a.date > b.date) {\n return 1;\n }\n \n return 0;\n }","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1688,"value":"margin = ({top: 0, right: 20, bottom: 1, left: 0})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1686,"value":"height = 40","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1684,"value":"width = window.innerWidth","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":343,"value":"html`

Internals: Mouse action, node colours, and libraries

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":849,"value":"drag = simulation => {\n \n function dragstarted(d) {\n if (!d3.event.active) simulation.alphaTarget(0.3).restart();\n d.fx = d.x;\n d.fy = d.y;\n }\n \n function dragged(d) {\n d.fx = d3.event.x;\n d.fy = d3.event.y;\n }\n \n function dragended(d) {\n if (!d3.event.active) simulation.alphaTarget(0);\n d.fx = null;\n d.fy = null;\n }\n \n return d3.drag()\n .on(\"start\", dragstarted)\n .on(\"drag\", dragged)\n .on(\"end\", dragended);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1148,"value":"nodecolour = d3.scaleSequential(d3.extent(graph_data.nodes, d => d.frequency), d3.interpolateYlOrBr);","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"d3 = require(\"d3@5\", \"d3-hexbin@0.2\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2313,"value":"import {form} from \"@mbostock/form-input\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":659,"value":"import { table } from \"@gampleman/table\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2442,"value":"id = 'UA-151170198-1'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2440,"value":"pageAnalytics = {\n await require(`https://www.googletagmanager.com/gtag/js?id=${id}`).catch(() => {});\n const dataLayer = window.dataLayer = window.dataLayer || [];\n const location = html``;\n gtag('js', new Date());\n gtag('config', id, {\n 'page_path': location.pathname,\n 'page_location' : location.href\n });\n return true;\n function gtag() { dataLayer.push(arguments); }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2444,"value":"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1"}} 31:"$30:body"