31:T94d,function createChart (record) { const height = 500 const { userId, digitDistribution, totalFriends } = record const expectedDistribution = benfordDistribution(totalFriends) const margin = {top: 50, right: 20, bottom: 30, left: 40} const maxY = _.max([ _.max(expectedDistribution), _.max(digitDistribution) ]) const x = d3.scaleLinear() .domain([1, 10]) .range([margin.left, width - margin.right]) const y = d3.scaleLinear() .domain([0, maxY + 1]).nice() .range([height - margin.bottom, margin.top]) function xAxis (g) { g.attr("transform", `translate(0, ${height - margin.bottom})`) .call(d3.axisBottom(x).tickValues(_.times(9, i => i + 1)).tickSizeOuter(0)) .call(g => { g.selectAll('text') .attr('x', (x(2) - x(1)) / 2) }) } function yAxis(g) { g.attr("transform", `translate(${margin.left},0)`) .call(d3.axisLeft(y).tickValues(_.times(maxY + 2, i => i)).tickFormat(d3.format('0'))) .call(g => g.select(".domain").remove()) .call(g => g.select(".tick:last-of-type text").clone() .attr("x", 4) .attr("text-anchor", "start") .attr("font-weight", "bold") .text('Counts')) } const svg = d3.create("svg") .attr("viewBox", [0, 0, width, height]) svg.append("path") .datum(digitDistribution) .attr("fill", "none") .attr("stroke", "steelblue") .attr("stroke-width", 1.5) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("d", d3.line() .x((d, i) => x(i + 1) + (x(2) - x(1)) / 2) .y(d => y(d)) .curve(d3.curveMonotoneX)) svg.append("path") .datum(expectedDistribution) .attr("fill", "none") .attr("stroke", "red") .attr("stroke-dasharray", ("1, 3")) .attr("stroke-width", 1.5) .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr('opacity', 0.3) .attr("d", d3.line() .x((d, i) => x(i + 1) + (x(2) - x(1)) / 2) .y(d => y(d)) .curve(d3.curveMonotoneX)) svg.append("g").call(xAxis) svg.append("g").call(yAxis) const { pValue } = cst.gof(digitDistribution, expectedDistribution) svg.append('text') .attr('x', 15) .attr('y', 30) .text(`ID = ${userId} • Friends = ${totalFriends}`) return svg.node() }32:[{"id":"4415f71021e19b5fe0d55319b909661e2840b3b21674221579b042e44dbd3ccb7728319a02001f91ad562caa5bfbd9bb614d69ad75315eae7c346ce6e25d783e","href":"https://static.observableusercontent.com/files/4415f71021e19b5fe0d55319b909661e2840b3b21674221579b042e44dbd3ccb7728319a02001f91ad562caa5bfbd9bb614d69ad75315eae7c346ce6e25d783e","name":"socfb-Caltech36.mtx","type":"application/octet-stream","lastModified":1605756653944.55,"size":128852},{"id":"0c62a5049d7f557b20c9d14c301fdea9239ddb13431d977d17d75edbbc2a494b21f3409a82e921604cc2804cf32873366f3bdd773c1827e1d03e00f7e2f6df73","href":"https://static.observableusercontent.com/files/0c62a5049d7f557b20c9d14c301fdea9239ddb13431d977d17d75edbbc2a494b21f3409a82e921604cc2804cf32873366f3bdd773c1827e1d03e00f7e2f6df73","name":"network.png","type":"image/png","lastModified":1605806104491.022,"size":1795334}] 2f:{"title":"Benford Bots","cells":[{"id":0,"value":"md`# Benford Bots`","pinned":false,"hidden":false,"mode":"ojs"},{"id":326,"value":"md`This is an attempted recreation of the findings from [Benford’s Law can detect malicious social bots](https://doi.org/10.5210/fm.v24i8.10163) by Jennifer Golbeck using data from [Caltech36 Facebook Network](http://networkrepository.com/socfb-Caltech36.php).`","pinned":false,"hidden":false,"mode":"ojs"},{"id":316,"value":"html`

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":31,"value":"md`- Download the [Caltech36 Facebook Network](http://networkrepository.com/socfb-Caltech36.php) data from [here](http://nrvis.com/download/data/socfb/socfb-Caltech36.zip). When extracted, this dataset will contain a file called \\`socfb-Caltech36.mtx\\` in [Matrix Market File Format](http://networkrepository.com/mtx-matrix-market-format.html).`","pinned":false,"hidden":false,"mode":"ojs"},{"id":17,"value":"input = FileAttachment(\"socfb-Caltech36.mtx\").text()","pinned":false,"hidden":false,"mode":"ojs"},{"id":36,"value":"md`- Read the contents into a [graph data structure](https://www.youtube.com/watch?v=c8P9kB1eun4). Each coordinate from the file represents a connection between two unique users.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":74,"value":"FileAttachment(\"network.png\").image()","pinned":false,"hidden":false,"mode":"ojs"},{"id":119,"value":"DOM.download(new Blob([toGraphml(g)], {type: \"text/graphml\"}), \"network.graphml\", \"Download as Graphml\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":49,"value":"g = {\n const g = cytoscape()\n const lines = input.split('\\n')\n const [, totalNodes, totalEdges] = lines[1].split(' ')\n input.split('\\n').forEach((line, i) => {\n if (i <= 1 || !line) return\n const [nodeA, nodeB] = line.split(' ')\n if (!g.$id(nodeA).length) g.add({ group: 'nodes', data: { id: nodeA } })\n if (!g.$id(nodeB).length) g.add({ group: 'nodes', data: { id: nodeB } })\n g.add({ \n group: 'edges', \n data: {\n id: `e${i}`,\n source: nodeA,\n target: nodeB\n }\n })\n })\n return g\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":66,"value":"md`- Traverse the graph and, for each user, visit all its connections and collect the number of friends that user has. Apply [Benford’s Law](https://en.wikipedia.org/wiki/Benford%27s_law#:~:text=Benford's%20law%2C%20also%20called%20the,life%20sets%20of%20numerical%20data.&text=If%20the%20digits%20were%20distributed,about%2011.1%25%20of%20the%20time.) to the collected friend counts of each friend of each user to transform that into a record containing:\n - user id\n - the distribution of first significant digit of each friend's friend count\n - the total number of friends`","pinned":false,"hidden":false,"mode":"ojs"},{"id":69,"value":"records = {\n return g.nodes().map(node => {\n const digitDistribution = _.times(9, () => 0)\n const friends = node.outgoers().nodes()\n friends.forEach(friend => {\n const friendCount = friend.indegree()\n const digit = parseInt(`${friendCount}`[0], 10)\n if (digit) {\n digitDistribution[digit - 1] += 1\n }\n })\n return {\n userId: node.data().id,\n totalFriends: friends.length,\n digitDistribution\n }\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":145,"value":"md`- For each record that has a friend count greater than X, test its [Goodness of Fit](https://stattrek.com/chi-square-test/goodness-of-fit.aspx) to the [expected frequency distribution](https://en.wikipedia.org/wiki/Benford%27s_law#Definition) of Benford’s Law. Output all the users that fall below a threshold of Y. And because low friend counts can give unreliable results`","pinned":false,"hidden":false,"mode":"ojs"},{"id":335,"value":"viewof minimumFriends = slider({\n value: 10,\n min: 1,\n max: _.max(g.nodes().map(n => n.indegree())),\n step: 1,\n description: \"Minimum number of friends to consider\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":375,"value":"recordsFilteredByX = _.filter(records, record => record.totalFriends > minimumFriends)","pinned":false,"hidden":false,"mode":"ojs"},{"id":181,"value":"viewof threshold = slider({\n value: 0.050,\n min: 0.001,\n max: 1.000,\n precision: 3,\n description: \"p-value threshold\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":381,"value":"recordsWithPvalue = _.map(recordsFilteredByX, record => {\n return { \n ...record,\n goodnessOfFitScore: goodnessOfFitScore(record)\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":161,"value":"bots = {\n const belowThreshold = _.filter(recordsWithPvalue, record => record.goodnessOfFitScore < threshold)\n return belowThreshold.sort((a, b) => {\n return a.goodnessOfFitScore - b.goodnessOfFitScore\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":304,"value":"md`## Bots (Possibly) - Sorted from worst score to best`","pinned":false,"hidden":false,"mode":"ojs"},{"id":196,"value":"chart = createChart(bots[botToView])","pinned":false,"hidden":false,"mode":"ojs"},{"id":237,"value":"viewof botToView = Scrubber(_.times(bots.length, i => i), {\n delay: 2000,\n autoplay: false\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":308,"value":"md`## Not Bots (Maybe) - Sorted from best score to worst`","pinned":false,"hidden":false,"mode":"ojs"},{"id":311,"value":"notBots = {\n const aboveOrEqualToThreshold = _.filter(recordsWithPvalue, record => record.goodnessOfFitScore >= threshold)\n return aboveOrEqualToThreshold.sort((a, b) => {\n return b.goodnessOfFitScore - a.goodnessOfFitScore\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":299,"value":"createChart(notBots[recordToView])","pinned":false,"hidden":false,"mode":"ojs"},{"id":295,"value":"viewof recordToView = Scrubber(_.times(notBots.length, i => i), {\n delay: 2000,\n autoplay: false\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":191,"value":"html`

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":75,"value":" md`## /support`","pinned":false,"hidden":false,"mode":"ojs"},{"id":127,"value":"function toGraphml(g) {\n return `\n\n\n \n ${g.nodes().map(n => ``).join('\\n ')}\n ${g.edges().map(e => ``).join('\\n ')}\n \n\n `.trim()\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":200,"value":"benfordDistribution = _.memoize(maxValue => {\n return [0.301, 0.176, 0.125, 0.097, 0.079, 0.067, 0.058, 0.051, 0.046].map(pct => pct * maxValue)\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":287,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":343,"value":"goodnessOfFitScore = _.memoize(record => {\n const { userId, totalFriends, digitDistribution } = record\n const expected = benfordDistribution(totalFriends)\n const { pValue } = cst.gof(digitDistribution, expected)\n return pValue\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"md`## /imports`","pinned":false,"hidden":false,"mode":"ojs"},{"id":11,"value":"d3 = require('d3@6')","pinned":false,"hidden":false,"mode":"ojs"},{"id":14,"value":"_ = require('lodash')","pinned":false,"hidden":false,"mode":"ojs"},{"id":27,"value":"cytoscape = { \n const cy = await require('cytoscape@3.17.0/dist/cytoscape.umd.js') \n // register plugins here\n return cy\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":151,"value":"cst = require('https://bundle.run/chi-sq-test@1.1.0')","pinned":false,"hidden":false,"mode":"ojs"},{"id":402,"value":"jszip = require(\"jszip@3/dist/jszip.min.js\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":174,"value":"import {slider} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":244,"value":"import {Scrubber} from \"@mbostock/scrubber\"","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}