2b:null 2c:["$","title",null,{"children":"D3 Force Dilemmas: Data Distortion | Chris Woods | Observable"}] 31:T580,{ tick; const container = d3.create('div').style('display','flex').style('align-items','center') const svg = container.append('svg').attr('width',250).attr('height',200) container.append('div').style('width','50%').html("1) A collide force that pushes overlapping circles away from eachother until they don't overlap") const nodes = demoData.map(d=> ({x: d.collideX, y: d.collideY, r: d.r})); const node = svg.append("g").attr('transform','translate(20,0)') .attr("fill", "#888") .attr("fill-opacity", 0.3) .attr("stroke", "#888") .selectAll("circle") .data(nodes) .join("circle") .attr("r", d=> d.r) .attr("cx", d => d.x) .attr("cy", d => d.y) .attr("opacity", 0 ) let ticks = -40 let ticker = d3.interval(elapsed => { if (elapsed > 2500) { node.attr('opacity',0) ticker.stop() } const simulation = d3.forceSimulation(nodes) .force("collide", d3.forceCollide().radius(d => 2+ d.r).strength(.3)) .velocityDecay(.90) .stop() .tick(ticks< 0 ? 0 : ticks); node .attr("opacity", d3.min([1, 1 + ticks/20])) .attr("cx", d => d.x) .attr("cy", d => d.y); ticks++ }, 20); // }) return container.node(); }32:T821,{ tick; const container = d3.create('div').style('display','flex').style('align-items','center') const svg = container.append('svg').attr('width',250).attr('height',200) container.append('div').style('width','50%').html("2) A directional force (forceX for horizontal beeswarms) that pushes each node towards the propper position on the axis mapped to the data values.") const nodes = demoData.map(d=> ({x: d.collideY, y: d.collideY, targetX: d.targetX, r: d.r})); const guide = svg.append('g').attr('transform', 'translate(20 130)') guide.append('g').selectAll('line').data(nodes).join('line').attr('stroke', '#444') .attr('x1',d=> d.targetX).attr('x2',d=> d.targetX) .attr('y1',0).attr('y2', -10) guide.append('line') .attr('x1',0).attr('x2',200) // .attr('x1',d3.min(nodes.map(d=>d.targetX))).attr('x2',d3.max(nodes.map(d=>d.targetX))) .attr('y1',0).attr('y2', 0).attr('stroke', '#444') guide.append('text').attr('x',100).attr('y', 20).text('target x positions').attr('text-anchor','middle') const node = svg.append("g").attr('transform','translate(20 0)') .attr("fill", "#888") .attr("fill-opacity", 0.3) .attr("stroke", "#888") .selectAll("circle") .data(nodes) .join("circle") .attr("r", d=> d.r) .attr("cx", d => d.x) .attr("cy", d => d.y) .attr("opacity", 0 ) let ticks = -40 let ticker = d3.interval(elapsed => { if (elapsed > 2500) { node.attr('opacity',0) ticker.stop() } const simulation = d3.forceSimulation(nodes) .force("forceX", d3.forceX(d=> d.targetX).strength(.01)) .stop() .tick(ticks< 0 ? 0 : ticks); node .attr("opacity", d3.min([1, 1 + ticks/20])) .attr("cx", d => {console.log(d); return d.x}) .attr("cy", d => d.y); ticks++ }, 20); // }) return container.node(); }33:T594,{ tick; const container = d3.create('div').style('display','flex').style('align-items','center') const svg = container.append('svg').attr('width',250).attr('height',200) container.append('div').style('width','50%').html("3) A directional force (forceY for horizontal beeswarms) that pushes each node towards towards the center of the axis that isn't mapped to data") const nodes = demoData.map(d=> ({x: d.x, y: d3.randomInt(50,150)(), targetY: 100, r: d.r})); const node = svg.append("g").attr('transform', 'translate(20 0)') .attr("fill", "#888") .attr("fill-opacity", 0.3) .attr("stroke", "#888") .selectAll("circle") .data(nodes) .join("circle") .attr("r", d=> d.r) .attr("cx", d => d.x) .attr("cy", d => d.y) .attr("opacity", 0 ) let ticks = -40 let ticker = d3.interval(elapsed => { if (elapsed > 2500) { node.attr('opacity',0) ticker.stop() } const simulation = d3.forceSimulation(nodes) .force("forceY", d3.forceY(d=> d.targetY).strength(.01)) .stop() .tick(ticks< 0 ? 0 : ticks); node .attr("opacity", d3.min([1, 1 + ticks/20])) .attr("cx", d => d.x) .attr("cy", d => d.y); ticks++ }, 20); // }) return container.node(); }34:T65d,{ tick; const container = d3.create('div').style('display','flex').style('align-items','center') const svg = container.append('svg').attr('width',250).attr('height',200) container.append('div').style('width','50%').html("In the case of beeswarms, each node is typically initialized with x and y at that node's target position (i.e. each bubble starts at the x and y positions that the forceX and forceY are told to push the node to).") const nodes = demoData.map(d=> ({x: d.x, y: d.collideY, r: d.r * 0.75})); const node = svg.append("g").attr('transform','translate(25 0)') .attr("fill", "#888") .attr("fill-opacity", 0.3) .attr("stroke", "#888") .selectAll("circle") .data(nodes) .join("circle") .attr("r", d=> d.r) .attr("cx", d => d.x) .attr("cy", d => d.y) .attr("opacity", 0 ) let ticks = -40 let ticker = d3.interval(elapsed => { if (elapsed > 2500) { node.attr('opacity',0) ticker.stop() } const simulation = d3.forceSimulation(nodes) .force("forceY", d3.forceY(d=> d.y).strength(.01)) .force("forceX", d3.forceX(d=> d.x).strength(1)) .force("collide", d3.forceCollide().radius(d=> d.r + 2).strength(.3)) .velocityDecay(0.9) .stop() .tick(ticks< 0 ? 0 : ticks); node .attr("opacity", d3.min([1, 1 + ticks/20])) .attr("cx", d => d.x) .attr("cy", d => d.y); ticks++ }, 20); // }) return container.node(); }35:T72c,htl.html`` /* Bubble Displacement from actual year Founded. 20+ years 10+ years 5+ years */ 36:T480,// chart 2 -- with interactivity. And color code of 5, 10, 15 off in each direction. { const container = d3.create('div').style('overflow', 'auto') const svg = container.append("svg").style('background-color', 'white') .attr("width", chartWidth + margin.left + margin.right) .attr("height", chartHeight + margin.top + margin.bottom) const button = container.append('div').style('padding', '10px').style('font-size', '1.2rem') .text('Move bubbles to show displacement:') .append('button').text('move to target X').style('margin', '10px') const g = svg.append("g") .attr("transform", `translate(${[ margin.left, margin.top]})`); g.call(addAxis) g.call(addDynamicLabelAndBubbles, swarmedDataDefaults, 'by years') let toggle = false const togglePoints = function() { toggle = !toggle button.text(toggle ? 'return bubbles' : 'move to target X') g.selectAll('circle') .transition().duration(300).attr('fill-opacity',toggle ? .4 : 1) .transition().duration(1100).attr('cx',d=> toggle ? xScale(d.Founded) : d.x) } button.on('click', togglePoints) return container.node(); }37:T48a,{ const container = d3.create('div').style('overflow', 'auto') const svg = container.append("svg").style('background-color', 'white') .attr("width", chartWidth + margin.left + margin.right) .attr("height", chartHeight + margin.top + margin.bottom) const button = container.append('div').style('padding', '10px').style('font-size', '1.2rem') .text('Move bubbles to show displacement:') .append('button').text('move to target X').style('margin', '10px') const g = svg.append("g") .attr("transform", `translate(${[ margin.left, margin.top]})`); g.call(addAxis) g.call(addDynamicLabelAndBubbles, swarmedDataXStrength2, 'none') let toggle = false const togglePoints = function() { toggle = !toggle button.text(toggle ? 'return bubbles' : 'show data dislocation') g.selectAll('circle') .transition().duration(300) .attr('fill', d=> colorByYears(xScale.invert(d.x) - xScale.invert(d.cx))) .attr('fill-opacity',toggle ? .4 : 1) .transition().duration(1100).attr('cx',d=> toggle ? xScale(d.Founded) : d.x) } button.on('click', togglePoints) return container.node(); }38:Tccd,// chart transitioning between x strength of 2,4, 8, 16. { const container = d3.create('div').style('overflow', 'auto') const svg = container.append("svg").style('background-color', 'white') .attr("width", chartWidth + margin.left + margin.right) .attr("height", chartHeight + margin.top + margin.bottom) const buttonGroup = container.append('div').style('padding', '10px').style('font-size', '1.2rem').text('ForceX strengths: ') const button = container.append('div').style('padding', '10px').style('font-size', '1.2rem') .text('Move bubbles to show displacement:') .append('button').text('move to target X').style('margin', '10px') const g = svg.append("g") .attr("transform", `translate(${[ margin.left, margin.top]})`); g.call(addAxis) g.call(addDynamicLabelAndBubbles, dataXHighStrengthDemo, 'by years', false ) let toggle = false let xStrengthCur = 'Default' const togglePoints = function() { toggle = !toggle button.text(toggle ? 'return bubbles' : 'move to target x') g.selectAll('circle') .transition().duration(300) .attr('fill', d=> colorByYears(xScale.invert(d.x) - xScale.invert(d.cx))) .attr('fill-opacity',toggle ? .4 : 1) .transition().duration(1100).attr('cx',d=> toggle ? xScale(d.Founded) : d['x' + xStrengthCur]) } button.on('click', togglePoints) const transitionPoints = function(str) { toggle = false button.text('move to target x') xStrengthCur = str toggle = false g.selectAll('circle') .transition().duration(600) .attr('cx', d=> d['x' + str]) .attr('cy', d=> d['y' + str]) .attr('fill', d=> colorByYears(xScale.invert(d['x' + str]) - xScale.invert(d.cx))) .attr('fill-opacity', 1) .on('end', () => { years.text(avgYearsDisplaced(dataXHighStrengthDemo, 'x' + xStrengthCur) + ' Years' + (xStrengthCur==16 ? " (FAILURE!)" : '')) displacementLabel.text('Average Displacement (forceX strength = ' + ( xStrengthCur == 'Default' ? 0.1 : xStrengthCur) + ')') }) } const mouseoverLabel = g.append('g').attr('transform','translate(' + 250 + ' ' + 25 + ')') const displacementLabel = mouseoverLabel.append("text") .attr("fill","black") .style("font-size","20px") .text('Average Displacement (forceX strength = ' + ( xStrengthCur == 'Default' ? 0.1 : xStrengthCur) + ')') const years = mouseoverLabel.append("text") .attr('y', 35) .attr("fill","black") .style("font-size","30px") .text(avgYearsDisplaced(dataXHighStrengthDemo, 'x' + xStrengthCur) + ' Years') //Math.round(d3.mean(dataXHighStrengthDemo.map(d=> Math.abs(xScale.invert(d['x' + xStrengthCur]) - d.Founded)))) + ' Years') buttonGroup.append('button').text('strength = default (0.1)').on('click', () => transitionPoints('Default')) buttonGroup.append('button').text('strength = 2').on('click', () => transitionPoints(2)) buttonGroup.append('button').text('strength = 4').on('click', () => transitionPoints(4)) buttonGroup.append('button').text('strength = 8').on('click', () => transitionPoints(8)) buttonGroup.append('button').text('strength = 16').on('click', () => transitionPoints(16)) return container.node(); }39:T4b2,{ const container = d3.create('div').style('overflow', 'auto') const chartHeight = 525 const svg = container.append("svg").style('background-color', 'white') .attr("width", chartWidth + margin.left + margin.right) .attr("height", chartHeight + margin.top + margin.bottom) const button = container.append('div').style('padding', '10px').style('font-size', '1.2rem') .text('Move bubbles to show displacement:') .append('button').text('move to target X').style('margin', '10px') const g = svg.append("g") .attr("transform", `translate(${[ margin.left, margin.top]})`); g.call(addAxis, chartHeight) g.call(addDynamicLabelAndBubbles, swarmedDataOptimized, 'by years') let toggle = false const togglePoints = function() { toggle = !toggle button.text(toggle ? 'return bubbles' : 'show data dislocation') g.selectAll('circle') .transition().duration(300) .attr('fill', d=> colorByYears(xScale.invert(d.x) - xScale.invert(d.cx))) .attr('fill-opacity',toggle ? .4 : 1) .transition().duration(1100).attr('cx',d=> toggle ? xScale(d.Founded) : d.x) } button.on('click', togglePoints) return container.node(); }3a:T48a,## The rest of my d3-force Collection This notebook is one of a few I have put together exploring different elements of d3 force algorithms. So far notebooks include: * **[d3 Force Dissected - Alpha](https://observablehq.com/@osserman/d3-force-dissected-alpha)**. How "temperature" is controlled over the source of simulation. * **[d3 Force Dissected - Movement](https://observablehq.com/@osserman/d3-force-dissected-movement)**. how node movement is controlled by force strength, simulation temperature (alpha) and simulation friction (velocityDecay). * **[d3 Force Dilemmas - Data Distortion](https://observablehq.com/@osserman/d3-force-dilemmas-data-distortion)** - It's too easy to accidentally over-distort data with force-based visualizations like beeswarms. This notebook explores why and offers some tips on addressing the issue. The link for the collection is here: **Note:** I've got a running list of potential topics for additional notebooks from talking with fellow data viz developers. If you have a pain-point or a commonly misunderstood part of d3 force simulations, please feel free to leave a comment here and I'll add it to my list! 3b:T699,addDynamicLabelAndBubbles = (g, data, fillStyle = 'none', addLabel= true, showDisplayYear = true) => { let fillFunction; if (fillStyle == 'none') { fillFunction = d=> "#666" } else if (fillStyle == 'years off') { fillFunction = d => xScale.invert(d.x) - xScale.invert(d.cx) >= 5 ? 'green' : xScale.invert(d.x) - xScale.invert(d.cx) <= -5 ? 'blue' : '#666' } else if (fillStyle == 'by years') { fillFunction = d => colorByYears(xScale.invert(d.x) - xScale.invert(d.cx)) } let school, years, enrollment; if(addLabel){ const mouseoverLabel = g.append('g').attr('transform','translate(' + 25 + ' ' + 25 + ')') school = mouseoverLabel.append("text") .attr("fill","black") .style("font-size","25px") years = mouseoverLabel.append("text") .attr('y', 25) .attr("fill","black") .style("font-size","20px") .text('hover or click to see details') enrollment = mouseoverLabel.append("text") .attr('y', 55) .attr("fill","black") .style("font-size","20px") } g.selectAll("circle") .data(data) .join("circle") .attr("stroke", "none") .attr('fill', fillFunction) .attr("cx", d => d.x) .attr("cy", d => d.y) .attr("r", d => d.r) .on('mouseover',(e,d) => { if(addLabel){ console.log('trying to add label') school.text(d.School || d.Name || d.Institution) years.text('Founded in ' + d.Founded + (showDisplayYear ? (' (displayed at ' + Math.round(xScale.invert(d.x)) + ')' ) : '') ) enrollment.text('~'+ d3.format('.2s')(d.Enrollment) + ' enrolled') } }) }3c:[{"id":"67ae36553cbfdf402706178a76c059c8978af644c577751947be2bcfca3d4bc39d98f2cb58c7321449ced2189f3d5f51658c20912c0b9e6275cfcd46a9f15261","href":"https://static.observableusercontent.com/files/67ae36553cbfdf402706178a76c059c8978af644c577751947be2bcfca3d4bc39d98f2cb58c7321449ced2189f3d5f51658c20912c0b9e6275cfcd46a9f15261","name":"swarmedDataAllStates_initial_attempt.json","type":"application/json","lastModified":1706714666258.046,"size":524853},{"id":"2bd38a9030169df234689090abc56411aa520c8db8339da0539a6b7bfa50118cfe5e4040f7aeb80749c71667228fa7fc1ae7a7da17f5c45b49822f51ce169365","href":"https://static.observableusercontent.com/files/2bd38a9030169df234689090abc56411aa520c8db8339da0539a6b7bfa50118cfe5e4040f7aeb80749c71667228fa7fc1ae7a7da17f5c45b49822f51ce169365","name":"swarmedDataAllStates_final.json","type":"application/json","lastModified":1706714666258.046,"size":521630},{"id":"ff66eb5a91bf61df09b6e6884399f99632f9a5c13d80cb961a3ff11bd9412502277f95bcd17db87e64c04fa8de96f3ce8351903b661a9f039a4a757c427e5774","href":"https://static.observableusercontent.com/files/ff66eb5a91bf61df09b6e6884399f99632f9a5c13d80cb961a3ff11bd9412502277f95bcd17db87e64c04fa8de96f3ce8351903b661a9f039a4a757c427e5774","name":"swarmedDataRandomSample-Defaults@1.json","type":"application/json","lastModified":1706714666258.046,"size":156209},{"id":"aec433002d019101efd9e2a1228c0eaacdec531304206a4ed7a911711294381eb81cb5d2bb646278fb4e599199c5ec6d80c2f5f3e9c6df7835008c1aa8bad2e7","href":"https://static.observableusercontent.com/files/aec433002d019101efd9e2a1228c0eaacdec531304206a4ed7a911711294381eb81cb5d2bb646278fb4e599199c5ec6d80c2f5f3e9c6df7835008c1aa8bad2e7","name":"swarmedDataRandomSample-XStrength2.json","type":"application/json","lastModified":1706714666258.046,"size":155961},{"id":"d55c6e0cf761b4b984d8aabde753cce710d49979b954be58128cbfddded373e228a902f47735a202ec77b05a3cdd251e30455d5271ecc5640c2b93cb2cd189d8","href":"https://static.observableusercontent.com/files/d55c6e0cf761b4b984d8aabde753cce710d49979b954be58128cbfddded373e228a902f47735a202ec77b05a3cdd251e30455d5271ecc5640c2b93cb2cd189d8","name":"swarmedDataRandomSample-XStrength4.json","type":"application/json","lastModified":1706714666258.046,"size":155938},{"id":"67f06caab1d3abdc228fe4c7ceb4a12f521126c2d19b1f8cac5e214d9f06bf58510a36a75b78f4faf5fcc15000625d8bbe4d441529d6930e15fdce546ee246ea","href":"https://static.observableusercontent.com/files/67f06caab1d3abdc228fe4c7ceb4a12f521126c2d19b1f8cac5e214d9f06bf58510a36a75b78f4faf5fcc15000625d8bbe4d441529d6930e15fdce546ee246ea","name":"swarmedDataRandomSample-XStrength8.json","type":"application/json","lastModified":1706714666258.046,"size":156360},{"id":"ad6ec790a1e3c2f2af9ff34ed81266f82ef9a91b19f39c8803b4954464add48d808fb97715893cc88daa8de6d696cfa9787433dac9d9586b7db7a9ab536baf05","href":"https://static.observableusercontent.com/files/ad6ec790a1e3c2f2af9ff34ed81266f82ef9a91b19f39c8803b4954464add48d808fb97715893cc88daa8de6d696cfa9787433dac9d9586b7db7a9ab536baf05","name":"swarmedDataRandomSample-XStrength16.json","type":"application/json","lastModified":1706714666258.046,"size":156352},{"id":"9b6e696f517671e1a0e83afe31e923fa6c35cf3a92ef3e98e4bab90c4d9b76be92533438a100ef7e74b1c7c657f42d90489b8a17c6b1b2d614b20cd620615bbb","href":"https://static.observableusercontent.com/files/9b6e696f517671e1a0e83afe31e923fa6c35cf3a92ef3e98e4bab90c4d9b76be92533438a100ef7e74b1c7c657f42d90489b8a17c6b1b2d614b20cd620615bbb","name":"swarmedDataRandomSample-Y-and-alpha-adjusted.json","type":"application/json","lastModified":1706714666258.046,"size":156269}] 2f:{"id":"a0f2f434e7a05415","slug":null,"title":"D3 Force Dilemmas: Data Distortion","thumbnail":"6e050cbc0d8b3aa7e2221d507e2508f3ac1c9b893fea9bb1524b39df05953a92","access_level":"public","create_time":"$D2024-01-31T15:24:26.258Z","update_time":"$D2024-01-31T15:24:26.258Z","view_time":null,"first_public_time":"$D2024-01-31T15:24:26.258Z","has_unpublished_edits":null,"fork":{"id":"749cf52e0a87ace2","slug":"d3-force-dilemmas-data-distortion","title":"D3 Force Dilemmas: Data Distortion","owner":{"id":"1be7bc7ec19604c8","login":"osserman","name":"Stephen Osserman"}},"stars":0,"starred":false,"owner":{"type":"team","id":"d341d19da0e1fab8","login":"chriswoodssays","name":"Chris Woods","avatar_url":"https://avatars-next.observableusercontent.com/12282d336153633f11d891d9487b6543b0a1d776c4d30a007177747f4c07a0b8","tier":"starter_2024"},"version":1259,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"6e87883179511b65","login":"chriswoodssays","name":"Chris Woods","avatar_url":"https://avatars-next.observableusercontent.com/12282d336153633f11d891d9487b6543b0a1d776c4d30a007177747f4c07a0b8","tier":null}],"members":[{"type":"individual","id":"6e87883179511b65","login":"chriswoodssays","name":"Chris Woods","avatar_url":"https://avatars-next.observableusercontent.com/12282d336153633f11d891d9487b6543b0a1d776c4d30a007177747f4c07a0b8","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"D3 Force Dilemmas: Data Distortion","cells":[{"id":0,"value":"# D3 Force Dilemmas: Data Distortion\n## Beeswarms and Avoiding Over-Distortion of Data","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":14,"value":"**D3 Force** simulations gives us an amazing power to **visualize broad patterns** in data while **maintaining granularity** and enabling interactivity. \n\n**Beeswarm charts are a great example** -- showing both individual data points and the shape of their distribution. \n\nIn running a force simulation to generate them though, distortion happens. Minor distortion is often not a problem, but the distortion is not always minor. This notebook:\n1. Demonstrates **why this data-distortion happens**;\n2. Offers some approaches to **evaluate its severity**; and\n3. **Suggests tips** for managing the distortion. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":577,"value":"## Quick Beeswarm Review\n\nHarry Steven's and others have shared [useful code for making beeswarms](https://observablehq.com/@harrystevens/force-directed-beeswarm). In that code and others examples, beeswarms are made via the combination of three forces: ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":787,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":651,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":801,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":596,"value":"For each tick of the simulation, each of these three forces is calculated for each node and the forces are blended together to determine how each node moves. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":858,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":874,"value":"That means the forceX and forceY only have any work to do when the collide force pushes nodes away from their starting/desired position. Given this set up, it seems like there shouldn't be a ton of data distortion, but there often is. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":924,"value":"To demonstrate, here's a beeswarm attempting to show the founding years of over 400 randomly sampled Colleges and Universities in the United States.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":19,"value":"## Founding Dates U.S. Colleges and Universities","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":25,"value":"// chart 1. using default force strength for creating Beeswarms. \n{\n const container = d3.create('div').style('overflow', 'auto')\n const svg = container.append(\"svg\").style('background-color', 'white')\n .attr(\"width\", chartWidth + margin.left + margin.right)\n .attr(\"height\", chartHeight + margin.top + margin.bottom)\n\n const g = svg.append(\"g\")\n .attr(\"transform\", `translate(${[ margin.left, margin.top]})`);\n \n g.call(addAxis)\n\n g.call(addDynamicLabelAndBubbles, swarmedDataDefaults, 'none', true, false)\n \n return container.node();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":27,"value":"Unfortunate **this representation doesn't come close to reflecting the actual data**! \n\nThe average school above is shown being founded ${avgYearsDisplaced(swarmedDataDefaults, 'x')} years away from when it actually was. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":293,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":29,"value":"$36","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":31,"value":"## So why is this distortion happening? \n \nThe problem is that unless we tell the force simulation to privilege the forceX -- which drives the placement along the X axis -- over the forceY -- which is just geared towards keeping things compact, the bubble will move to fill up space in a way that can substantially distort the data. \n\nIn this initial beeswarm, we left all the forces -- collide, forceX and forceY -- at their defaults strengths. The most **obvious solution is to up the strength of the X force** -- essentially pulling each node harder towards our desired X position. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1049,"value":"### Chart from simulation with forceX strength = 2","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":43,"value":"$37","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":53,"value":"This has helped significantly, but there is still enough distortion to cause problems for many use-cases. We've already upped the forceX strength well above the recommended range. But since it worked, let's keep going and see what happens. **(click the buttons below the chart with different forceX strengths to see)**","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":65,"value":"$38","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":461,"value":"### Why does this approach fail so badly as X-strength increases? \n\nBecause strength of the force not actually how hard the pull is towards the desired point, but rather how much of the way towards the defined point it should accelerate on each tick. The default -- 0.1 -- is [described in the documentation]((https://d3js.org/d3-force/position#x_strength) as being equivalent of the node accelerating 1/10th of the way towards its target destination every tick. Strength of 1 would mean pulling it immediately all the way to the destination in one tick (absent other forces and simulation \"temperature\" and \"friction\"). \n\nSo the documentation recommends keeping the strength in [0,1] range. It sometimes works to go a bit above 1, but once you get above a certain strength it can breakaway. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":73,"value":"So **a better approach: pair a modest increase in X-force strength, with a reduction in the Y-strength**. When the nodes are being pulled more gently to the center of the Y axis there is more space for them to come towards their X destination. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":551,"value":"This version combines a forceX strength to 1.3 with a forceY strength to 0.003. It achieves an **average displacement of just ${avgYearsDisplaced(swarmedDataOptimized,'x')} Years.** \n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":535,"value":"$39","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1024,"value":"## Last Step: Applying to the full dataset","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":78,"value":"Now that we've come to a good solution, let's apply to the full dataset of 1400+ schools. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1082,"value":"{\n const container = d3.create('div').style('overflow', 'auto')\n const chartHeight = 550\n const svg = container.append(\"svg\").style('background-color', 'white')\n .attr(\"width\", chartWidth + margin.left + margin.right)\n .attr(\"height\", chartHeight + margin.top + margin.bottom)\n \n const g = svg.append(\"g\")\n .attr(\"transform\", `translate(${[ margin.left, margin.top]})`);\n \n g.call(addAxis, chartHeight)\n\n g.call(addDynamicLabelAndBubbles, swarmedDataAllStates_attempt1, 'by years')\n\n return container.node();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1099,"value":"## Argh!! \n\nWe started with a random sample of data points because with the full dataset each attempt at simulating takes quite a bit of time to run and rerun. But those parameters so carefully tuned for a random sample don't work so great on the full data set. \n\nLuckily **one fairly minor set of tweaks addresses this new problem**. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1104,"value":"### Radii and distortion \nThe bigger the radii of each node the harder it will be for the node to fit nicely near the target position we want it in. Using smaller bubbles and the same other settings will minimize the data-distortion of a beeswarm, but at the cost of legibility and interactivity. But with this many data points it's necessary to shrink them. (The other minor change I made here was lowering the alpha decay parameter, but that's a topic for another notebook). ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1112,"value":"{\n const container = d3.create('div').style('overflow', 'auto')\n const chartHeight = 550\n const svg = container.append(\"svg\").style('background-color', 'white')\n .attr(\"width\", chartWidth + margin.left + margin.right)\n .attr(\"height\", chartHeight + margin.top + margin.bottom)\n \n const g = svg.append(\"g\")\n .attr(\"transform\", `translate(${[ margin.left, margin.top]})`);\n \n g.call(addAxis, chartHeight)\n\n g.call(addDynamicLabelAndBubbles, swarmedDataAllStates_Final, 'by years')\n\n return container.node();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1118,"value":"The Parameters here are: forceX strength = 1.3, forceY strength = 0.003, alphaDecay = 0.005, with a radius-scale domain of [1, 10] (compared to [1, 30] in the charts above). The raw data, tweak-able parameters and simulation code are all in [this notebook](https://observablehq.com/d/9673de17cfcd168a). ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":606,"value":"# Take-away tips\n\n* **Evaluate the distortion** of points by looking at their distance traveled along the axis mapped to the data (in this case the x-axis mapped to Year each school was founded). \n* If the distortion is more than acceptable for your use case, **tweak the strengths of the position force - forceX and forceY**, increasing the one mapped to data (forceX for horizontal beeswarms), and lowering the one that pushes bubbles to the center (forceY for vertical beeswarms). \n* **Set appropriate radii**. Larger radii will lead to improved readability and improved potential for interactivity, but will increase data distortion. Find a good balance for your use case!\n* **You can start with a sample, but expect to need changes**. Most simulations run much faster with fewer nodes. Tuning the simulation with a random sample of your data can help... but expect to need to make changes once you switch to your full dataset. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1183,"value":"**Note on the dataset**: This notebooks uses the 1400+ schools easily scrape-able from Wikipedia's lists of U.S. Colleges and Universities by state. There are lots of inconsistencies and many states with missing data. I wasn't too worried since this is mainly about d3 forces and beeswarms, not the specific data, but if you have a cleaner dataset of US Colleges and Universities with enrollment and founding dates, please let me know, would be happy to swap it in!","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1255,"value":"$3a","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":86,"value":"## Appendix","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1139,"value":"### Nodes/Data Sets","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":610,"value":"// demo data for 200 x 200 \ndemoData = d3.range(20).map(()=> ({\n r: d3.randomInt(3,15)(),\n x: d3.randomInt(0,200)(),\n y: 100,\n collideX: 100 + Math.random(),\n collideY: 100 + Math.random()\n })\n).map(d=> ({...d, targetX: d.x, targetY: d.y}))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1238,"value":"These datasets were downloaded with different sets of parameters from this [simulation notebook](https://observablehq.com/d/9673de17cfcd168a). ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":88,"value":"swarmedDataDefaults = FileAttachment(\"swarmedDataRandomSample-Defaults@1.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":414,"value":"swarmedDataXStrength2 = FileAttachment(\"swarmedDataRandomSample-XStrength2.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":439,"value":"swarmedDataXStrength4 = FileAttachment(\"swarmedDataRandomSample-XStrength4.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":443,"value":"swarmedDataXStrength8 = FileAttachment(\"swarmedDataRandomSample-XStrength8.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":445,"value":"swarmedDataXStrength16 = FileAttachment(\"swarmedDataRandomSample-XStrength16.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":448,"value":"dataXHighStrengthDemo = swarmedDataDefaults\n .map((d,i)=> ({...d, \n x2: swarmedDataXStrength2[i].x, y2: swarmedDataXStrength2[i].y,\n x4: swarmedDataXStrength4[i].x, y4: swarmedDataXStrength4[i].y, \n x8: swarmedDataXStrength8[i].x, y8: swarmedDataXStrength8[i].y, \n x16: swarmedDataXStrength16[i].x, y16: swarmedDataXStrength16[i].y,\n xDefault: swarmedDataDefaults[i].x, yDefault: swarmedDataDefaults[i].y\n }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":528,"value":"swarmedDataOptimized =FileAttachment(\"swarmedDataRandomSample-Y-and-alpha-adjusted.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":532,"value":"optimizedSampledDataForceParams = ({\n maxRadius: 30, \n xStrength: 1.3, \n yStrength: 0.003, \n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1089,"value":"swarmedDataAllStates_attempt1 = FileAttachment(\"swarmedDataAllStates_initial_attempt.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1108,"value":"swarmedDataAllStates_Final = FileAttachment(\"swarmedDataAllStates_final.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":482,"value":"### Helper Functions","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":969,"value":"//Average dislocation in years from given dataset and force-moved xField name\navgYearsDisplaced = function(data, xField, format= '.1f'){\n return d3.format(format)(d3.mean(data, d=> Math.abs(xScale.invert(d[xField]) - d.Founded)))\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":216,"value":"colorByYears = function(yearsOff) { \n const absYrsOff = Math.abs(yearsOff)\n if (absYrsOff < 5) {return '#777'}\n let h = yearsOff < 0 ? 120 : 240;\n let s = 1; \n let l = absYrsOff >= 20 ? 0.35 : absYrsOff >= 10 ? 0.6 : 0.85 \n return d3.hsl(h, s, l)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":107,"value":"xDiff = function(yrs){\n return xScale(yrs) - xScale(0)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1149,"value":"### Chart Setup","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":92,"value":"margin = ({left: 20, right: 20, top: 20, bottom: 20})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1171,"value":"Chart width, height and xScale (along with radius scale) need to be set before the simulation is run, so these next values are hard-coded here to match the parameters I used to generate the simulations. ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":94,"value":"chartWidth = 900 // manually set to match settings when simulation were run. ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":96,"value":"chartHeight = 350","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":99,"value":"xScale = d3.scaleLinear()\n .domain([1636,2020]) // manually pulled from extent of data\n .range([0, chartWidth])\n .nice()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1158,"value":"### Functions to Add Chart Elements","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":114,"value":"addAxis = function(g, height = chartHeight) {\n const axis = g.append(\"g\")\n .call(d3.axisBottom(xScale).tickSize(height ).tickFormat(d3.format(\"d\"))) \n .style(\"font-size\",\"17px\")\n .style(\"color\",\"#666\")\n axis.selectAll('line').attr('stroke', '#ccc')\n axis.select('path').style('display','none');\n return axis\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":126,"value":"$3b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1169,"value":"### Tick every 3 seconds to reload / rerun demo simulations at top","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":742,"value":"tick = {\n while (true) {\n yield Promises.delay(3000);\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W3c","stdlib":"1","titling":"auto"}} 30:"$2f:body"