2d:null 2e:["$","title",null,{"children":"Visualizing ideas you perceive in text | Lucas Tostes | Observable"}] 33:T1593,viz = { const height = 370; const colors = d3.schemeObservable10 ?? d3.schemeTableau10; const { prp, text_data: raw_text_data } = prp_obj; const svg = d3 .create("svg") .attr("viewBox", [0, 0, width, height]) .attr("width", width) .attr("height", height) .attr("style", "max-width: 100%; height: auto; overflow:visible;"); const textStartX = d3.sum(prp.widths) / 2; const base_gap = 2; const gap = 6; const base_dist_text = 8; const dist_text = 6; const linesBasetransform = `translate(${20},${50})`; const text_data = raw_text_data .filter((f) => (f.kind = "text")) .map((d) => ({ ...d, betweenness: graphObj.betweenness.find( ({ term }) => normalize(term) == normalize(d.segment) )?.betweenness })); const clinks = text_data .map(({ segment }, i) => [ ...text_links.filter( (f) => normalize(f.source) == normalize(segment) && text_data .filter((ff, fi) => fi > i) .map((d) => normalize(d.segment)) .includes(normalize(f.target)) ), ...text_links .filter( (f) => normalize(f.target) == normalize(segment) && text_data .filter((ff, fi) => fi > i) .map((d) => normalize(d.segment)) .includes(normalize(f.source)) ) .map(({ source, target, count }) => ({ source: target, target: source, count })) ]) .flat() .filter((f) => f.source); console.log(clinks); const links_basedata = clinks .map((d) => ({ ...d, source_seg: text_data.find( (f) => normalize(f.segment) == normalize(d.source) ), target_seg: text_data.find( (f) => normalize(f.segment) == normalize(d.target) ) })) .filter((f) => f.source_seg && f.target_seg) .map((d) => ({ ...d, dist: d.target_seg.word_start - d.source_seg.word_end })) .filter((f) => f.dist > 0); const links_ordered_data = d3.sort(links_basedata, (a, b) => d3.ascending(a.dist, b.dist) ); // Counting anchor line order text_data.map(({ i, segment }) => { let count = 0; links_ordered_data.forEach((link, l_index) => { if (normalize(link.source) == normalize(segment)) { links_ordered_data[l_index].source_order = count; count += 1; } if (normalize(link.target) == normalize(segment)) { links_ordered_data[l_index].target_order = count; count += 1; } }); }); const links_data = links_ordered_data .map((d) => ({ ...d, source_gap: base_gap + gap * d.source_order, source_marker_y: base_dist_text + dist_text * d.source_order, target_gap: base_gap + gap * d.target_order, target_marker_y: base_dist_text + dist_text * d.target_order })) .map((d) => ({ ...d, r: (d.target_seg.word_start - d.source_seg.word_end + d.source_gap + d.target_gap) / 2, source_marker_x1: d.source_seg.word_start + d.source_gap, source_marker_x2: d.source_seg.word_end - d.source_gap, target_marker_x1: d.target_seg.word_start + d.target_gap, target_marker_x2: d.target_seg.word_end - d.target_gap })); // console.log(links_data.map(({ // swapped, source, target // }) => ({ // swapped, source, target // }))); // return { links_data, text_links, clinks, text_data, raw_text_data }; const linkPath = (d) => { const y = base_dist_text + dist_text * Math.max(d.source_order, d.target_order); const p = d3.path(); p.moveTo(d.source_marker_x1, d.source_marker_y); p.lineTo(d.source_marker_x2, d.source_marker_y); p.lineTo(d.source_marker_x2, y); p.arc(d.source_marker_x2 + d.r, y, d.r, Math.PI, 0, true); p.lineTo(d.target_marker_x1, d.target_marker_y); p.lineTo(d.target_marker_x2, d.target_marker_y); return p.toString(); }; const countDomain = [...new Set(links_data.map((d) => d.count))].sort( d3.ascending ); const color = d3 .scaleOrdinal() .domain(countDomain.reverse()) .range([ "#d53e4f", "#f46d43", "#fdae61", "#fee08b", "#e6f598", "#abdda4", "#66c2a5", "#3288bd" ]); const paths = svg .append("g") .attr("transform", linesBasetransform) .selectAll("path") .data(links_data) .join("path") .attr("d", linkPath) .attr("stroke", (d) => color(d.count)) .attr("stroke-width", "4px") .attr("fill", "none"); const betweennessColor = d3 .scaleLinear() .domain([0, d3.max(graphObj.betweenness, (d) => d.betweenness)]) .range(["black", d3.schemeRdYlBu[6][5]]); const text = svg .append("g") .attr("transform", linesBasetransform) .selectAll("text") .data(text_data) .join("text") .text((d) => d.segment) .attr("fill", (d) => !showbetweenness ? "black" : betweennessColor(d.betweenness) ) .attr("transform", (d) => `translate(${d.cw - d.w},0)`) .attr("font-family", font_family) .attr("font-size", font_size); const legend = Plot.legend({ color: { type: "categorical", domain: countDomain, range: countDomain.map(color) }, width: 20, height: 100, label: "co-occurrence weight" }); return html`

${svg.node()}

${legend}

`; }34:T4bc,stemObj = { const { allTokens, sentenceTokens } = tokenProcessing; const stopwordSet = new Set( stopwordsText .split(/\s+/) .filter(Boolean) .map((w) => keepAccents ? w.toLowerCase() : w.toLowerCase().normalize("NFD").replace(/\p{M}/gu, "") ) ); const contentSentences = sentenceTokens.map((ts) => ts .filter((w) => !stopwordSet.has(w) && w.length > 1) .map((w) => (useStems ? stemmer.stem(w) : w)) ); const contentTokens = allTokens .filter((t) => !stopwordSet.has(t.word) && t.word.length > 1) .map((t) => ({ ...t, stem: stemmer.stem(t.word), unit: useStems ? stemmer.stem(t.word) : t.word })); const m = new Map(); for (const t of contentTokens) { if (!m.has(t.stem)) m.set(t.stem, new Map()); const g = m.get(t.stem); g.set(t.word, (g.get(t.word) ?? 0) + 1); } const stemGroups = [...m.entries()] .map(([stem, g]) => ({ stem, count: [...g.values()].reduce((a, b) => a + b, 0), variants: [...g.keys()].join(", ") })) .sort((a, b) => b.count - a.count) .slice(0, 30); return { stopwordSet, contentSentences, contentTokens, stemGroups }; }35:Tc55,chart = { const width = 928; const height = 560; const colors = d3.schemeObservable10 ?? d3.schemeTableau10; const color = d3.scaleOrdinal(colors); let selectedNode = ""; const selectNode = (name) => { selectedNode.value = name; }; const clearSelectedNode = () => { selectedNode.value = ""; }; const nodes = graphObj.graph.mapNodes((n) => { const info = graphObj.nodeInfo.get(n); return { id: n, r: 4 + 2 * Math.sqrt(info.count), ...info }; }); const links = graphObj.graph.mapEdges((e, attrs, source, target) => ({ source, target, weight: attrs.weight })); const labeled = new Set( [...nodes] .sort((a, b) => b.count - a.count) .slice(0, 30) .map((d) => d.id) ); const simulation = d3 .forceSimulation(nodes) .force( "link", d3 .forceLink(links) .id((d) => d.id) .distance(40) ) .force("charge", d3.forceManyBody().strength(-120)) .force("center", d3.forceCenter(width / 2, height / 2)) .force( "collide", d3.forceCollide((d) => d.r + 2) ); const svg = d3 .create("svg") .attr("viewBox", [0, 0, width, height]) .attr("width", width) .attr("height", height) .attr("style", "max-width: 100%; height: auto;"); const link = svg .append("g") .attr("stroke", "#999") .attr("stroke-opacity", 0.5) .selectAll("line") .data(links) .join("line") .attr("stroke-width", (d) => Math.sqrt(d.weight)); const node = svg .append("g") .attr("stroke", "#fff") .attr("stroke-width", 1) .selectAll("circle") .data(nodes) .join("circle") .attr("r", (d) => d.r) .attr("fill", (d) => color(d.community)) .style("cursor", "pointer") .on("click", (event, d) => selectNode(d.id)); node .append("title") .text( (d) => `${d.id}\ncount: ${d.count}\ndegree: ${d.degree}\nweighted degree: ${ d.wdegree }\nbetweenness: ${d.betweenness.toFixed(3)}\ncommunity: ${d.community}` ); const label = svg .append("g") .selectAll("text") .data(nodes.filter((d) => labeled.has(d.id))) .join("text") .text((d) => d.id) .attr("font-size", 9) .attr("dx", (d) => d.r + 2) .attr("dy", 3) .attr("pointer-events", "none"); node.call( d3 .drag() .on("start", (event, d) => { if (!event.active) simulation.alphaTarget(0.3).restart(); d.fx = d.x; d.fy = d.y; }) .on("drag", (event, d) => { d.fx = event.x; d.fy = event.y; }) .on("end", (event, d) => { if (!event.active) simulation.alphaTarget(0); d.fx = null; d.fy = null; }) ); simulation.on("tick", () => { link .attr("x1", (d) => d.source.x) .attr("y1", (d) => d.source.y) .attr("x2", (d) => d.target.x) .attr("y2", (d) => d.target.y); node.attr("cx", (d) => d.x).attr("cy", (d) => d.y); label.attr("x", (d) => d.x).attr("y", (d) => d.y); }); invalidation.then(() => simulation.stop()); return svg.node(); return { graph_disp: svg.node(), links }; }36:T6f8,graphObj = { const m = new Map(); for (const t of stemObj.contentTokens) { if (!m.has(t.unit)) m.set(t.unit, { term: t.unit, count: 0, variants: new Set() }); const e = m.get(t.unit); e.count++; e.variants.add(t.word); } const termTable = [...m.values()] .map((e) => ({ ...e, variants: [...e.variants].join(", ") })) .sort((a, b) => b.count - a.count); const top = new Set(termTable.slice(0, maxNodes).map((d) => d.term)); const graph = new Graph({ type: "undirected" }); for (const t of termTable.slice(0, maxNodes)) graph.addNode(t.term, { count: t.count }); for (const e of wcObj.coocEdges) if (e.weight >= minEdgeWeight && top.has(e.a) && top.has(e.b)) graph.addEdge(e.a, e.b, { weight: e.weight }); for (const n of [...graph.nodes()]) if (graph.degree(n) === 0) graph.dropNode(n); const g = () => { const m2 = new Map(); if (graph.order === 0) return m2; const communities = louvain(graph, { getEdgeWeight: "weight" }); const btw = betweennessCentrality(graph, { getEdgeWeight: null }); graph.forEachNode((n, attrs) => { let wdeg = 0; graph.forEachEdge(n, (e, ea) => { wdeg += ea.weight; }); m2.set(n, { count: attrs.count, degree: graph.degree(n), wdegree: wdeg, betweenness: btw[n], community: communities[n] }); }); return m2; }; const nodeInfo = g(); const betweenness = [...nodeInfo.entries()] .map(([term, info]) => ({ term, count: info.count, degree: info.degree, wdegree: info.wdegree, betweenness: info.betweenness, community: info.community })) .sort((a, b) => b.betweenness - a.betweenness); return { graph, nodeInfo, betweenness }; }37:T1451,FONT_OPTIONS = { const FAMILIES = [ // --- Sans (system, no loading needed) --- { name: "System UI", category: "Sans", css: `system-ui, -apple-system, "Segoe UI", sans-serif` }, { name: "Helvetica", category: "Sans", css: `"Helvetica Neue", Helvetica, Arial, sans-serif` }, { name: "Arial", category: "Sans", css: `Arial, Helvetica, sans-serif` }, { name: "Verdana", category: "Sans", css: `Verdana, Geneva, sans-serif` }, { name: "Tahoma", category: "Sans", css: `Tahoma, Verdana, sans-serif` }, { name: "Trebuchet", category: "Sans", css: `"Trebuchet MS", Tahoma, sans-serif` }, // --- Serif --- { name: "Georgia", category: "Serif", css: `Georgia, "Times New Roman", serif` }, { name: "Times", category: "Serif", css: `"Times New Roman", Times, serif` }, { name: "Palatino", category: "Serif", css: `Palatino, "Palatino Linotype", "Book Antiqua", serif` }, // --- Mono --- { name: "System Mono", category: "Mono", css: `ui-monospace, Menlo, Consolas, monospace` }, { name: "Courier", category: "Mono", css: `"Courier New", Courier, monospace` }, // --- Condensed / display --- { name: "Arial Narrow", category: "Condensed", css: `"Arial Narrow", Arial, sans-serif` }, { name: "Impact", category: "Display", css: `Impact, "Arial Black", sans-serif` }, // --- Google Fonts: require an explicit load before measuring (webfont: true) --- { name: "Inter", category: "Sans", css: `Inter, system-ui, sans-serif`, webfont: true }, { name: "Roboto", category: "Sans", css: `Roboto, system-ui, sans-serif`, webfont: true }, { name: "Open Sans", category: "Sans", css: `"Open Sans", system-ui, sans-serif`, webfont: true }, { name: "Lato", category: "Sans", css: `Lato, system-ui, sans-serif`, webfont: true }, { name: "Montserrat", category: "Sans", css: `Montserrat, system-ui, sans-serif`, webfont: true }, { name: "Poppins", category: "Sans", css: `Poppins, system-ui, sans-serif`, webfont: true }, { name: "Work Sans", category: "Sans", css: `"Work Sans", system-ui, sans-serif`, webfont: true }, { name: "DM Sans", category: "Sans", css: `"DM Sans", system-ui, sans-serif`, webfont: true }, { name: "Source Sans 3", category: "Sans", css: `"Source Sans 3", system-ui, sans-serif`, webfont: true }, { name: "Nunito", category: "Sans", css: `Nunito, system-ui, sans-serif`, webfont: true }, { name: "Playfair Display", category: "Serif", css: `"Playfair Display", Georgia, serif`, webfont: true }, { name: "Merriweather", category: "Serif", css: `Merriweather, Georgia, serif`, webfont: true }, { name: "Lora", category: "Serif", css: `Lora, Georgia, serif`, webfont: true }, { name: "EB Garamond", category: "Serif", css: `"EB Garamond", Georgia, serif`, webfont: true }, { name: "Libre Baskerville", category: "Serif", css: `"Libre Baskerville", Georgia, serif`, webfont: true }, { name: "Crimson Text", category: "Serif", css: `"Crimson Text", Georgia, serif`, webfont: true }, { name: "IBM Plex Sans", category: "Sans", css: `"IBM Plex Sans", system-ui, sans-serif`, webfont: true }, { name: "IBM Plex Serif", category: "Serif", css: `"IBM Plex Serif", Georgia, serif`, webfont: true }, { name: "IBM Plex Mono", category: "Mono", css: `"IBM Plex Mono", ui-monospace, monospace`, webfont: true }, { name: "JetBrains Mono", category: "Mono", css: `"JetBrains Mono", ui-monospace, monospace`, webfont: true }, { name: "Fira Code", category: "Mono", css: `"Fira Code", ui-monospace, monospace`, webfont: true }, { name: "Space Grotesk", category: "Sans", css: `"Space Grotesk", system-ui, sans-serif`, webfont: true } ]; const ctx = document.createElement("canvas").getContext("2d"); const probe = "mmmwwwiiilll@#WM"; const generics = ["monospace", "serif", "sans-serif"]; const baseline = generics.map((g) => { ctx.font = `72px ${g}`; return ctx.measureText(probe).width; }); const isAvailable = (family) => generics.some((g, i) => { ctx.font = `72px "${family}", ${g}`; return ctx.measureText(probe).width !== baseline[i]; }); const firstFamily = (css) => css .split(",")[0] .trim() .replace(/^["']|["']$/g, ""); const AVAILABLE = FAMILIES.filter( (f) => f.webfont || isAvailable(firstFamily(f.css)) ); return AVAILABLE; }38:[] 31:{"id":"93725fc01e31937b","slug":"visualizing-perceptions-in-text","title":"Visualizing ideas you perceive in text","thumbnail":"1332dff93f28bb4085b91ed9c3c0b45dd5643b831ba179548e76207a615e2801","access_level":"public","create_time":"$D2026-08-20T19:22:20.691Z","update_time":"$D2026-09-06T10:54:54.569Z","view_time":null,"first_public_time":"$D2026-08-25T02:17:48.086Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"38ebb3822b8f9dbc","login":"ltostes","name":"Lucas Tostes","avatar_url":"https://avatars-next.observableusercontent.com/92c420e46700aada20009e3e35b3e3f0a37cede401b581cea97187885c8f3022","tier":"starter_2024"},"version":936,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"c62f659f7cb37ce9","login":"ltostes","name":"Lucas Tostes","avatar_url":"https://avatars-next.observableusercontent.com/92c420e46700aada20009e3e35b3e3f0a37cede401b581cea97187885c8f3022","tier":null}],"members":[{"type":"team","id":"38ebb3822b8f9dbc","login":"ltostes","name":"Lucas Tostes","avatar_url":"https://avatars-next.observableusercontent.com/92c420e46700aada20009e3e35b3e3f0a37cede401b581cea97187885c8f3022","tier":"starter_2024","role":"editor"},{"type":"individual","id":"c62f659f7cb37ce9","login":"ltostes","name":"Lucas Tostes","avatar_url":"https://avatars-next.observableusercontent.com/92c420e46700aada20009e3e35b3e3f0a37cede401b581cea97187885c8f3022","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Visualizing ideas you perceive in text","cells":[{"id":678,"value":"","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":807,"value":"","pinned":true,"hidden":false,"mode":"ts","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":362,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":563,"value":"While researching ways to analyze and visualize text as data, one idea kept making me uneasy: any analysis frames the reader's understanding. How can one analyze text without introducing bias?\n\nThis experiment flips that around and invites you to treat your bias as input: what have you perceived in the text you've read? The arcs reveal how often the words you chose appear together inside the text itself.\n\nPaste something you've read, write the sentence it inspired in you, and see your reading take shape. **Try it!** ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":216,"value":"viewof input_text = Inputs.textarea({\n label: \"Visualization phrase\",\n rows: 1,\n submit: true,\n placeholder: \"Enter your visualization text here\",\n value: \"Visualizing ideas you perceive in text\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"viewof analysis_text = Inputs.textarea({\n label: \"Reference text\",\n rows: 6,\n submit: true,\n placeholder: \"Paste any PT or EN text here, then hit Submit...\",\n value: example_text\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":463,"value":"viewof font_test = Inputs.select(\n d3.sort(FONT_OPTIONS, (d) => d.category),\n {\n label: \"Font\",\n format: (d) => `${d.category} - ${d.name}`,\n value: FONT_OPTIONS.find((d) => d.name === \"Georgia\")\n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":805,"value":"viewof showbetweenness = Inputs.toggle({\n label: \"Betweenness highlight\",\n value: false\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":78,"value":"## Text processing steps","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":180,"value":"### #1. Mapping stopwords\n\nStopwords are frequent words that don't individually bring distinct meaning to phrases, and for some analyses it is useful to have them aside.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":133,"value":"viewof langChoice = Inputs.radio([\"auto\", \"pt\", \"en\"], {\n label: \"Language\",\n value: \"auto\",\n submit: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":136,"value":"lang = {\n if (langChoice !== \"auto\") return langChoice;\n const words = (rawText.toLowerCase().match(/\\p{L}+/gu) ?? []).slice(0, 2000);\n const hits = (list) => {\n const s = new Set(list);\n return words.filter((w) => s.has(w)).length;\n };\n return hits(stopwords_pt) >= hits(stopwords_en) ? \"pt\" : \"en\";\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":144,"value":"viewof stopwordsText = {\n return Inputs.textarea({\n label: \"Stopword list (editable, one per line)\",\n rows: 8,\n submit: true,\n value: (lang == \"en\" ? stopwords_en : stopwords_pt).join(\"\\n\")\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":80,"value":"### #2. From characters to tokens\nSentence segmentation here is a regex: split after a `.`, `!`, `?`, or `…`\nwhen it is followed by whitespace and then a capital letter, quote, or open\nparenthesis. It is fast and dependency-free, but it fails on abbreviations\n(\"Dr. Silva\", \"etc.\") and any punctuation pattern that does not fit that\nshape - a known, accepted limitation for an experiment.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":230,"value":"`${tokenProcessing.sentences.length} sentences, ${tokenProcessing.allTokens.length} tokens, ${tokenProcessing.uniqueTokenCount} unique tokens`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":186,"value":"Inputs.table(\n tokenProcessing.sentences.map((s, i) => ({\n sentence: s,\n tokens: tokenProcessing.sentenceTokens[i].join(\",\")\n }))\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":87,"value":"viewof keepAccents = Inputs.toggle({\n label: \"Keep accents\",\n value: lang == \"pt\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":91,"value":"tokenProcessing = {\n const normalize = (w) => {\n w = w.toLowerCase();\n return keepAccents ? w : w.normalize(\"NFD\").replace(/\\p{M}/gu, \"\");\n };\n const sentences = rawText\n .split(/(?<=[.!?…])\\s+(?=[\\p{Lu}\"“(])/u)\n .map((s) => s.trim())\n .filter((s) => s.length > 1);\n const sentenceTokens = sentences.map((s) =>\n (s.match(/\\p{L}[\\p{L}\\p{M}'-]*/gu) ?? []).map((w) =>\n normalize(w).replace(/['-]+$/, \"\")\n )\n );\n const allTokens = sentenceTokens.flatMap((ts, sent) =>\n ts.map((word, i) => ({ word, sent, pos: i }))\n );\n const uniqueTokenCount = new Set(allTokens.map((t) => t.word)).size;\n return { normalize, sentences, sentenceTokens, allTokens, uniqueTokenCount };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":237,"value":"### #3. Stemming\nA stem is what is left after chopping off a word’s inflectional endings; it is not always a real word. A lemma is the dictionary form (“run” for “ran”), produced by proper morphological analysis. Stemming is cheaper and rougher: it strips suffixes with a fixed set of rules, so it sometimes over-cuts (“universe” and “university” can collide) and sometimes under-cuts.\n\nGrouping by stem instead of the raw token helps us make more meaningful connections.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":280,"value":"viewof useStems = Inputs.toggle({ label: \"Group by stem\", value: true })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":296,"value":"Inputs.table(stemObj.stemGroups)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":285,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":312,"value":"### #4. Word graph and co-ocurrence networks\nA co-occurrence network turns text into a graph: each surviving term (after stopword removal) becomes a node, and an edge between two terms records how many times they occur close to each other. Edge weight is just that count - the more often two words appear near each other, the stronger their connection.\n\n“Close” needs a precise definition, and here we propose a simple approach: linking any two terms that share a sentence. It favors semantic association but grows dense fast, since a sentence with n terms contributes n(n-1)/2 edges.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":326,"value":"Inputs.table(wcObj.coocEdges)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":786,"value":"The connection is shown below as a graph. Nodes are colored based on Louvain community detection: it looks for groups that are more densely connected to each other than to the rest of the graph. Here, each community is a rough proxy for a sub-topic or theme running through the text.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":734,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":775,"value":"### #5. Betweenness\nA word's betweenness value here is based on how often a node sits on the shortest path between other pairs of node in the graph.\n\nHigh-betweenness words bridge topics: they connect vocabulary that would otherwise stay in separate clusters, even when their raw frequency or degree is unremarkable. The table below lists the 20 highest-betweenness nodes with all their metrics for comparison.\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":778,"value":"Inputs.table(graphObj.betweenness)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":323,"value":"wcObj = {\n const counts = new Map();\n const add = (a, b) => {\n if (a === b) return;\n const k = a < b ? `${a}|${b}` : `${b}|${a}`;\n counts.set(k, (counts.get(k) ?? 0) + 1);\n };\n\n for (const ts of stemObj.contentSentences) {\n const uniq = [...new Set(ts)];\n for (let i = 0; i < uniq.length; i++)\n for (let j = i + 1; j < uniq.length; j++) add(uniq[i], uniq[j]);\n }\n\n const coocEdges = [...counts.entries()]\n .map(([k, weight]) => {\n const [a, b] = k.split(\"|\");\n return { a, b, weight };\n })\n .sort((x, y) => y.weight - x.weight);\n\n return { counts, coocEdges };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":712,"value":"viewof maxNodes = Inputs.range([10, 120], {\n label: \"Max nodes (by frequency)\",\n value: 50,\n step: 5\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":716,"value":"viewof minEdgeWeight = Inputs.range([1, 10], {\n label: \"Min edge weight\",\n value: 1,\n step: 1\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":719,"value":"$36","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":364,"value":"### Visual processing","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":475,"value":"font_family = font_test.css","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":434,"value":"font_size = {\n const family = font_family;\n const target = width * 0.85; // leave room for your arc gutters\n const [MIN, MAX] = [10, 48];\n const REF = 100;\n\n const natural = (px) =>\n pretext.measureNaturalWidth(\n pretext.prepareWithSegments(input_text, `${px}px ${family}`)\n );\n\n let size = (REF * target) / natural(REF);\n for (let i = 0; i < 2; i++) size *= target / natural(size); // converges fast\n\n size = Math.max(MIN, Math.min(MAX, Math.floor(size)));\n return size;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":375,"value":"text_links = wcObj.coocEdges.map((d) => ({\n source: d.a,\n target: d.b,\n count: d.weight\n}))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":381,"value":"normalize = (word) => {\n const word_nopunctuation = word.replace(\n /^[\\p{P}\\p{S}]+|[\\p{P}\\p{S}]+$/gu,\n \"\"\n );\n const word_lc = word_nopunctuation.toLowerCase();\n const pure_word = keepAccents\n ? word_lc\n : word_lc.normalize(\"NFD\").replace(/\\p{M}/gu, \"\");\n const word_stem = useStems ? stemmer.stem(pure_word) : pure_word;\n\n return word_stem;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":385,"value":"prp_obj = {\n const prp = pretext.prepareWithSegments(\n input_text,\n `${font_size}px ${font_family}`\n );\n\n const text_data = prp.segments\n .map((segment, i) => ({\n segment,\n w: prp.widths[i],\n cw: d3.cumsum(prp.widths)[i],\n kind: prp.kinds[i],\n i\n }))\n .map((d) => ({\n ...d,\n word_start: d.cw - d.w,\n word_end: d.cw\n }));\n\n return { prp, text_data };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":93,"value":"rawText = {\n return analysis_text ?? \"\";\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":539,"value":"example_text = `Visualizing a text often begins the moment you perceive how its ideas repeat and cluster. Each time you perceive two ideas sharing a sentence in the text, an invisible thread forms between them. What you perceive as mere repetition, the text records as structure. Visualizing those threads turns the text into a network where ideas become nodes. The heavier the thread, the more often the text pairs those ideas, and visualizing that weight is the whole point. By visualizing frequency as thickness, the graph lets the text argue for which ideas belong together. Visualizing a text this way is less about reading and more about mapping. In the end, the map shows the text not as a line of words but as a constellation of ideas.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":121,"value":"### Extra tooling","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":106,"value":"import { npm } from \"@pkg/npm\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":109,"value":"stopwords_en = (await npm(\"stopwords\")).english","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":168,"value":"stopwords_pt = (await npm(\"stopwords-pt\")).default","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":242,"value":"new_stemmer = await npm.run(\"snowball-stemmers\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":253,"value":"stemmer = new_stemmer.newStemmer(lang == \"pt\" ? \"portuguese\" : \"english\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":387,"value":"pretext = await npm(\"@chenglou/pretext\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":458,"value":"$37","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":701,"value":"Graph = (await npm(\"graphology\")).Graph","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":703,"value":"louvain = (await npm(\"graphology-communities-louvain\")).default","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":706,"value":"betweennessCentrality = (await npm(\"graphology-metrics/centrality/betweenness\"))\n .default","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W38","stdlib":"1","titling":"auto"}} 32:"$31:body"