40:null 41:["$","title",null,{"children":"Tool for Exploring Document Relatedness via Topical Analysis | Luke Snyder | Observable"}] 47:T512,# Preamble This tool enables the following: 1. Filtering by authors 2. Extraction of paper topics via LDA analysis on the abstracts (which can be interactively filtered) -- This analysis is intended for discovering topically-related papers 3. A t-SNE visualization of the provided Semantic Scholar citation embeddings While the embeddings use citation-informed transformers, I was curious how topical clusters might align with the embeddings (which obviously depends on a number of factors, such as the number of topics, t-SNE perplexity, etc.). Interestingly, certain papers are well-aligned, e.g., Fidyll and Idyll are in the same topic groups -- as are the reverse-engineering papers -- but not always as close in the t-SNE visualization (which again may depend on the model initialization and a variety of other factors). I would also be curious how these results extend to papers included from different areas (e.g., CS, Psychology, Economics, etc.) or on actual citation network graphs. NOTE: I did not anticipate having as much difficulty with getting the LDA model working in JS (which still isn't as informative as I would have hoped). As a result, I had to submit this assignment a day late. I, of course, accept any late penalty incurred and sincerely apologize for the inconvenience.48:Tb01,stopwords = Object({ "a": 1, "about": 1, "above": 1, "after": 1, "again": 1, "against": 1, "ain": 1, "all": 1, "am": 1, "an": 1, "and": 1, "any": 1, "are": 1, "aren": 1, "aren't": 1, "as": 1, "at": 1, "be": 1, "because": 1, "been": 1, "before": 1, "being": 1, "below": 1, "between": 1, "both": 1, "but": 1, "by": 1, "can": 1, "couldn": 1, "couldn't": 1, "d": 1, "did": 1, "didn": 1, "didn't": 1, "do": 1, "does": 1, "doesn": 1, "doesn't": 1, "doing": 1, "don": 1, "don't": 1, "down": 1, "during": 1, "each": 1, "few": 1, "for": 1, "from": 1, "further": 1, "had": 1, "hadn": 1, "hadn't": 1, "has": 1, "hasn": 1, "hasn't": 1, "have": 1, "haven": 1, "haven't": 1, "having": 1, "he": 1, "her": 1, "here": 1, "hers": 1, "herself": 1, "him": 1, "himself": 1, "his": 1, "how": 1, "i": 1, "if": 1, "in": 1, "into": 1, "is": 1, "isn": 1, "isn't": 1, "it": 1, "it's": 1, "its": 1, "itself": 1, "just": 1, "ll": 1, "m": 1, "ma": 1, "me": 1, "mightn": 1, "mightn't": 1, "more": 1, "most": 1, "mustn": 1, "mustn't": 1, "my": 1, "myself": 1, "needn": 1, "needn't": 1, "no": 1, "nor": 1, "not": 1, "now": 1, "o": 1, "of": 1, "off": 1, "on": 1, "once": 1, "only": 1, "or": 1, "other": 1, "our": 1, "ours": 1, "ourselves": 1, "out": 1, "over": 1, "own": 1, "re": 1, "s": 1, "same": 1, "shan": 1, "shan't": 1, "she": 1, "she's": 1, "should": 1, "should've": 1, "shouldn": 1, "shouldn't": 1, "so": 1, "some": 1, "such": 1, "t": 1, "than": 1, "that": 1, "that'll": 1, "the": 1, "their": 1, "theirs": 1, "them": 1, "themselves": 1, "then": 1, "there": 1, "these": 1, "they": 1, "this": 1, "those": 1, "through": 1, "to": 1, "too": 1, "under": 1, "until": 1, "up": 1, "ve": 1, "very": 1, "was": 1, "wasn": 1, "wasn't": 1, "we": 1, "were": 1, "weren": 1, "weren't": 1, "what": 1, "when": 1, "where": 1, "which": 1, "while": 1, "who": 1, "whom": 1, "why": 1, "will": 1, "with": 1, "won": 1, "won't": 1, "wouldn": 1, "wouldn't": 1, "y": 1, "you": 1, "you'd": 1, "you'll": 1, "you're": 1, "you've": 1, "your": 1, "yours": 1, "yourself": 1, "yourselves": 1, "could": 1, "he'd": 1, "he'll": 1, "he's": 1, "here's": 1, "how's": 1, "i'd": 1, "i'll": 1, "i'm": 1, "i've": 1, "let's": 1, "ought": 1, "she'd": 1, "she'll": 1, "that's": 1, "there's": 1, "they'd": 1, "they'll": 1, "they're": 1, "they've": 1, "we'd": 1, "we'll": 1, "we're": 1, "we've": 1, "what's": 1, "when's": 1, "where's": 1, "who's": 1, "why's": 1, "would": 1 })49:T3225,LDA = { /** * Based on implementation on https://github.com/mimno/jsLDA and https://github.com/primaryobjects/lda */ // const fs = require('fs'); // const path = require('path'); // const snowball = require('node-snowball'); class topicModelling { constructor(settings, sentences, dict) { this.settings = settings || {}; if (!this.settings.language) { this.settings.language = 'en'; } if (dict) { this.dict = dict; } if (!isNaN(this.settings.numberTopics) && this.settings.numberTopics > 0) { this.numTopics = this.settings.numberTopics; } else { this.numTopics = 10; } this.documentTopicSmoothing = 0.1; this.topicWordSmoothing = 0.01; this.docSortSmoothing = 10.0; this.sumDocSortSmoothing = this.docSortSmoothing * this.numTopics; this.completeSweeps = 0; this.reqiestedSweeps = 0; // vocabulary this.vocabularySize = 0; this.vocabularyCounts = {}; if (this.settings.stem !== undefined) { this.stem = this.settings.stem; } if (this.settings.displayingStopWords !== undefined) { this.displayingStopwords = settings.displayingStopWords; } //documents this.documents = []; this.wordTopicCounts = {}; this.topicWordCounts = []; this.topicScores = this.zeros(this.numTopics); this.tokensPerTopic = this.zeros(this.numTopics); this.topicWeights = this.zeros(this.numTopics); this.loadVocabulary(); this.prepareData(sentences); if (this.settings.sweeps !== undefined) { this.requestedSweeps = this.settings.sweeps; } else { this.requestedSweeps = 500; } while (this.completeSweeps <= this.requestedSweeps) { this.sweep(); } } loadVocabulary() { // let stopwordsFileName = `./languages/${this.settings.language}/stopwords_${this.settings.language}.json`; // try { // // eslint-disable-next-line // this.stopwords = JSON.parse( // fs.readFileSync(path.resolve(__dirname, stopwordsFileName)) // ); // } catch (ex) { // this.stopwords = {}; // } this.stopwords = stopwords; if (this.dict !== undefined) { this.addStopWords(); } } prepareData(documents) { if (!documents || documents.length < 0) { return; } let wordPattern = XRegExp('\\p{L}[\\p{L}\\p{P}]*\\p{L}', 'g'); for (let item of documents) { if (item.text == '') { continue; } let sentence = Array.isArray(item.text) ? item.text : item.text.toLowerCase().match(wordPattern); let docID = item.id; let tokens = []; let topicCounts = this.zeros(this.numTopics); if (sentence == null) { continue; } sentence.forEach(word => { if (word !== '') { let topic = Math.floor(Math.random() * this.numTopics); if (word.length <= 2) { this.stopwords[word] = 1; } let isStopword = this.stopwords[word]; if (this.stem && !isStopword) { // word = snowball.stemword(word, this.settings.language); word = stemmer(word); isStopword = this.stopwords[word]; } if (isStopword) { // Record counts for stopwords, but nothing else if (!this.vocabularyCounts[word]) { this.vocabularyCounts[word] = 1; } else { this.vocabularyCounts[word] += 1; } } else { this.tokensPerTopic[topic]++; if (!this.wordTopicCounts[word]) { this.wordTopicCounts[word] = {}; this.vocabularySize++; this.vocabularyCounts[word] = 0; } if (!this.wordTopicCounts[word][topic]) { this.wordTopicCounts[word][topic] = 0; } this.wordTopicCounts[word][topic] += 1; this.vocabularyCounts[word] += 1; topicCounts[topic] += 1; } tokens.push({ word: word, topic: topic, isStopword: isStopword }); } }); this.documents.push({ originalOrder: documents.length, id: docID, originalText: item.text, tokens: tokens, topicCounts: topicCounts }); } } sweep() { let topicNormalizers = this.zeros(this.numTopics); for (let topic = 0; topic < this.numTopics; topic++) { topicNormalizers[topic] = 1.0 / (this.vocabularySize * this.topicWordSmoothing + this.tokensPerTopic[topic]); } for (let doc = 0; doc < this.documents.length; doc++) { let currentDoc = this.documents[doc]; let docTopicCounts = currentDoc.topicCounts; for (let position = 0; position < currentDoc.tokens.length; position++) { let token = currentDoc.tokens[position]; if (token.isStopword) { continue; } this.tokensPerTopic[token.topic]--; let currentWordTopicCounts = this.wordTopicCounts[token.word]; currentWordTopicCounts[token.topic]--; if (currentWordTopicCounts[token.topic] == 0) { //delete(currentWordTopicCounts[ token.topic ]); } docTopicCounts[token.topic]--; topicNormalizers[token.topic] = 1.0 / (this.vocabularySize * this.topicWordSmoothing + this.tokensPerTopic[token.topic]); let sum = 0.0; for (let topic = 0; topic < this.numTopics; topic++) { if (currentWordTopicCounts[topic]) { this.topicWeights[topic] = (this.documentTopicSmoothing + docTopicCounts[topic]) * (this.topicWordSmoothing + currentWordTopicCounts[topic]) * topicNormalizers[topic]; } else { this.topicWeights[topic] = (this.documentTopicSmoothing + docTopicCounts[topic]) * this.topicWordSmoothing * topicNormalizers[topic]; } sum += this.topicWeights[topic]; } // Sample from an unnormalized discrete distribution let sample = sum * Math.random(); let i = 0; sample -= this.topicWeights[i]; while (sample > 0.0) { i++; sample -= this.topicWeights[i]; } token.topic = i; this.tokensPerTopic[token.topic]++; if (!currentWordTopicCounts[token.topic]) { currentWordTopicCounts[token.topic] = 1; } else { currentWordTopicCounts[token.topic] += 1; } docTopicCounts[token.topic]++; topicNormalizers[token.topic] = 1.0 / (this.vocabularySize * this.topicWordSmoothing + this.tokensPerTopic[token.topic]); } } //console.log("sweep in " + (Date.now() - startTime) + " ms"); this.completeSweeps += 1; if (this.completeSweeps >= this.requestedSweeps) { this.sortTopicWords(); } } byCountDescending(a, b) { return b.count - a.count; } topNWords(wordCounts, n) { return wordCounts .slice(0, n) .map(d => { return d.word; }) .join(' '); } sortTopicWords() { this.topicWordCounts = []; for (let topic = 0; topic < this.numTopics; topic++) { this.topicWordCounts[topic] = []; } for (let word in this.wordTopicCounts) { for (let topic in this.wordTopicCounts[word]) { this.topicWordCounts[topic].push({ word: word, count: this.wordTopicCounts[word][topic] }); } } for (let topic = 0; topic < this.numTopics; topic++) { this.topicWordCounts[topic].sort(this.byCountDescending); } } getTopicWords() { let topicTopWords = []; for (let topic = 0; topic < this.numTopics; topic++) { topicTopWords.push(this.topNWords(this.topicWordCounts[topic], 10)); } this.calcDominantTopic(); let topicData = topicTopWords.map((words, index) => { return { id: index, topicText: words, score: this.topicScores[index] }; }); return topicData; } calcDominantTopic() { this.documents.map((doc, i) => { let topic = -1; let score = -1; for ( let selectedTopic = 0; selectedTopic < this.numTopics; selectedTopic++ ) { let tempScore = (doc.topicCounts[selectedTopic] + this.docSortSmoothing) / (doc.tokens.length + this.sumDocSortSmoothing); if (tempScore >= score) { score = tempScore; topic = selectedTopic; } } this.topicScores[topic] += 1; }); this.topicScores = this.topicScores.map(val => val / this.documents.length); } getDocuments() { let sentences = []; for ( let selectedTopic = 0; selectedTopic < this.numTopics; selectedTopic++ ) { let documentVocab = this.getVocab(selectedTopic, true); let scores = this.documents.map((doc, i) => { return { docID: i, score: (doc.topicCounts[selectedTopic] + this.docSortSmoothing) / (doc.tokens.length + this.sumDocSortSmoothing) }; }); scores.sort((a, b) => { return b.score - a.score; }); let docinfo = []; for (let val of scores) { if (this.documents[val.docID].topicCounts[selectedTopic] > 0) { docinfo.push({ id: this.documents[val.docID].id, text: this.documents[val.docID].originalText, score: val.score }); } } sentences.push({ topic: selectedTopic, documents: docinfo, documentVocab }); } return sentences; } // // Vocabulary // mostFrequentWords(includeStops, sortByTopic, selectedTopic) { // Convert the random-access map to a list of word:count pairs that // we can then sort. let wordCounts = []; if (sortByTopic) { for (let word in this.vocabularyCounts) { if ( this.wordTopicCounts[word] && this.wordTopicCounts[word][selectedTopic] ) { wordCounts.push({ word: word, count: this.wordTopicCounts[word][selectedTopic] }); } } } else { for (let word in this.vocabularyCounts) { if (includeStops || !this.stopwords[word]) { wordCounts.push({ word: word, count: this.vocabularyCounts[word] }); } } } wordCounts.sort(this.byCountDescending); return wordCounts; } entropy(counts) { counts = counts.filter(function(x) { return x > 0.0; }); let sum = this.sum(counts); return ( Math.log(sum) - (1.0 / sum) * this.sum(counts.map(x => x * Math.log(x))) ); } specificity(word) { if (this.wordTopicCounts[word] == undefined) { return 0; } return ( 1.0 - this.entropy(Object.values(this.wordTopicCounts[word])) / Math.log(this.numTopics) ); } getVocab(selectedTopic, sortVocabByTopic) { let vocab = []; let wordFrequencies = this.mostFrequentWords( this.displayingStopwords, sortVocabByTopic, selectedTopic ).slice(0, 499); wordFrequencies.forEach(d => { let isStopword = this.stopwords[d.word]; let score = this.specificity(d.word); vocab.push({ word: d.word, count: d.count, stopword: isStopword, specificity: score }); }); return vocab; } truncate(s) { return s.length > 300 ? s.substring(0, 299) + '...' : s; } zeros(n) { var x = new Array(n); for (var i = 0; i < n; i++) { x[i] = 0.0; } return x; } sum(arr) { return arr.reduce((sum, currentValue) => { return sum + currentValue; }); } addStopWords() { if (this.dict) { Object.keys(this.dict).forEach(key => { this.stopwords[key] = this.dict[key]; }); } } } return topicModelling; }4a:[] 45:{"id":"ff4ff96fcb1ffb3a","slug":null,"title":"Tool for Exploring Document Relatedness via Topical Analysis","thumbnail":"ef3c1da02b8f56ff04f1350a622138e9cec0db96dbcffc0c48a163707a71e571","access_level":"public","create_time":"$D2022-10-27T23:44:12.529Z","update_time":"$D2024-02-21T21:43:55.917Z","view_time":null,"first_public_time":"$D2022-10-30T05:51:37.339Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"cc00293cc3788b65","login":"luke-s-snyder","name":"Luke Snyder","avatar_url":"https://avatars-next.observableusercontent.com/8fe1767ce8d041d91556f9ab13bd94bdda7e2a1227888705b4a2c3c0153198a4","tier":"starter_2024","bio":"","following":false},"version":752,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"4b69da7dd934c9a5","login":"luke-s-snyder","name":"Luke Snyder","avatar_url":"https://avatars-next.observableusercontent.com/8fe1767ce8d041d91556f9ab13bd94bdda7e2a1227888705b4a2c3c0153198a4","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"4b69da7dd934c9a5","login":"luke-s-snyder","name":"Luke Snyder","avatar_url":"https://avatars-next.observableusercontent.com/8fe1767ce8d041d91556f9ab13bd94bdda7e2a1227888705b4a2c3c0153198a4","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Tool for Exploring Document Relatedness via Topical Analysis","cells":[{"id":0,"value":"# Tool for Exploring Document Relatedness via Topical Analysis","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":641,"value":"$47","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":402,"value":"## Author List","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":89,"value":"viewof selectedAuthors = Inputs.select(['----'].concat([...new Set(data.map(d => d.Authors.split(',')).flat())].sort()), { multiple: true })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":404,"value":"## Topics","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":406,"value":"viewof numTopics = Inputs.range([1, 10], {value: 5, step: 1, label: \"Number of Topics\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":388,"value":"viewof selectedTopics = Inputs.table(topics, { header: { id: 'Topic #', topicText: 'Topic Words (stemmed)', score: 'Score' }, align: { id: 'left' }, width: { id: 200, topicText: 600 }, format: { id: (x, i) => htl.html`

${x}` }, required: false })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":412,"value":"## Paper List","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":705,"value":"NOTE: Papers without an abstract in the metadata are not included in topical analysis.","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":17,"value":"viewof rows = {\n var _data = (!selectedAuthors.length || selectedAuthors[0] === '----') \n ? data \n : data.filter(d => selectedAuthors.filter(s => d.Authors.includes(s)).length);\n\n if (TOPICS.length) {\n _data = _data.filter(d => TOPICS.includes(documentTopics[d.ID].topic));\n }\n \n return Inputs.table(_data, { columns: ['Title', 'Authors', 'Year', 'Venue', 'Citations'], width: { Title: 450, Authors: 200 }, format: { Title: x => \n htl.html`

d.Title).indexOf(x)].ID].topic)).copy({opacity:0.4})}\">${x}` \n }, \n required: false}) \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":437,"value":"{\n const svg = d3.create('svg')\n .attr('width', width)\n .attr('height', 500);\n const X = d3\n .scaleLinear()\n .domain(d3.extent(solution, d => d[0]))\n .range([20, width - 20]);\n const Y = d3\n .scaleLinear()\n .domain(d3.extent(solution, d => d[1]))\n .range([20, 500 - 20]);\n\n svg.selectAll('.circle')\n .data(solution)\n .enter()\n .append('circle')\n .attr('cx', d => X(d[0]))\n .attr('cy', d => Y(d[1]))\n .attr('r', 15)\n .style('stroke-width', 4)\n .style('stroke', (_, i) => rows.map(d => d.ID).includes(data[i].ID) ? 'black' : 'none' )\n .style('fill', (_, i) => TOPICS.includes(documentTopics[data[i].ID].topic) ? colors(documentTopics[data[i].ID].topic) : 'gray')\n return svg.node();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":563,"value":"colors = d3.scaleOrdinal(d3.schemeCategory10)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":708,"value":"colorFn = (i) => i === -1 ? '#ffffff' : colors(i)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":554,"value":"TOPICS = selectedTopics.map(d => d.id)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":427,"value":"tsnejs = require(\"https://raw.githack.com/karpathy/tsnejs/master/tsne.js\").then(\n null,\n _ => window.tsnejs\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":423,"value":"model = new tsnejs.tSNE({\n dim: 2,\n perplexity: 10\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":433,"value":"solution = {\n model.initDataDist(data.map(d => d.Vector));\n var cost = 100,\n cost0 = 0;\n while (Math.abs(cost - cost0) > 1e-6) {\n cost = cost0;\n cost0 = cost * 0.9 + 0.1 * model.step();\n // yield [cost, cost0];\n yield model.getSolution();\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"response = fetch('https://uwdata.github.io/future-scholarly-communication/22au/ex2/cse599d-paper-data.json')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":11,"value":"data = (await response.json()).map(function(data) {\n var id = data.doi;\n \n if (data.s2id) {\n data = data.s2data;\n var title = data.title;\n var authors = data.authors.map(d => d.name).join(', ');\n var year = data.year;\n var venue = data.venue;\n var citations = data.citationCount;\n var abstract = data.abstract;\n var vector = data.embedding.vector;\n } else {\n data = data.csl;\n var title = data.title;\n var authors = data.author[0].literal;\n var year = data.created ? data.created['date-parts'][0] : data.issued['date-parts'][0][0];\n var venue = data['container-title'] ? data['container-title'] : data.publisher;\n var citations = null;\n var abstract = data.abstract;\n var vector = new Array(784).fill(0);\n }\n\n return {\n Title: title,\n Authors: authors,\n Year: String(year),\n Venue: venue,\n Citations: citations,\n Abstract: abstract,\n Vector: vector,\n ID: id\n };\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":168,"value":"$48","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":214,"value":"XRegExp = import('https://cdn.skypack.dev/xregexp@5.1.1?min').then(d => d.default)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":279,"value":"stemmer = (await require('https://bundle.run/en-stemmer@1.0.3')).stemmer","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":158,"value":"$49","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":368,"value":"lda = {\n const document = data.map(function(d) {\n return {\n id: d.ID,\n text: d.Abstract\n };\n }).filter(d => d.text);\n\n const options = {\n displayingStopwords: false,\n language: 'en',\n numberTopics: numTopics,\n sweeps: 10000,\n stem: true,\n };\n\n return new LDA(options, document, {});\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":376,"value":"topics = lda.getTopicWords()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":378,"value":"documentTopics = {\n const ids = {};\n data.forEach(d => ids[d.ID] = {topic: -1, score: 0});\n lda.getDocuments().forEach(function({topic, documents}) {\n documents.forEach(function(document) {\n const _data = ids[document.id];\n if (document.score > _data.score) {\n _data.topic = topic;\n _data.score = document.score;\n }\n });\n });\n return ids;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W4a","stdlib":"1"}} 46:"$45:body"