31:T678,function wordFrequency(paragraph, stopwords, maxWords) { d3.select(".results").remove(); // Takes the paragraph and returns an array of words const wordArray = paragraph // replace everything that's not a letter, number, or space with a null .replace(/[^A-Z 0-9]/ig, "") // convert the paragraph to lower case .toLowerCase() // split on spaces to return an array of words .split(" ") const failed = []; const passed = []; // Takes the array of words and returns the same array with stopwords filtered out const noStopwords = wordArray // Map through each word. If the word is included in the list of stopwords, replace it with a null .map(word => { if(stopwords.includes(word)) { paragraph.replace(word, null) failed.push(word) } else { passed.push(word) return word } }) // Filter the array of words to remove nulls and blank cells .filter(word => word != null) .filter(word => word != "") // Takes the word array and returns an object with a unique list of words and their frequencies const frequency = noStopwords.reduce((prev, curr) => (prev[curr] = ++prev[curr] || 1, prev), {}) // Sorts the frequency object into an array and slices for the bottom and top indices provided const sortedFreqArr = Object.entries(frequency) .sort((a, b) => b[1] - a[1]) .slice(0, maxWords) // Turns the sorted array back into an object const freqObj = Object.fromEntries([...sortedFreqArr]) return {freqObj, failed, passed} }32:T4fc,stopwords = ["a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any","are","arent","as","at","be","because","been","before","being","below","between","both","but","by","cant","cannot","could","couldnt","did","didnt","do","does","doesnt","doing","dont","down","during","each","few","for","from","further","had","hadnt","has","hasnt","have","havent","having","he","hed","hell","hes","her","here","heres","hers","herself","him","himself","his","how","hows","i","id","ill","im","ive","if","in","into","is","isnt","it","its","its","itself","lets","me","more","most","mustnt","my","myself","no","nor","not","of","off","on","once","only","or","other","ought","our","ours","ourselves","out","over","own","same","shant","she","shed","shell","shes","should","shouldnt","so","some","such","than","that","thats","the","their","theirs","them","themselves","then","there","theres","these","they","theyd","theyll","theyre","theyve","this","those","through","to","too","under","until","up","very","was","wasnt","we","wed","well","were","weve","were","werent","what","whats","when","whens","where","wheres","which","while","who","whos","whom","why","whys","with","wont","would","wouldnt","you","youd","youll","youre","youve","your","yours","yourself","yourselves"];33:[{"id":"48eb75b68a818efd89a2213980adfd4f2ff84ac8b82be9383eb503c31b33d9c70858020a70be51430d338c6b2392d0325981de7910c86676bc50da1a5c01978c","href":"https://static.observableusercontent.com/files/48eb75b68a818efd89a2213980adfd4f2ff84ac8b82be9383eb503c31b33d9c70858020a70be51430d338c6b2392d0325981de7910c86676bc50da1a5c01978c","name":"media@3.csv","type":"text/csv","lastModified":1601848931643.013,"size":151079}] 2f:{"title":"Exploring Military / Hollywood Collaboration Data","cells":[{"id":0,"value":"md`# Exploring Military / Hollywood Collaboration Data\n#### Interactive Data Visualization Studio ⚡ Observable Prototype #3\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":13,"value":"md`\nThis week, I looked at a few different ways of exploring text in the Pentagon / Hollywood Collaboration database, a record of **~500 films** that requested assistance from the military. Each film in the database includes **remarks** from the Pentagon about why the film was approved or denied.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":428,"value":"md` `","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"md `## Explore the data by topic and film`","pinned":false,"hidden":false,"mode":"ojs"},{"id":645,"value":"md`Explore the films in the dataset by a **topic / keywords** contained in the remarks. Select a keyword and a film to see its approval status and full remarks. \n\nIn this section, I am also testing a script to transform a paragraph of uppercase text to sentence case -- it doesn't always succeed, but I'm working on it!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":545,"value":"viewof topic = select({\n title: \"Filter By Topic / Keyword\",\n options: topics,\n value: \"ALL\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":433,"value":"viewof title = select({\n title: \"Select a Film\",\n options: Object.keys(getFrequency(\"Title\", getDataByTopic(topic, data))).sort(),\n value: \"TOP GUN\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":657,"value":"md`**Add a topic (in all uppercase) to the \"topics\" array below to search your own!**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":543,"value":"topics = [\"ALL\", \"ALIEN\", \"VIETNAM\", \"WORLD WAR I\", \"WORLD WAR II\", \"COMEDY\", \"NEGATIVE\", \"POSITIVE\"]","pinned":false,"hidden":false,"mode":"ojs"},{"id":443,"value":"html`\n

\n
\n

\n
\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":445,"value":"function getDescription(title, data) {\n const filtered = data.filter(d => d.Title === title);\n const remarksRaw = filtered[0][\"Remarks\"]\n const remarks = paraToSentenceCase(remarksRaw, uppercaseWords);\n const status = filtered[0][\"Status\"]\n let colorStatus;\n if (status === \"APPROVED\") {\n colorStatus = \"lightgreen\"\n } else if (status === \"DENIED\") {\n colorStatus = \"#ffcccb\"\n } else colorStatus = \"white\"\n \n const year = filtered[0][\"Year\"]\n d3.select('.description').text(`REMARKS: ${remarks}`)\n d3.select('.title').text(`${title} | ${year}`)\n d3.select('.status').text(`APPROVAL STATUS: ${status}`)\n d3.select('.status').style(\"background-color\", colorStatus)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":449,"value":"getDescription(title, data)","pinned":false,"hidden":false,"mode":"ojs"},{"id":484,"value":"function sentenceCase(str, stayUppercase) {\n const lower = str.toLowerCase()\n let replaced = lower;\n stayUppercase.map(word => {\n replaced = replaced.replace(word.toLowerCase(), word);\n })\n const split = replaced.split(' ')\n const mapped = split.map((word, index) => {\n if (index === 0){\n return word.replace(word[0], word[0].toUpperCase());\n } else return word;\n })\n return mapped.join(' ');\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":486,"value":"function paraToSentenceCase(str, stayUppercase) {\n const sentences = str.split(\".\").filter(d => d != \"\")\n const cased = sentences.map(sentence => {\n return sentenceCase(sentence.trim(), stayUppercase);\n }).join(\". \")\n return cased;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":552,"value":"function getDataByTopic(topic, data) {\n if (topic === \"ALL\") {\n return data\n } else {\n const filtered = data.filter(d => d[\"Remarks\"].includes(topic))\n return filtered\n } \n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":422,"value":"md` `","pinned":false,"hidden":false,"mode":"ojs"},{"id":23,"value":"md `## Explore the data by word frequency`","pinned":false,"hidden":false,"mode":"ojs"},{"id":330,"value":"md`Explore the **most frequent non-stopwords** that show up in the dataset by **approval status**. \n\nAlong with a list of regular English stopwords, I included additional stopwords (listed in the Data & Reference section below) to filter out generic references to branches of the military, films, and filming.\n\nIt's interesting to note that the remarks for films that were **approved** have words like *war*, *cooperation*, *positive*, and *submarine*. The remarks for films that were **denied** include words like *script*, *vietnam*, *inaccurate*, and *negative*.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":344,"value":"viewof maxWords = slider({\n min: 2,\n max: 20,\n value: 12,\n step: 1,\n title: \"How many words do you want to see?\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":354,"value":"viewof approvalStatus = select({\n title: \"Approval Status\",\n description: \"Filter for an approval status.\",\n options: Object.keys(getFrequency(\"Status\", data)).sort(),\n value: \"APP\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":685,"value":"md`**Most frequent ${maxWords} words (and number of instances) in the remarks for films with the approval status: ${approvalStatus}**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":669,"value":"html`\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":679,"value":"getList();","pinned":false,"hidden":false,"mode":"ojs"},{"id":301,"value":"function getList() {\n d3.selectAll(\".results\").remove();\n const listObj = wordFrequency(allRemarks, allStopwords, maxWords).freqObj\n Object.entries(listObj).forEach(([key, value]) => {\n\td3.select(\".list\")\n .append(\"div\")\n .classed(\"results\", true)\n .text(` - ${key}: ${value}`)\n});\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":293,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":297,"value":"allRemarks = getColData(data.filter(d => d[\"Status\"] === approvalStatus), \"Remarks\", true).join(\" \")","pinned":false,"hidden":false,"mode":"ojs"},{"id":310,"value":"allStopwords = [...stopwords, ...moreStopwords]","pinned":false,"hidden":false,"mode":"ojs"},{"id":276,"value":"md` `","pinned":false,"hidden":false,"mode":"ojs"},{"id":195,"value":"md`## Helper Scripts`","pinned":false,"hidden":false,"mode":"ojs"},{"id":103,"value":"// Returns an array of values for a given column name\n// Flag for unique values only (true) or all non-null values (false)\nfunction getColData(obj, colName, uniqueFlag) {\n const col = obj.map((row, index) => {\n return Object.values(obj)[index][colName];\n }).filter((row) => row != null)\n if (uniqueFlag) {\n const set = new Set(col);\n return Array.from(set);\n } else {\n return col;\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":138,"value":"function getFrequency(fieldName, data) {\n const arr = getColData(data, fieldName, false);\n const frequency = arr.reduce((prev, curr) => (prev[curr] = ++prev[curr] || 1, prev), {})\n return frequency;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":172,"value":"function getFrequencyByCol(targetName, sortingName, data) {\n const targets = getColData(data, targetName, true);\n const obj = {}\n const map = targets.map((value, index) => {\n const filtered = data.filter(d => d[targetName] === value);\n const frequency = getFrequency(sortingName, filtered)\n return obj[value] = frequency\n })\n return obj\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":281,"value":"md` `","pinned":false,"hidden":false,"mode":"ojs"},{"id":36,"value":"md `## Data & Reference`","pinned":false,"hidden":false,"mode":"ojs"},{"id":39,"value":"data = d3.csvParse(await FileAttachment(\"media@3.csv\").text(), d3.autoType)","pinned":false,"hidden":false,"mode":"ojs"},{"id":230,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":307,"value":"moreStopwords = [\"film\", \"filming\", \"provided\", \"use\", \"full\", \"made\", \"ii\", \"also\", \"army\", \"navy\", \"air\", \"force\", \"military\", \"dod\", \"marine\", \"marines\"]","pinned":false,"hidden":false,"mode":"ojs"},{"id":236,"value":"uppercaseWords = [\"World War II\", \"Vietnam\", \"AWOL\", \"June\", \"Nov\", \"Navy\", \"Marine\", \"World War I\", \"Europe\", \"UFO\", \"Air Force\", \"Pearl Harbor\", \"Quaker\", \"Navy\", \"JFK\", \"P-\", \"Rock Hudson\", \"Wisconsin\", \"Air National Guard\", \"DOD\", \"Russian\", \"Youngstown\", \"German\", \"Hitler\", \"Stanley Kramer\", \"Hollywood\", \"Georgetown University Library\", \"SUID\", \"Marines\", \"Red October\", \"Kabul\", \"AFB\", \"West Point\"]","pinned":false,"hidden":false,"mode":"ojs"},{"id":284,"value":"md` `","pinned":false,"hidden":false,"mode":"ojs"},{"id":28,"value":"md `## Imports`","pinned":false,"hidden":false,"mode":"ojs"},{"id":31,"value":"d3 = require('d3@6', 'd3-random')","pinned":false,"hidden":false,"mode":"ojs"},{"id":342,"value":"import {slider, select} from \"@jashkenas/inputs\"","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"}