31:[{"id":"f115b8685c56139ad496779507330c5d30e1990999d206bf9103cf5a76e7b2c5b9ba1e29f0e325b093004b01c0c461dfd0fda24829a4183471ffc32ed3d17123","href":"https://static.observableusercontent.com/files/f115b8685c56139ad496779507330c5d30e1990999d206bf9103cf5a76e7b2c5b9ba1e29f0e325b093004b01c0c461dfd0fda24829a4183471ffc32ed3d17123","name":"tripadvisor_reviews_multi_downsampled_trans.parquet","type":"application/octet-stream","lastModified":1687357636704.119,"size":5198100}] 2f:{"title":"Traditional text analysis of complaints","cells":[{"id":0,"value":"# Traditional text analysis of complaints","pinned":false,"hidden":false,"mode":"md"},{"id":199,"value":"viewof toggle = Inputs.toggle({label: \"Remove stop words\", value: true})","pinned":false,"hidden":false,"mode":"ojs"},{"id":373,"value":"viewof select1 = Inputs.select(uniq_lang.map(d => d.lang), {label: \"Choose language\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":380,"value":"viewof select2 = Inputs.select([\"text\", \"text_clean\", \"translated_text\"], {label: \"Select text type\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":279,"value":"viewof select = Inputs.select(uniq_airline.map(d => d.airline), {label: \"Select Airline\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":203,"value":"viewof range = Inputs.range([0, 100], {label: \"Top N\", value: 20, step: 1})","pinned":false,"hidden":false,"mode":"ojs"},{"id":225,"value":"viewof maxCF = Inputs.range([0, 100], {label: \"max CFW\", value: 0, step: 1})","pinned":false,"hidden":false,"mode":"ojs"},{"id":359,"value":"// ADD CUSTOM STOPWORDS HERE\ncustomStopwords = ['airtransat', 'rouge', 'tel', 'aircanada', 'miami', 'francfort', 'frankfurt', 'lima', 'air', 'canada', 'flight', 'newark', 'aviv', 'ons', 'yeah', 'faudra']","pinned":true,"hidden":false,"mode":"ojs"},{"id":311,"value":"html`\n

${[plot_word_count()]}

${[plot_tf()]}

${[plot_tfidf()]}

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":409,"value":"- Left plot are top words ranked by count. If you keep stop words, note that it mostly becomes function words.\n- Middle plot is the overall text frequency (across airlines).\n- Right Plot are top words ranked by [tfidf](https://www.tidytextmining.com/tfidf.html#tfidf). See link for introduction. But basically he words that are mostly representative of the chosen airline relative to other airlines.\n- See [@danielbustillos/introduction-to-text-analysis-with-tf-idf](https://observablehq.com/@danielbustillos/introduction-to-text-analysis-with-tf-idf) for an introduction to text analysis on `Observablehq`. This is where I got the `CFW` thing.","pinned":false,"hidden":false,"mode":"md"},{"id":482,"value":"#### Cosine similarity","pinned":false,"hidden":false,"mode":"md"},{"id":491,"value":"{\n const mat = d3.merge(\n matrix.matrix.map((d, i) =>\n d.map((cos_sim, j) => ({ row: i, column: j, cos_sim })))\n )\n return Plot.plot({\n color: {legend: true, scheme: \"Blues\"},\n marginLeft: 300,\n height: 400,\n marginBottom: 75,\n marks: [\n Plot.axisY({label: \"\", tick: \"\"}),\n Plot.axisX({label: \"\", tick: \"\", tickRotate: 30}),\n Plot.cell(mat, {\n x: d => matrix.identifiers[d.row],\n y: d => matrix.identifiers[d.column],\n fill: \"cos_sim\",\n tip: true\n })\n ]\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":561,"value":"### raw data","pinned":false,"hidden":false,"mode":"md"},{"id":566,"value":"dat","pinned":true,"hidden":false,"mode":"ojs"},{"id":210,"value":"## Appendix","pinned":false,"hidden":false,"mode":"md"},{"id":461,"value":"matrix = similarity.getDistanceMatrix()","pinned":true,"hidden":false,"mode":"ojs"},{"id":451,"value":"similarity = new tfidf.Similarity(corpus)","pinned":true,"hidden":false,"mode":"ojs"},{"id":263,"value":"plot_tfidf = function() {\n const word_tfidf = corpus\n .getTopTermsForDocument(select)\n .map(d => ({\n Term: d[0],\n Weight: d[1]\n }))\n .sort((a, b) => d3.descending(a.Weight, b.Weight))\n .slice(0, range)\n \n return Plot.plot({\n y: {label: \"\"},\n x: {label: \"tfidf weight\"},\n width: width / 3, \n marginLeft: marginwidth,\n marks: [\n Plot.barX(word_tfidf, {x: \"Weight\", y: \"Term\", sort: {y: \"x\", reverse: true}}),\n Plot.ruleX([0]),\n Plot.frame()\n ]\n})\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":221,"value":"plot_tf = function() {\n const word_count = corpus\n .getTerms()\n .map(term => ({\n Term: term,\n CF: corpus.getCollectionFrequency(term),\n CFW: corpus.getCollectionFrequencyWeight(term)\n }))\n .filter(d => d.CF >= maxCF)\n .sort((a, b) => b.CF - a.CF).slice(0, range)\n \n return Plot.plot({\n y: {label: \"\"},\n width: width / 3,\n marginLeft: marginwidth,\n marks: [\n Plot.barX(word_count, {x: \"CFW\", y: \"Term\"}),\n Plot.ruleX([0]),\n Plot.frame()\n ]\n})\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":272,"value":"plot_word_count = function(){\n const document = corpus.getDocument(select)\n const word_count = document\n .getUniqueTerms()\n .filter(d => !corpus.getStopwords().includes(d))\n .map(t => ({\n Term: t,\n Frequency: document.getTermFrequency(t)\n }))\n .sort((a, b) => d3.descending(a.Frequency, b.Frequency))\n .slice(0, range)\n \n return Plot.plot({\n width: width / 3,\n marginLeft: marginwidth,\n y: {label: \"\"},\n marks: [\n Plot.barX(word_count, {x: \"Frequency\", y: \"Term\", sort: {y: \"x\", reverse: true}}),\n Plot.ruleX([0]),\n Plot.frame()\n ]\n})\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":443,"value":"#### Data wrangling","pinned":false,"hidden":false,"mode":"md"},{"id":41,"value":"dat = db.query(`SELECT GROUP_CONCAT(${select2}) as text, airline FROM data WHERE lang = '${select1}' GROUP BY airline`)","pinned":true,"hidden":false,"mode":"ojs"},{"id":56,"value":"airlines = dat.map(d => d.airline)","pinned":true,"hidden":false,"mode":"ojs"},{"id":194,"value":"complaints = dat.map(d => d.text)","pinned":true,"hidden":false,"mode":"ojs"},{"id":445,"value":"#### Data processing","pinned":false,"hidden":false,"mode":"md"},{"id":54,"value":"corpus = new tfidf.Corpus(\n airlines,\n complaints,\n toggle,\n customStopwords,\n K1,\n b\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":425,"value":"marginwidth = 80","pinned":true,"hidden":false,"mode":"ojs"},{"id":66,"value":"K1 = 2.0","pinned":true,"hidden":false,"mode":"ojs"},{"id":68,"value":"b = 0.75","pinned":true,"hidden":false,"mode":"ojs"},{"id":24,"value":"#### MISC","pinned":false,"hidden":false,"mode":"md"},{"id":388,"value":"uniq_lang = db.query(`SELECT DISTINCT(lang) as lang from data`)","pinned":true,"hidden":false,"mode":"ojs"},{"id":394,"value":"uniq_airline = db.query(`SELECT DISTINCT(airline) as airline from data`)","pinned":true,"hidden":false,"mode":"ojs"},{"id":43,"value":"db = DuckDBClient.of({ data: FileAttachment(\"tripadvisor_reviews_multi_downsampled_trans.parquet\") })","pinned":true,"hidden":false,"mode":"ojs"},{"id":4,"value":"tfidf = import(\"tiny-tfidf\")","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}