2d:null 2e:["$","title",null,{"children":"Analyzing popular K-pop song lyrics | Grégoire Geis | Observable"}] 33:T454,{ const pageSize = 10 const ranges = range === 'top5' ? Array.from({ length: 5 }, (_, i) => [i + 1, i + 1]) : range === 'top10' ? Array.from({ length: 10 }, (_, i) => [i + 1, i + 1]) : range === 'top20' ? Array.from({ length: 10 }, (_, i) => [i * 2 + 1, i * 2 + 2]) : Array.from({ length: 10 }, (_, i) => [i * 10 + 1, i * 10 + 10]) const categories = ranges.map(range => range[0] === range[1] ? `${range[0] + offset}` : `${range[0] + offset}-${range[1] + offset}`) const columns = Object.values(words).slice(offset, offset + pageSize).map(word => { const ranks = [word.word] for (const [start, end] of ranges) { let sum = 0 for (let i = start; i <= end; i++) { if (word.ranks[i]) sum += word.ranks[i].songs.size } ranks.push(sum) } return ranks }) return c3({ data: { columns: [ ...columns, ], }, axis: { x: { type: 'category', categories, }, }, }, this) }34:[{"id":"05c5ca53556e7f38b9e9358e46e078a146e9e6760acb1abae027b0f447d2164f7f3bad319f872496f4f3a4e957bdb7805bcee6c96b764c6f44acfc001e6c9fb7","href":"https://static.observableusercontent.com/files/05c5ca53556e7f38b9e9358e46e078a146e9e6760acb1abae027b0f447d2164f7f3bad319f872496f4f3a4e957bdb7805bcee6c96b764c6f44acfc001e6c9fb7","name":"kpop.json@2.gz","type":"application/gzip","lastModified":1582555315157.641,"size":2645396},{"id":"5cf178e1627fc33c5c3147078aaf5bd5fc74eaabfe23213742094e1ad15fccacf021b6eeafcb3e074029984a5f09b6d1228cb064a0573b01ae10ebf57b070864","href":"https://static.observableusercontent.com/files/5cf178e1627fc33c5c3147078aaf5bd5fc74eaabfe23213742094e1ad15fccacf021b6eeafcb3e074029984a5f09b6d1228cb064a0573b01ae10ebf57b070864","name":"kpop-notebook.ipynb","type":"application/octet-stream","lastModified":1582558082747.937,"size":7394}] 31:{"id":"0c27e2d038ff440b","slug":"analyzing-popular-k-pop-song-lyrics","title":"Analyzing popular K-pop song lyrics","thumbnail":"c7dbe141654494217ff74a2f2ac926ccc85fc2aafe5b7b95b1a034049c7e5cfc","access_level":"public","create_time":"$D2019-07-13T12:02:39.454Z","update_time":"$D2022-11-23T18:22:11.412Z","view_time":null,"first_public_time":"$D2019-10-07T22:52:46.435Z","has_unpublished_edits":null,"fork":false,"stars":5,"starred":false,"owner":{"type":"team","id":"6817f8df4b71afcf","login":"71","name":"Grégoire Geis","avatar_url":"https://avatars-next.observableusercontent.com/046456a6fcb599cae062b5a172a9e2c85acef403c376bdb60a4cc6d135a60aad","tier":"starter_2024"},"version":843,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"3881cfc7f3117323","login":"71","name":"Grégoire Geis","avatar_url":"https://avatars-next.observableusercontent.com/046456a6fcb599cae062b5a172a9e2c85acef403c376bdb60a4cc6d135a60aad","tier":null}],"members":[{"type":"individual","id":"3881cfc7f3117323","login":"71","name":"Grégoire Geis","avatar_url":"https://avatars-next.observableusercontent.com/046456a6fcb599cae062b5a172a9e2c85acef403c376bdb60a4cc6d135a60aad","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":{"id":"77c5dc3ca20e10fc","slug":"articles","title":"Articles","owner":{"id":"6817f8df4b71afcf","login":"71","name":"Grégoire Geis"}},"type":"notebook","body":{"title":"Analyzing popular K-pop song lyrics","cells":[{"id":0,"value":"md`# Analyzing popular K-pop song lyrics\n\nDuring my exchange semester in [Sejong University](http://eng.sejong.ac.kr), my\n_Probability and Statistics Programming_ class was given the following group\nassignement:\n\n> Find a claim or hypothesis made online, and either accept it or reject it\n> after computing its Z-value.\n\nAfter a few hours of brainstorming, my group finally decided to analyze K-pop\ndata for the assignement, since it is now a global phenomenon (and mostly because it\nwas our most fun idea). The claim we came up with was:\n\n> Around 75% of Korean songs mention the word \"love.\"\n\nThe purpose of this notebook is not to confirm nor deny the claim, but simply\nto showcase the data I collected in various ways.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":30,"value":"{\n const allSongs = songs.length.toLocaleString()\n const songsWithLyrics = songs.reduce((cnt, song) => cnt + (song.lyrics.length > 0), 0).toLocaleString()\n\n return md`## Data\n\n A total of ${allSongs} songs were fetched from [Melon](https://www.melon.com)'s Top-100 charts for each year from ${years.min.getFullYear()} to ${years.max.getFullYear()}. Of these ${allSongs} songs, ${songsWithLyrics} had lyrics available on [Melon](https://www.melon.com).\n\n For each of these ${songsWithLyrics} songs, [KoNLPy](https://konlpy-ko.readthedocs.io) was used\n to tokenize ${detail('each individual line', 'It was either tokenizing line-by-line, or tokenizing the whole text at once.')} of the lyrics into processing-friendly data.\n\n Finally, all this data was exported to a SQLite file loaded below.\n `\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"import { lyrics as songs } from \"@71/analyzing-popular-k-pop-song-lyrics-data\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":172,"value":"md`#### Table of all loaded songs\n\nThe songs are sorted by year and then by rank.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":100,"value":"table(\n songs.map(x => (\n { 'Year': ''+x.year,\n 'Rank': x.rank+' ',\n 'Artist': x.artist,\n 'Title': x.title,\n 'Has lyrics': x.lyrics.length > 0 ? 'Yes' : 'No',\n }\n)), {\n pageSize: 25\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":123,"value":"md`## Curiosity\n\nNow, let's see how many songs mention the word \"사랑\" (love).\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":127,"value":"songs_that_mention_love = songs.filter(has_token((word, type) => word === '사랑' && type === 'Noun'))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":137,"value":"md`${songs_that_mention_love.length} songs, huh? Not bad. How does it compare to other ${detail('words', 'Feel free to change this array; all results will be updated right away.')} though?`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":140,"value":"words_to_count = ['여자', 'Girl', '남자', 'Boy', 'Man', 'Woman', '사랑', 'Love', '안녕', 'Hey', 'Baby']","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":141,"value":"table(\n words_to_count.map(word => (\n { '': word,\n 'Usages (songs)': songs.filter(has_token(word)).length,\n 'Usages (total)': songs.reduce((acc, v) => acc + v.lyrics.filter(has_token(word)).length, 0),\n }\n ))\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":180,"value":"md`## Trends over time\n\nSince we have the year at which the songs appeared in the Top-100,\nwe might be able to find trends that change over time.\n\nThe table below displays how many times a given word appeared in the lyrics\nof the Top-100 songs, sorted in decreasing order.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":241,"value":"viewof hist_data = {\n const sorted_hist = Object.values(words).sort((a, b) => b.songs.size - a.songs.size)\n\n return table(sorted_hist\n .map(({ word, songs }, i) => ({\n 'Word': word,\n 'Compare': html``,\n 'Songs': songs.size,\n get 'Total'() { return [...songs].reduce(\n (acc, song) => acc + song.lyrics.reduce(\n (acc, tok) => acc + (tok === word), 0),\n 0) },\n })))\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":438,"value":"md`And now, the evolution of the most popular words over the past years.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":458,"value":"viewof totalOrPerSong = radio({\n options: [\n { label: 'Total occurences', value: 'total' },\n { label: 'Songs', value: 'persong' },\n ],\n value: 'persong',\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":561,"value":"{\n const wordsToCompare = hist_data.data\n .filter(x => x['Compare'].checked)\n .map(x => words[x['Word']])\n .sort((a, b) => b.total - a.total)\n\n const wordYears = totalOrPerSong === 'total'\n ? word => years.asIntegers().map(year =>\n [...word.songs].reduce((acc, song) =>\n acc + (song.year === year ? count_word(word.word, song) : 0), 0))\n : word => years.asIntegers().map(year =>\n [...word.songs].reduce((acc, song) =>\n acc + (song.year === year ? 1 : 0), 0))\n\n return c3({\n data: {\n x: 'x',\n xFormat: '%Y',\n columns: [\n ['x', ...years],\n\n ...wordsToCompare.map(word => [\n word.word, ...wordYears(word),\n ]),\n ],\n },\n axis: {\n x: {\n type: 'timeseries',\n tick: {\n format: '%Y',\n },\n },\n },\n }, this)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":402,"value":"md`\nThe word \"사랑\" seems to appear less over time. In 2010, it appeared a total of 404 times\nin 68 songs. In 2013, it was down to 233 appearances in 59 songs, and in 2018, it\nappeared 173 times in 53 songs.\n\nDespite this decrease in usage, \"사랑\" remains one of the most used words in lyrics of the\npast decade, along with \"I\" and \"날\".\n\nThis makes me wonder, has the diversity of words increased in the past decade? Well, let's find out!\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":412,"value":"{\n const yearsData = []\n\n for (let i = years.startYear; i <= years.endYear; i++) {\n const set = new Set\n\n for (const song of songs) {\n if (song.year !== i) continue\n\n for (const { text, stem, pos } of song.lyrics) {\n if (INTERESTING_POS.has(pos))\n set.add(stem ?? text)\n }\n }\n\n yearsData.push(set.size)\n }\n\n return c3({\n data: {\n x: 'x',\n xFormat: '%Y',\n columns: [\n ['x', ...years],\n ['Diversity', ...yearsData],\n ],\n },\n axis: {\n x: {\n type: 'timeseries',\n tick: {\n format: '%Y',\n },\n },\n },\n })\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":423,"value":"md`While the number of different words used in songs peaked in 2015, the overall diversity more than doubled since 2000.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":185,"value":"md`### The recipe to success\n\nSimilarly, can we use the rank of each song as well as its\ncontent to cook up a magic recipe to topping the Korean charts?\n\nThe following table shows words in order of popularity (regardless of the year). For instance, \"사랑\" appears in 15 of the 20 songs that topped the charts from ${years.startYear} to ${years.endYear}. It similarly appeared in 18 of the 20 songs that were second in the charts.\n\nSo yes, \"사랑\" appears in most songs, but it also leads to very popular songs.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":294,"value":"{\n const rows = []\n const bold = text => `${text}`\n\n for (const w in words) {\n const word = words[w]\n const row = { 'Word': w }\n\n if (word.ranks[1])\n row[bold('1')] = word.ranks[1].songs.size\n if (word.ranks[2])\n row[bold(2)] = word.ranks[2].songs.size\n if (word.ranks[3])\n row[bold(3)] = word.ranks[3].songs.size\n if (word.ranks[4])\n row[bold(4)] = word.ranks[4].songs.size\n if (word.ranks[5])\n row[bold(5)] = word.ranks[5].songs.size\n\n for (const [start, end] of [[6, 10], [11, 20], [21, 30], [31, 40], [41, 50], [51, 100]]) {\n let total = 0\n\n for (let i = start; i <= end; i++) {\n if (word.ranks[i])\n total += word.ranks[i].songs.size\n }\n\n if (total > 0)\n row[bold(`${start}-${end}`)] = total\n }\n\n rows.push(row)\n }\n\n return table(rows)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":338,"value":"viewof offset = slider({\n min: 0,\n max: 800,\n step: 10,\n value: 0,\n title: 'Offset',\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":378,"value":"viewof range = radio({\n title: 'Ranges',\n options: [\n { label: 'Top 5' , value: 'top5' },\n { label: 'Top 10' , value: 'top10' },\n { label: 'Top 20' , value: 'top20' },\n { label: 'Top 100', value: 'top100' },\n ],\n value: 'top5',\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":318,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":199,"value":"md`### Claim\n\nEven though this is not the topic of the notebook, I thought I'd explain how\nwe came up with the claim \"around 75% of Korean songs mention love\" anyway.\n\nWe asked 31 (Korean and French) friends of ours the following question:\n\n> In your opinion, what percentage of Korean songs ${detail(\"mention love?\", \"This question is actually flawed, and should have been 'In your opinion, what percentage of Korean songs mention the word 사랑?', but we realized that too late.\")}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":210,"value":"table(\n Object.entries({\n '15%': 2,\n '20%': 1,\n '30%': 2,\n '60%': 1,\n '70%': 4,\n '75%': 3,\n '80%': 4,\n '90%': 7,\n '95%': 2,\n '96%': 1,\n '98%': 1,\n '99%': 3,\n }).map(x => ({ 'Estimated percentage': x[0], 'Number of responses': x[1] }))\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":215,"value":"md`Out of curiosity, we also asked them to estimate on a scale from 1 to 5 how much they were familiar to K-pop, and the average we obtained was 4.3 out of 5; we should then have expected the results of the survey to have been accurate.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"md`### Appendix`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":591,"value":"years = {\n const startYear = d3.min(songs, song => song.year)\n const endYear = d3.max(songs, song => song.year)\n\n const length = endYear - startYear + 1\n const years = Array.from({ length }, (_, i) => new Date(startYear + i, 0))\n\n return Object.assign(\n years,\n {\n min: new Date(startYear, 0),\n max: new Date(endYear, 0),\n startYear,\n endYear,\n asIntegers: () => Array.from({ length }, (_, i) => startYear + i),\n }\n )\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":275,"value":"words = {\n const words = {}\n\n for (const song of songs) {\n const { year, rank } = song\n\n for (const { text: word, pos: type } of song.lyrics) {\n if (!INTERESTING_POS.has(type)) continue\n\n const wordObj = words[word] ||\n (words[word] = { word, type, score: 0, repeatScore: 0, ranks: {}, songs: new Set })\n const rankObj = wordObj.ranks[rank] ||\n (wordObj.ranks[rank] = { score: 0, songs: new Set })\n\n wordObj.songs.add(song)\n rankObj.songs.add(song)\n rankObj.score += 100 - rank\n }\n }\n\n for (const word of Object.values(words)) {\n for (const i in word.ranks) {\n const rank = word.ranks[i]\n\n if (rank !== undefined) {\n word.score += (100 - i) * rank.songs.size\n word.repeatScore += rank.score\n }\n }\n }\n\n return Object.fromEntries(\n Object.entries(words).sort((a, b) => b[1].score - a[1].score)\n )\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":130,"value":"has_token = (predicateOrWord, type = undefined) => obj => {\n const predicate = typeof predicateOrWord === 'function'\n ? predicateOrWord\n : (w, t) => w.toLowerCase() === predicateOrWord.toLowerCase()\n && (type === undefined || t === type)\n\n if (obj.lyrics != undefined)\n // Song given\n return obj.lyrics.some(x => predicate(x.text, x.pos))\n else if (obj.text && obj.pos)\n // Tuple of lyrics given\n return predicate(obj.text, obj.pos)\n else\n throw new Error(JSON.stringify(obj))\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":95,"value":"function detail(span, text) {\n return html`${span}*`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":750,"value":"count_word = (word, song) =>\n song.lyrics.reduce((acc, [_, tokens]) => acc + tokens.reduce((acc, [tok, _]) => acc + (tok === word), 0), 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":824,"value":"INTERESTING_POS = new Set([\"Noun\", \"Verb\", \"Adverb\", \"Adjective\"])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":827,"value":"ALL_POS = new Set(songs.flatMap((song) => song.lyrics.map((token) => token.pos)))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":778,"value":"DOM.download(await FileAttachment('kpop-notebook.ipynb').blob(), 'kpop-notebook.ipynb', 'Download Jupyter Notebook for generating the data')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":686,"value":"md`#### Imports`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":18,"value":"d3 = require('d3@5')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":545,"value":"pako = require('pako@1/dist/pako_inflate.min.js')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":540,"value":"import { c3 } from '@71/c3'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":75,"value":"import { table } from '@tmcw/tables/2'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":265,"value":"import { radio, slider } from '@jashkenas/inputs'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W34","stdlib":"1","titling":"auto"}} 32:"$31:body"