2d:null 2e:["$","title",null,{"children":"Hate Speech Detection - Diverging Stacked Bar Chart Visualization | Kasi Viswanath | Observable"}] 33:[{"id":"86edd30d38e8c77c7676eb8f25edf7955594a7478e639efd404e4fb02b535c6058b0f01633fc1e13f8cede5da83ac2991532ba1e0b927d4129d0f1b02d06d96a","href":"https://static.observableusercontent.com/files/86edd30d38e8c77c7676eb8f25edf7955594a7478e639efd404e4fb02b535c6058b0f01633fc1e13f8cede5da83ac2991532ba1e0b927d4129d0f1b02d06d96a","name":"idf_values.json","type":"application/json","lastModified":1660163738730.106,"size":2035632},{"id":"5a3f002cf9c521bec61a7fd94e6d3526fbd2429fc1e79cd68500f8ad88c853b3566e5104ed6f67c764071f87170aed3883c662ee5014f205f5e966059a24eecc","href":"https://static.observableusercontent.com/files/5a3f002cf9c521bec61a7fd94e6d3526fbd2429fc1e79cd68500f8ad88c853b3566e5104ed6f67c764071f87170aed3883c662ee5014f205f5e966059a24eecc","name":"xgb_classifier@2.json","type":"application/json","lastModified":1660288471307.32,"size":672638}] 31:{"id":"d2d3e6357f676d60","slug":"hsd-visualization","title":"Hate Speech Detection - Diverging Stacked Bar Chart Visualization","thumbnail":"622d3efa5bc1aa2e73a33dd4a8a81f35957a84b37c31d15bd397f68076d3874b","access_level":"public","create_time":"$D2022-08-09T18:16:41.636Z","update_time":"$D2023-10-09T04:06:49.969Z","view_time":null,"first_public_time":"$D2022-08-09T19:32:56.056Z","has_unpublished_edits":null,"fork":false,"stars":2,"starred":false,"owner":{"type":"team","id":"b760ea566a896d7d","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":"starter_2024"},"version":1374,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"fcd94e1c1ca12ae9","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":null}],"members":[{"type":"individual","id":"fcd94e1c1ca12ae9","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"title":"Hate Speech Detection - Diverging Stacked Bar Chart Visualization","cells":[{"id":0,"value":"# Hate Speech Detection - Diverging Stacked Bar Chart Visualization\n\nHate speech detection is the task of detecting if communication such as text, audio, and so on contains hatred and or encourages violence towards a person or a group of people.
\n\nPlease check the below examples and test it with your custom input","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1138,"value":"

Implementation Details

\n Classifier : XGBoost + TF-IDF
\n Accuracy : 0.90 | F1-Score : 0.94 | Precision : 0.94
\n Notebook : Git, Kaggle
\n The input sentence is classified based on the loaded XGBoost model with IDF values
\n Thanks @gnestor for pyodide(python distribution for browser).\n \n

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1252,"value":"## Example with One Sentence","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1181,"value":"Example1 = {\n textInput1;\n let sentences;\n try {\n sentences = eval(textInput1);\n } catch {\n sentences = [textInput1];\n } finally {\n let target = html`

`;\n let generateData_ = Promise.all(sentences.map((d) => generateData(d))).then(\n (resolve) => {\n resolve.forEach((d) => {\n let chart_ = chart(d);\n target.appendChild(\n Legend(chart_.scales.color, { title: \"prediction\" })\n );\n target.appendChild(chart_);\n });\n }\n );\n return target;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1326,"value":"X, Y axis defines the cumulative prob of sentence classification till that word\n- In case of above example,\n - probability of classifying I = classification('I')\n - probability of classifying hate = classification('I','hate')\n - probability of classifying stupid = classification('I','hate','stupid')\n - probability of classifying people = classification('I','hate','stupid','people')","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1260,"value":"viewof textInput1 = {\n let target = html`

`;\n  target.value = \"I hate stupid people\";\n  target.addEventListener(\"input\", () => {\n    if (!target.value) {\n      target.style[\"border-color\"] = \"red\";\n    } else {\n      target.style[\"border-color\"] = \"green\";\n    }\n  });\n  return target;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1266,"value":"## Example with Multiple Setences","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1250,"value":"Example2 = {\n  textInput2;\n  let sentences;\n  try {\n    sentences = eval(textInput2);\n  } catch {\n    sentences = [textInput2];\n  } finally {\n    console.log(sentences, typeof sentences);\n    let target = html`<div></div>`;\n    let generateData_ = Promise.all(sentences.map((d) => generateData(d))).then(\n      (resolve) =>\n        resolve.forEach((d) => {\n          let chart_ = chart(d);\n\n          target.appendChild(\n            Legend(chart_.scales.color, { title: \"prediction\" })\n          );\n          target.appendChild(chart_);\n        })\n    );\n    return target;\n  }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":833,"value":"viewof textInput2 = {\n  let target = html`<textarea style='font-size: 16px; line-height: 1.5; width: 100%; box-sizing: border-box; min-height: ${\n    16 * 1.5 * 4\n  }px;'/>`;\n  target.value = `[\"I hate all the people who do not come in time to office\", \"You are welcome to come to my country\",\"I hate _ people\"]`;\n  target.addEventListener(\"input\", () => {\n    if (!target.value) {\n      target.style[\"border-color\"] = \"red\";\n    } else {\n      target.style[\"border-color\"] = \"green\";\n    }\n  });\n  return target;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1276,"value":"function classify(data) {\n  return data[0].slice(data.length - 3).sort((d) => d[\"prob\"])[0][\"label\"];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1090,"value":"function chart(sentence) {\n  let target = StackedBarChart(sentence, {\n    x: (d) => d.word + \" (\" + d.index + \")\",\n    y: (d) => d.prob,\n    z: (d) => d.label,\n    yLabel: \"↑ Prob\",\n    zDomain: [\"hate\", \"offensive\", \"neither\"],\n    colors: d3.schemeSpectral[3],\n    width,\n    height: 500\n  });\n\n  return target;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1051,"value":"async function generateData(sentence) {\n  let res = [];\n  let words = sentence.split(\" \");\n  let labels = [\"hate\", \"offensive\", \"neither\"];\n  for (let index = 1; index <= words.length; index += 1) {\n    let prediction = await predictClasses(words.slice(0, index));\n    for (let [ind, val] of prediction[0].entries()) {\n      res.push({\n        label: labels[ind],\n        index: index,\n        prob: val,\n        word: words[index - 1]\n      });\n    }\n  }\n  return res;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":556,"value":"function predictClasses(words) {\n  return py`\nimport xgboost as xg\nimport numpy as np\nimport json\nmodel = xg.Booster()\nmodel.load_model(\"xgb.json\")\nimport numpy as np\nfrom scipy.sparse import csr_matrix\nimport json\nfrom sklearn import preprocessing \nfrom scipy import sparse\n\nf = open(\"idf.json\",\"r\")\nidf = json.loads(f.read())\ndef count_occuerence(words):\n    freq_word = {}\n    for word in words: \n        if word in freq_word:\n            freq_word[word]+=1\n        else:\n            freq_word[word]=1\n    return freq_word\n\nfind_count = count_occuerence(${words})\nres = []\nfor word in idf:\n    if word in find_count:\n        res.append(find_count[word] * idf[word][\"idf\"])\n    else:\n        res.append(0)\ntest = preprocessing.normalize([res], norm='l2')\ndtest = xg.DMatrix(data = sparse.csc_matrix(np.array(test)))\nmodel.predict(dtest)`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":742,"value":"idf_values = FileAttachment(\"idf_values.json\").text()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":669,"value":"xgb_classifier = await FileAttachment(\"xgb_classifier@2.json\").text()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":655,"value":"loadFiles = {\n  py`\nimport json\nf = open(\"xgb.json\",\"w\")\njson.dump(json.loads(${xgb_classifier}),f)\nf1 = open(\"idf.json\",\"w\")\njson.dump(json.loads(${idf_values}),f1)\n`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":347,"value":"import { py, pyodide } from \"@kasivisu4/pyodide-library\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1046,"value":"import { StackedBarChart } from \"@d3/stacked-bar-chart\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1094,"value":"import { Legend, Swatches } from \"@d3/color-legend\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}}
32:"$31:body"
</article></body></html>