2c:null 2d:["$","title",null,{"children":"Korean NLP | Grégoire Geis | Observable"}] 32:T15c8,viewof input = { const w = "100%", h = "100px", tokenColors = { Noun: "#228be6", Verb: "#fd7e14", Josa: "#7950f2", KoreanParticle: "#be4bdb", Alpha: "#20c997", Number: "#20c997", Punctuation: "#20c997", }; const rootElement = html`

`; const inputElement = rootElement.querySelector(".input"), sentencesElement = rootElement.querySelector(".sentences"), descriptionElement = rootElement.querySelector(".desc"); const createTokenElement = ({ text, pos: kind, stem }) => { const el = html`${text}`; if (kind !== undefined) { const descr = `${kind}${stem ? `: ${stem}` : ""}`; el.addEventListener("mouseenter", () => { descriptionElement.textContent = descr === "Space" ? "" : descr; descriptionElement.style.color = tokenColors[kind] ?? "inherit"; }); el.addEventListener("mouseleave", () => { if (descriptionElement.textContent === descr) descriptionElement.textContent = ""; }); } return el; }; const getAbsoluteOffset = (container, offset) => { // https://developer.mozilla.org/en-US/docs/Web/API/Range/startOffset let absoluteOffset = offset; for (;;) { while (container.previousSibling === null) { container = container.parentNode; if (container === inputElement) { return absoluteOffset; } } container = container.previousSibling; absoluteOffset += container.textContent.length; } }; const getNodeAndRelativeOffset = (node, absoluteOffset) => { for (;;) { if (node instanceof Text) { const nodeText = node.nodeValue; if (absoluteOffset <= node.length) { return [node, absoluteOffset]; } absoluteOffset -= nodeText.length; } if (node.firstChild !== null) { node = node.firstChild; continue; } for (;;) { if (node.nextSibling !== null) { node = node.nextSibling; break; } node = node.parentNode; if (node === inputElement) throw new Error("internal error"); } } }; const onChange = (e) => { // Tokenize and split input. const input = inputElement.textContent.replace("\xa0", " "), tokens = tokenize(input), sentences = splitSentences(input); // Mark sentences. const selection = getSelection(), ranges = Array.from({ length: selection.rangeCount }, (_, i) => { const range = selection.getRangeAt(i), start = getAbsoluteOffset(range.startContainer, range.startOffset), end = range.collapsed ? start : getAbsoluteOffset(range.endContainer, range.endOffset); return { start, end }; }); sentencesElement.textContent = ""; for (let i = 0; i < sentences.length; i++) { const sentence = sentences[i]; if (i > 0) { const start = sentences[i - 1].offset + sentences[i - 1].length, end = sentence.offset; sentencesElement.append(input.slice(start, end)); } sentencesElement.appendChild(createTokenElement({ text: sentence.text })); } // Mark tokens; we don't actually do this is the user is composing a Korean character. if (e?.isComposing) { return; } inputElement.textContent = ""; selection.removeAllRanges(); for (const token of tokens) { inputElement.appendChild(createTokenElement(token)); } for (const { start, end } of ranges) { const [startNode, startOffset] = getNodeAndRelativeOffset(inputElement.firstChild, start), [endNode, endOffset] = start === end ? [startNode, startOffset] : getNodeAndRelativeOffset(inputElement.firstChild, end), range = document.createRange(); range.setStart(startNode, startOffset); range.setEnd(endNode, endOffset); selection.addRange(range); } // Notify of change. rootElement.value = input; rootElement.dispatchEvent(new CustomEvent("input")); }; inputElement.textContent = "맞아 어젯밤에 봤어! ㅋㅋ"; inputElement.addEventListener("input", onChange); inputElement.addEventListener("keypress", (e) => e.which === 13 && e.preventDefault()); onChange(); return rootElement; }33:[] 30:{"id":"64e2af71a5929a03","slug":"korean-nlp","title":"Korean NLP","thumbnail":"e1c2e5675b0fc76ce6900dfe1c3f77dec00459d5f4eee69a6e0f8ee916c8fc96","access_level":"public","create_time":"$D2022-07-27T17:16:22.296Z","update_time":"$D2022-12-03T21:59:50.178Z","view_time":null,"first_public_time":"$D2022-07-27T17:33:09.152Z","has_unpublished_edits":null,"fork":false,"stars":2,"starred":false,"owner":{"type":"team","id":"6817f8df4b71afcf","login":"71","name":"Grégoire Geis","avatar_url":"https://avatars-next.observableusercontent.com/046456a6fcb599cae062b5a172a9e2c85acef403c376bdb60a4cc6d135a60aad","tier":"starter_2024"},"version":411,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"3881cfc7f3117323","login":"71","name":"Grégoire Geis","avatar_url":"https://avatars-next.observableusercontent.com/046456a6fcb599cae062b5a172a9e2c85acef403c376bdb60a4cc6d135a60aad","tier":null}],"members":[{"type":"individual","id":"3881cfc7f3117323","login":"71","name":"Grégoire Geis","avatar_url":"https://avatars-next.observableusercontent.com/046456a6fcb599cae062b5a172a9e2c85acef403c376bdb60a4cc6d135a60aad","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"title":"Korean NLP","cells":[{"id":0,"value":"# Korean NLP","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":10,"value":"Korean Natural Language Processing, powered by [`okt.js`](https://github.com/71/oktjs).","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":306,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":61,"value":"### Normalize\n\n```ts\nfunction normalize(text: string): string;\n```","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":67,"value":"normalized = normalize(input)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":57,"value":"### Tokenize\n\n```ts\nfunction tokenize(text: string): KoreanToken[];\n```","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":32,"value":"tokenize(normalized)\n .filter((x) => x.pos !== \"Space\")\n .map((x) => `${x.text} (${x.pos}${x.stem ? `: ${x.stem}` : \"\"})`)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":49,"value":"### Extract phrases\n\n```ts\nfunction extractPhrases(text: string): KoreanPhrase[];\nfunction extractPhrases(tokens: KoreanToken[]): KoreanPhrase[];\n```","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":80,"value":"extractPhrases(normalized).map((x) => `${x.text} (${x.pos})`)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":51,"value":"### Split sentences\n\n```ts\nfunction splitSentences(text: string): KoreanSentence[];\n```","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":92,"value":"splitSentences(normalized).map((x) => x.text)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":24,"value":"## Appendix","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":21,"value":"normalize = Okt.normalize","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":26,"value":"tokenize = Okt.tokenize","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"extractPhrases = Okt.extractPhrases","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":85,"value":"splitSentences = Okt.splitSentences","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"Okt = import(\"https://cdn.jsdelivr.net/npm/oktjs@0.1.1/index.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}} 31:"$30:body"