3e:null 3f:["$","title",null,{"children":"Tokenize text and generate n-grams | Shubhanshu Mishra | Observable"}] 45:[] 43:{"id":"09f7a5953266b094","slug":"tokenize-text-and-generate-n-grams","title":"Tokenize text and generate n-grams","thumbnail":"111313ac89afb7d84330dfb3d6ec06be2b12f2d387084e610d449f84741318b9","access_level":"public","create_time":"$D2020-12-11T18:29:46.653Z","update_time":"$D2020-12-11T22:22:32.668Z","view_time":null,"first_public_time":"$D2020-12-11T18:32:19.005Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"3df8ffe3d85471be","login":"napsternxg","name":"Shubhanshu Mishra","avatar_url":"https://avatars-next.observableusercontent.com/38949e46a60167fcf6722d089a015b3ab69512009eaa466fffcf77bc884707c4","tier":"starter_2024","bio":"Researcher in Machine learning, Data Mining, Social Science, and Natural Language Processing\r\nProgramming languages: Python, Java, and Java Script\r\n"},"version":71,"publish_version":71,"role":null,"authored":false,"authors":[{"type":"individual","id":"88a899f83ba8101d","login":"napsternxg","name":"Shubhanshu Mishra","avatar_url":"https://avatars-next.observableusercontent.com/38949e46a60167fcf6722d089a015b3ab69512009eaa466fffcf77bc884707c4","tier":null,"bio":"Researcher in Machine learning, Data Mining, Social Science, and Natural Language Processing\r\nProgramming languages: Python, Java, and Java Script\r\n"}],"members":[{"type":"individual","id":"88a899f83ba8101d","login":"napsternxg","name":"Shubhanshu Mishra","avatar_url":"https://avatars-next.observableusercontent.com/38949e46a60167fcf6722d089a015b3ab69512009eaa466fffcf77bc884707c4","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Tokenize text and generate n-grams","cells":[{"id":0,"value":"md`# Tokenize text and generate n-grams`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":52,"value":"viewof data = form(html``)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":54,"value":"data","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":11,"value":"Array.from(new Set(ngrams));","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":35,"value":"max_n = parseInt(data.max_n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"min_n = parseInt(data.min_n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"ngrams = tokenize(\"This is a great work @jack wow\");","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"function get_chunks(tokens){\n let chunks = [];\n let chunk = [];\n for (let i = 0; i < tokens.length; i++){\n if(chunk.length > 0 && (!tokens[i].groups.word)){\n chunks.push(chunk);\n chunk = []\n } else if (tokens[i].groups.word) {\n chunk.push(tokens[i][0]);\n }\n }\n if(chunk.length > 0) chunks.push(chunk);\n return chunks;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":34,"value":"Array.from({length:max_n-min_n+1},(v,k)=>k+min_n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":14,"value":"\nfunction get_ngrams(chunks, max_n=3, min_n=2){\n const nvalues = Array.from({length:max_n-min_n+1},(v,k)=>k+min_n)\n const ngrams = chunks.flatMap(chunk => {\n const ng = chunk.flatMap((x,i) => {\n return nvalues\n .filter((j) => (i+j < chunk.length))\n .map((j) => {\n return chunk.slice(i, j+i)\n .map(x => x.toString()).join(\" \").toLowerCase()\n });\n });\n return ng;\n });\n return ngrams;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"function tokenize(text) {\n \n let tokens = [\n ...text.matchAll(\n /(?[\\w]+:\\/\\/[\\S]+)|(?@\\w+)|(?[\\w]+)|(?[\\s\\n]+)|(?((?!\\s)[\\W])+)/g\n )\n ];\n tokens = tokens.map(({input, ...t}) => {\n return { ...t, end_index: t.index + t[0].length };\n });\n tokens = tokens.filter(t => !t.groups.space) \n const chunks = get_chunks(tokens);\n const ngrams = get_ngrams(chunks, max_n, min_n);\n return Array.from(new Set(ngrams));\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":50,"value":"import {form} from \"@mbostock/form-input\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W45","stdlib":"1"}} 44:"$43:body"