31:[] 2f:{"title":"Tokenize text and generate n-grams","cells":[{"id":0,"value":"md`# Tokenize text and generate n-grams`","pinned":true,"hidden":false,"mode":"ojs"},{"id":52,"value":"viewof data = form(html``)","pinned":true,"hidden":false,"mode":"ojs"},{"id":54,"value":"data","pinned":true,"hidden":false,"mode":"ojs"},{"id":11,"value":"Array.from(new Set(ngrams));","pinned":true,"hidden":false,"mode":"ojs"},{"id":35,"value":"max_n = parseInt(data.max_n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":38,"value":"min_n = parseInt(data.min_n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"ngrams = tokenize(\"This is a great work @jack wow\");","pinned":true,"hidden":false,"mode":"ojs"},{"id":5,"value":"function get_chunks(tokens){\n let chunks = [];\n let chunk = [];\n for (let i = 0; i < tokens.length; i++){\n if(chunk.length > 0 && (!tokens[i].groups.word)){\n chunks.push(chunk);\n chunk = []\n } else if (tokens[i].groups.word) {\n chunk.push(tokens[i][0]);\n }\n }\n if(chunk.length > 0) chunks.push(chunk);\n return chunks;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":34,"value":"Array.from({length:max_n-min_n+1},(v,k)=>k+min_n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":14,"value":"\nfunction get_ngrams(chunks, max_n=3, min_n=2){\n const nvalues = Array.from({length:max_n-min_n+1},(v,k)=>k+min_n)\n const ngrams = chunks.flatMap(chunk => {\n const ng = chunk.flatMap((x,i) => {\n return nvalues\n .filter((j) => (i+j < chunk.length))\n .map((j) => {\n return chunk.slice(i, j+i)\n .map(x => x.toString()).join(\" \").toLowerCase()\n });\n });\n return ng;\n });\n return ngrams;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":16,"value":"function tokenize(text) {\n \n let tokens = [\n ...text.matchAll(\n /(?[\\w]+:\\/\\/[\\S]+)|(?@\\w+)|(?[\\w]+)|(?[\\s\\n]+)|(?((?!\\s)[\\W])+)/g\n )\n ];\n tokens = tokens.map(({input, ...t}) => {\n return { ...t, end_index: t.index + t[0].length };\n });\n tokens = tokens.filter(t => !t.groups.space) \n const chunks = get_chunks(tokens);\n const ngrams = get_ngrams(chunks, max_n, min_n);\n return Array.from(new Set(ngrams));\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":50,"value":"import {form} from \"@mbostock/form-input\"","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}