31:T44b,// can this method be rewritten to account for line indices? // it seems that calling encode once is much faster than calling it multiple times ents = { let t1 = performance.now() let dict = {} let freq = new Uint32Array(2**16) let init = corpusEN[0].join(' ')//.match(/../g)//.split(/../) let span = init.length // round to nearest multiple of 2 or 4 depending on view type (16/32-bits) let buf = new ArrayBuffer(2*Math.ceil(span * 0.5)); let view = new Uint16Array(buf) let exit = new Uint8Array(buf) let utf8 = new TextEncoder() utf8.encodeInto(init,exit) span = view.length for (let i = 0; i < span; ++i) { let key = view[i] //let dat = freq.get(key); //dat ? freq.set(key,dat+1) : freq.set(key,1) freq[key] ? freq[key]++ : freq[key] = 1 } span = freq.length for (let i = 0; i < span; ++i) { let val = freq[i] if(val) dict[i] = val } dict = Object.entries(dict) .sort((a,b)=>b[1]-a[1])//.slice(3) .map(d=>[ d[0], d[1]]) return { t: performance.now()-t1, freq,init,exit,view,dict } }32:T6df,ents2 = { let t1 = performance.now() let dict = {} let freq = new Map() // map when 32-bit view, obj when 16-bit view let init = corpusEN[0].concat(corpusEN[1])//.slice(0) let span = init.length let exit = Array(span) let utf8 = new TextEncoder() let mark = 2**13; for (let i = 0; i < span; ++i) { let data = init[i] let size = data.length; let line = exit[i] = [] // pad string to nearest multiple of 2 when 16-bit view // pad string to nearest multiple of 4 when 32-bit view // let pads = (2*Math.ceil((size) * 0.5)) // data += (' ').repeat(pads -size) data = utf8.encode(data).buffer data = //new Uint16Array(data.transfer(2*Math.ceil((data.byteLength) * 0.5))) new Uint32Array(data.transfer(4*Math.ceil((data.byteLength) * 0.25))) size = data.length for (let j = 0; j < size; ++j) { let key = data[j] if( key < mark ) continue let dat = freq.get(key) dat ? dat.at(-1) == i ? null : dat.push(i) : freq.set(key,dat = [i]); //let dat = freq[key] // dat ? dat.at(-1) == i ? null : dat.push(i) : freq[key] = dat = [i] line.push(dat) } } /* let data = new QuickSet({span,slot:4, high:(2**16)-1}) for (let i = 0; i < span; ++i) { let list = exit[i] let size = list.length for (let j = 0; j < size; ++j) { let nums = list[j] let iter = nums.length // if(iter>512) continue for (let n = 0; n < iter; ++n) { data.winsum(nums[n]) } } exit[i] = data.top().filter(d=>d[1]>0).map(k=>[init[k[0]],k[1]]) data.clear(true) }*/ return { t: performance.now()-t1, freq,exit,dict } }33:T454,ents3 = { let t1 = performance.now() let bits = 4; let mark = (String.fromCodePoint(2**14)).repeat(bits*2) // add safe padding by doubling bit width let init = corpusEN[0].concat([' ']).join(mark) // pad with one empty row // init = init.concat(init.slice(1)) let freq = new Map(); let span = init.length let [view,exit] = expect(span,bits); let [eofi,stop] = expect(bits*2,bits) let utf8 = new TextEncoder(); utf8.encodeInto(init,exit); utf8.encodeInto(mark,stop); eofi = eofi[0]; let id = 0, sure = 0; let prev; span = view.length; exit = 0 //Array.from({length:corpusEN[0].length},()=>[]); for (let i = 0; i < span; ++i) { let hash = view[i]; if (prev == eofi && hash !== eofi ) { ++id }; // id indices are off by some margin if(hash !== eofi) { let data = freq.get(hash) data ? data.at(-1) == id ? null : data.push(id) : freq.set(hash,data = [id]) } prev = hash; } return { t:performance.now()-t1,id, eofi,view,freq,exit//view.filter((d,i,f)=> d!==eofi ||(d == eofi && d !== f[i+1]) ) } }34:T677,ents4 = { let t1 = performance.now(); let bits = 4, divs = 1/bits; let calc = span => (bits * Math.ceil(span * divs)) //- span; let data = corpusEN[0].concat(corpusEN[1]) //.map(line=> line.padEnd(calc(line.length),' '))//line + (' ').repeat(calc(line.length))) //['hellooasda','b','c','d','e'] let span = data.length let init = Array(span) for (let i = 0; i < span; ++i) { let line = data[i] // line = line + ' ' + line.substring(1) //line = transliterable(line) ? encodeURIComponent(line) : line; line = line.padEnd(calc(line.length),' ') init[i] = line } let pads = bits * 1 let mark = String.fromCodePoint(0).repeat(pads) let rope = init.join(mark) + mark let size = rope.length; let [ view, exit ] = expect(size,bits) let utf8 = new TextEncoder(); utf8.encodeInto(rope,exit); let freq = new Map(); let temp = Array.from({length:span},()=>[]) let from, till = 0, ceil = Math.ceil; for (let i = 0; i < span; ++i) { from = till; till += ceil((init[i].length + (pads)) * divs) for (let n = from; n < till; ++n) { let hash = view[n] let data = freq.get(hash) data? data.at(-1) == i ? null : data.push(i) : freq.set(hash,data = [i]) //let data = freq[hash] // data ? data.at(-1) == i ? null : data.push(i) : freq[hash] = data = [i] temp[i].push(data) } } let dict = [] //[...freq.entries()].sort((a,b)=>b[1].length-a[1].length).slice(1) //Object.entries(freq).sort((a,b)=>b[1].length-a[1].length) return { t:performance.now()-t1, init,view,freq,temp,dict } }35:[] 2f:{"title":"Numeric string encoding","cells":[{"id":0,"value":"# Numeric string encoding\n## Based on n-gram frequency encoding","pinned":false,"hidden":false,"mode":"md"},{"id":19,"value":"___","pinned":false,"hidden":false,"mode":"md"},{"id":819,"value":"(\"requisition\").split(/ |(.[aeoiu]+.)/).filter(Boolean)","pinned":true,"hidden":false,"mode":"ojs"},{"id":1041,"value":"Plot.plot({\n marks: [\n Plot.ruleY([0]),\n Plot.lineY(ents4.dict.map(d=>d[1].length), {x: (_,i)=>i, y: d=>d}),\n // Plot.ruleY([mean])\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1078,"value":"mean = d3.mean(\n char.dict.map(d=>d[1])\n)*2.75","pinned":false,"hidden":false,"mode":"ojs"},{"id":1115,"value":"rgx = {\n \n let chars = char.dict.filter(d=>d[1]>mean).map(d=>d[0]).join('')\n return new RegExp(`\\\\s|(.[${chars}]{1,2}.)`,'g')\n\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":892,"value":"char = { \n let t1 = performance.now()\n\n \n let dict = {}\n let init = corpusEN.join(' ')//.replaceAll(' ','')\n let span = init.length \n\n for (let i = 0; i < span; ++i) {\n let key = init.codePointAt(i)\n dict[key] ? dict[key]++ : dict[key] = 1;\n }\n\n dict = Object.entries(dict)\n .sort((a,b)=>b[1]-a[1]).slice(1)\n .map(d=>[String.fromCodePoint(d[0]),d[1]])\n\n return {\n t: performance.now()-t1,\n dict\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1498,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":1927,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":2583,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":3284,"value":"$34","pinned":true,"hidden":false,"mode":"ojs"},{"id":3742,"value":"function transliterable(string) { \n return (/\\P{ASCII}/gu).test(string)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2645,"value":"function expect(span = 512, bits = 1) {\n \n let buff, exit, calc;\n switch (true) {\n case bits <= 1:\n calc = (span) => span;\n buff = new ArrayBuffer(span);\n return [exit = new Uint8Array(buff), exit,calc];\n\n case bits <= 2:\n calc = (span) => 2 * Math.ceil(span * 0.5);\n buff = new ArrayBuffer(calc(span))\n return [new Uint16Array(buff), new Uint8Array(buff),calc ];\n\n case bits <= 4:\n calc = (span) => 4 * Math.ceil(span * 0.25)\n buff = new ArrayBuffer(calc(span));\n return [new Uint32Array(buff), new Uint8Array(buff),calc];\n\n case bits <= 8:\n calc = (span) => 8 * Math.ceil(span * 0.125)\n buff = new ArrayBuffer(calc(span));\n return [new BigInt64Array(buff), new Uint8Array(buff),calc];\n\n default: throw new Error('Maximum bits is 8')\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":1166,"value":"function chop(string, /*A = [], B = [],*/dict, index, prevA, prevB, i = 0, w = 5) {\n \n //let c = char.dict[i][0];\n \n if (string && i < char.dict.length) {\n let pre = string.substring(0, w);\n let end = string.substr(-w);\n \n pre.length ==w && end.length ==w ? (\n prevA !== pre ? dict[pre] ? dict[pre].push(index) : dict[pre] = [index] : null,\n prevB !== end ? dict[end] ? dict[end].push(index) : dict[end] = [index] : null\n ): null\n \n return chop(string.replaceAll(char.dict[i][0], \"\"), dict, index, pre,end, ++i);\n \n } \n\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":806,"value":"dist = Object.entries(coded.dict).sort((a,b)=>b[1].length-a[1].length).filter(d=>d[1].length>16 && d[1].length< 512)","pinned":true,"hidden":false,"mode":"ojs"},{"id":577,"value":"coded = {\n let t1 = performance.now()\n\n let init = corpusEN[0]\n let span = init.length\n let exit = Array(span)\n\n \n let refs = new Map();\n let dict = new Object();\n let n = 0;\n \n for (var i = 0; i < span; ++i) {\n \n var line = init[i].split(' ')\n var size = line.length\n for (var j = 0; j < size; ++j) {\n\n var item = line[j]\n // item = refs.get(item) || (refs.set(item,++n),n)\n \n var data = dict[item];\n data ? data.push(i) : dict[item] = [i]; \n\n line[j] = dict[item];\n \n }\n exit[i] = line\n }\n\n return {\n t: performance.now()-t1,\n dict,exit\n }\n \n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":611,"value":"function encode(string) {\n\n let span = string.length\n let exit = new Uint32Array(span)\n for (let i = 0; i < span; ++i) {\n exit[i] = string.codePointAt(i)\n }\n return exit\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":23,"value":"function chunk({string = '', window = 3, lower = window, upper = lower,dict = {}}={}) {\n\n let span = string.length;\n\n let list = []; //new Array(( (upper-lower) * 0.5 ) * ( lower + upper -1) * span)\n\n\n // --lower \n // for (window; window > lower ; --window) {\n ++upper \n for (window; window < upper ; window++) {\n for (var j = 0; j < span - window + 1; j++) {\n\n var hash = string.substring(j, j + window);\n dict[hash] ? dict[hash]++ : dict[hash] = 1;\n\n list.push(hash)\n }\n }\n\n return list\n \n \n}\n","pinned":true,"hidden":false,"mode":"ojs"},{"id":5,"value":"___\n\n### Imports","pinned":false,"hidden":false,"mode":"md"},{"id":946,"value":"QuickSet = (await import('https://unpkg.com/@suptxt/quickset/dist/index.js?module')).default","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"import {newsweb} from \"@dleeftink/datasets\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":1044,"value":"corpusJA = fetch('https://raw.githubusercontent.com/yusugomori/jesc_small/master/test.ja').then(d=>d.text()).then(d=>d.split('\\n'))","pinned":false,"hidden":false,"mode":"ojs"},{"id":600,"value":"corpusEN = newsweb","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}