31:[] 2f:{"title":"Llama encoder and decoder","cells":[{"id":72,"value":"# Llama encoder and decoder\n\nFor more information on this tool, read [Understanding GPT tokenizers](https://simonwillison.net/2023/Jun/8/gpt-tokenizers/)\n\nThis variant uses https://github.com/belladoreai/llama-tokenizer-js\n\nThis tool has some bugs - try entering \"`This is llama`\" and you will get back \"`This is ll ma`\" for example.\n\nHere's a better alternative: https://belladoreai.github.io/llama-tokenizer-js/example-demo/build/\n\nEnter text to tokenize it:","pinned":false,"hidden":false,"mode":"md"},{"id":77,"value":"viewof text = Inputs.textarea({ value: \"Example text is here\" })","pinned":false,"hidden":false,"mode":"ojs"},{"id":82,"value":"html`${encode(text).join(\" \")}
${\n encode(text).length\n} token${encode(text).length == 1 ? \"\" : \"s\"}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":160,"value":"html`${spans}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":88,"value":"
Or convert tokens to text:","pinned":false,"hidden":false,"mode":"md"},{"id":93,"value":"viewof tokens = Inputs.textarea({\n placeholder: \"Paste space separated tokens here\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":337,"value":"Note that this tool uses the GPT-2 tokenizer, which differs slightly from the tokenizer used by more recent models. This is useful primarily as an educational tool for understanding how tokenization works.","pinned":false,"hidden":false,"mode":"md"},{"id":106,"value":"html`

${decode(tokens.split(/[^\\d+]/g).map((t) => parseInt(t, 10)))}

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":185,"value":"spans = await generateSpans(text)","pinned":true,"hidden":false,"mode":"ojs"},{"id":162,"value":"async function generateSpans(text) {\n const tokens = encode(text).slice(1);\n const htmlPromises = tokens.map(async (t) => {\n let text = decode([1, t]);\n return `${t}${text}`;\n });\n const html = await Promise.all(htmlPromises);\n return html.join(\"\");\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":362,"value":"llamatok = import(\n \"https://belladoreai.github.io/llama-tokenizer-js/llama-tokenizer.js\"\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":9,"value":"llamatok.default.encode(\"one and two and three\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":385,"value":"function encode(text) {\n return llamatok.default.encode(text);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":25,"value":"function decode(tokens) {\n return llamatok.default.decode(tokens);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":154,"value":"decode([1, 697])","pinned":true,"hidden":false,"mode":"ojs"},{"id":170,"value":"async function stringToPastelColor(str) {\n const msgUint8 = new TextEncoder().encode(str);\n const hashBuffer = await crypto.subtle.digest(\"SHA-256\", msgUint8);\n const hashArray = Array.from(new Uint8Array(hashBuffer)).slice(0, 3); // Taking only the first 3 bytes\n let color = \"#\";\n for (const value of hashArray) {\n let adjustedValue = Math.floor(value / 2) + 128; // Adjust the range to be between 128 and 256\n color += (\"00\" + adjustedValue.toString(16)).substr(-2);\n }\n return color;\n}","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}