30:T8c6,loadModels = { let button = html` `; button.onclick = () => { if (!mutable worker) { mutable worker = new Worker(script, { type: "module" }); mutable worker.addEventListener("message", ({ data }) => { handleMessages(data); }); } }; mutable btnRef = button.querySelector("button"); return { button, caption, narration }; }31:T6c9,script = { const blob = new Blob( [ ` import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.5.1'; import * as webllm from "https://esm.run/@mlc-ai/web-llm@0.2.78"; import * as kokoro from "https://unpkg.com/kokoro-js@1.2.0/dist/kokoro.web.js?module"; postMessage({type:"loading", content: "Models loading..."}); const caption = await pipeline( "image-to-text", "Xenova/vit-gpt2-image-captioning" ); const llama = await webllm.CreateMLCEngine( "Llama-3.2-1B-Instruct-q4f32_1-MLC" ); const tts = await kokoro.KokoroTTS.from_pretrained( "onnx-community/Kokoro-82M-v1.0-ONNX", { dtype: "fp32", device: "webgpu" } ); postMessage({type:"loaded", content: "Models loaded!"}); onmessage = async ({data: {imgURL, prompt}}) => { const [res] = await caption(imgURL); postMessage({type: "img-to-text", content: res.generated_text}); const messages = [ { role: "system", content: prompt }, { role: "user", content: res.generated_text } ]; const stream = await llama.chat.completions.create({ messages, temperature: 0 }); postMessage({type: "textgen", content:stream.choices[0].message.content}) const streamer = new kokoro.TextSplitterStream(); streamer.push(stream.choices[0].message.content); streamer.close(); const voiceStream = tts.stream(streamer, { voice: "bm_george", speed: 1 }); for await (const { text, audio } of voiceStream) { console.log("sending blob audio: ", text); postMessage({type: "blob", content: audio.toBlob()}); } }; ` ], { type: "application/javascript" } ); const script = URL.createObjectURL(blob); invalidation.then(() => URL.revokeObjectURL(script)); return script; }32:[{"id":"8e676c6fc0b65ba7551342782f7e4a07617385b02138ca956f637e6814332ecf109a61f8b321b718899fa565236d68e52bc6de84a8aa1b5ded6e6df8c1b79ba5","href":"https://static.observableusercontent.com/files/8e676c6fc0b65ba7551342782f7e4a07617385b02138ca956f637e6814332ecf109a61f8b321b718899fa565236d68e52bc6de84a8aa1b5ded6e6df8c1b79ba5","name":"download.png","type":"image/png","lastModified":1746495084812.374,"size":289492}] 2e:{"title":"Loads models in a worker","cells":[{"id":0,"value":"# Loads models in a worker\n\nThis helper notebook loads 3 models for captioning, text generation and text-to-speech. \n\n```js\nimport { loadModels } from \"@ryanseddon/narrator-worker\"\n```\n\n----","pinned":false,"hidden":false,"mode":"md"},{"id":191,"value":"loadModels.button","pinned":false,"hidden":false,"mode":"ojs"},{"id":93,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":180,"value":"audio = {\n const player = html`

`;\n\n if (blob) {\n player.src = URL.createObjectURL(blob);\n player.play();\n }\n\n return player;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":128,"value":"viewof prompt = Inputs.textarea({ label: \"Prompt\", value: textGenIn, rows: 6 })","pinned":false,"hidden":false,"mode":"ojs"},{"id":87,"value":"generateNarration = worker","pinned":false,"hidden":false,"mode":"ojs"},{"id":64,"value":"imgURL = FileAttachment(\"download.png\").url()","pinned":false,"hidden":false,"mode":"ojs"},{"id":57,"value":"{\n let button = html`\n \n \n `;\n button.onclick = () => {\n if (mutable modelsLoaded) {\n generateNarration?.postMessage({ imgURL, prompt });\n }\n };\n\n return button;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":33,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":130,"value":"textGenIn = `You are Sir David Attenborough. Narrate the text in quotes describing an image of the human as if it is a nature documentary. Make it snarky and funny. Don't repeat yourself. Make it extremely short. If I do anything remotely interesting, make a big deal about it!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":138,"value":"handleMessages = ({ type, content }) => {\n switch (type) {\n case \"loading\":\n mutable btnRef.innerText = content;\n mutable btnRef.classList.add(\"loading\");\n break;\n case \"loaded\":\n mutable modelsLoaded = true;\n mutable btnRef.innerText = content;\n break;\n case \"blob\":\n mutable blob = content;\n break;\n case \"img-to-text\":\n mutable caption = content;\n break;\n case \"textgen\":\n mutable narration = content;\n break;\n default:\n console.log(content);\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":144,"value":"mutable btnRef = null","pinned":false,"hidden":false,"mode":"ojs"},{"id":166,"value":"mutable modelsLoaded = false","pinned":false,"hidden":false,"mode":"ojs"},{"id":232,"value":"mutable worker = null","pinned":false,"hidden":false,"mode":"ojs"},{"id":203,"value":"mutable caption = null","pinned":false,"hidden":false,"mode":"ojs"},{"id":205,"value":"mutable narration = null","pinned":false,"hidden":false,"mode":"ojs"},{"id":212,"value":"mutable blob = null","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}