30:T5e2,# Narrator 100% client-side Browser based implementation of the very cool [narrator project](https://github.com/cbh123/narrator). Requires WebGPU: ${!!hasWebGPU ? "✅" : "🚫"}. This is the 100% client-side version of my previous [narrator notebook](https://observablehq.com/@ryanseddon/narrator). The previous version used the huggingface API and elevenlabs for the voice generation.

More info

Well 18 months on we can do this with 3 models that can all run directly in the browser! Amazing progress in such a short time.

Models used:

Image captioning uses [Xenova/vit-gpt2-image-captioning](https://huggingface.co/Xenova/vit-gpt2-image-captioning)
Generating script uses [Llama-3.2-1B-Instruct-q4f32_1-MLC](https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f32_1-MLC)
Text to speech uses [Kokoro-82M-v1.0-ONNX](https://huggingface.co/onnx-community/Kokoro-82M-v1.0-ONNX)

Getting started

Hit the start capture button after 3 seconds the first screenshot will be taken every ${duration}s thereafter
A caption will be generated and fed into the narration model
Finally Kokoro will generate and audio clip based on the description text

The screenshot is taken every ${duration}s due to response time and length of audio taking up most of that time, you can adjust this via the range slider below.

31:T633,checkMediaDevices = { let button = html` `; button.onclick = () => { if (navigator.mediaDevices.getUserMedia) { navigator.mediaDevices .getUserMedia({ video: { width: { ideal: width }, height: { ideal: width / 1.6 }, facingMode: "user" } }) .then((stream) => { video.srcObject = stream; mutable capabilities = stream.getVideoTracks()[0].getCapabilities(); }) .catch((error) => { mutable capabilities = "No Video Camera"; }); Promises.delay(3000).then(() => { // wait 3 seconds to get video feed snapshot mutable photo = snapShot(); }); return "OK"; } return "navigator does not support mediaDevices"; }; return button; }32:[] 2e:{"title":"Narrator 100% client-side","cells":[{"id":3,"value":"$30","pinned":false,"hidden":false,"mode":"md"},{"id":540,"value":"loadAIStuff = loadModels.button","pinned":false,"hidden":false,"mode":"ojs"},{"id":616,"value":"modelWarning = md`Warning: Clicking this button will download ~1GB of files.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":5,"value":"## Screenshot taken every ${duration}s","pinned":false,"hidden":false,"mode":"md"},{"id":74,"value":"mainView = html`\n

Webcam feed

\n ${video}\n

Snapshot

\n ${photo}\n

Note: You'll need to keep this in view otherwise the screenshot will come out as a black screen

\n \n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"{\n if (countdown) {\n return md`### Countdown to next screenshot ${countdown}s`;\n }\n\n return md`### Hit Start Capture button to begin`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":160,"value":"#### Check the below checkbox to auto snapshot from video feed","pinned":false,"hidden":false,"mode":"md"},{"id":12,"value":"viewof narrationText = Inputs.textarea({\n placeholder: \"Narration text will appear here\",\n label: \"Description of screenshot in the style of David Attenborough\",\n disabled: true,\n rows: 5,\n cols: 100,\n value: narration ?? \"\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":13,"value":"viewof captionText = Inputs.textarea({\n disabled: true,\n placeholder: \"Image caption text will appear here\",\n value: caption ?? \"\",\n label: \"Image caption\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":14,"value":"viewof prompt = Inputs.textarea({ label: \"Prompt\", value: textGenIn, rows: 6 })","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"## Generated audio\nWill autoplay once response comes back","pinned":false,"hidden":false,"mode":"md"},{"id":11,"value":"ttsAudio = audio","pinned":false,"hidden":false,"mode":"ojs"},{"id":146,"value":"viewof autoSnapshot = Inputs.checkbox([\"Auto Snapshots\"])","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"viewof duration = Inputs.range([10, 120], {\n label: \"Seconds between screenshots\",\n step: 1,\n value: 45\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":20,"value":"

Appendix

All cells below here are to make this demo work, only explore if you're interested in how it works.\n

","pinned":false,"hidden":false,"mode":"html"},{"id":413,"value":"hasWebGPU = !!navigator.gpu","pinned":false,"hidden":false,"mode":"ojs"},{"id":6,"value":"mutable photo = html`No photo taken yet`","pinned":false,"hidden":false,"mode":"ojs"},{"id":19,"value":"video = html`\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"function snapShot() {\n const height = video.videoHeight / 4;\n const width = video.videoWidth / 4;\n const context = DOM.context2d(width, height);\n context.drawImage(video, 0, 0, width, height);\n return context.canvas;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":136,"value":"timerEvent = audio.addEventListener(\n \"timeupdate\",\n (e) => (mutable timeLeft = audio.duration - audio.currentTime)\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":132,"value":"mutable timeLeft = audio.duration || 0","pinned":false,"hidden":false,"mode":"ojs"},{"id":23,"value":"mutable capabilities = \"Waiting...\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"countdown = {\n if (capabilities !== \"Waiting...\" && !!autoSnapshot[0]) {\n let i = duration;\n while (true) {\n yield Promises.delay(1000, i === 0 ? (i = duration) : --i);\n }\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":25,"value":"{\n if (countdown === 0 && capabilities !== \"Waiting...\") {\n mutable photo = snapShot();\n return \"Photo taken\";\n } else {\n return \"Waiting...\";\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":28,"value":"imgURL = {\n console.log(\"I should run once\", Date.now());\n return await canvasToObjectURL(photo).catch((err) => false);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":547,"value":"generateAudio = {\n if (imgURL) {\n console.log({ imgURL, prompt });\n generateNarration?.postMessage({ imgURL, prompt });\n return true;\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":30,"value":"textGenIn = `You are Sir David Attenborough. Narrate the text in quotes describing an image of the human as if it is a nature documentary. Make it snarky and funny. Don't repeat yourself. Make it extremely short. If I do anything remotely interesting, make a big deal about it!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":33,"value":"function canvasToObjectURL(canvas, mimeType = 'image/png', qualityArgument) {\n return new Promise((resolve, reject) => {\n canvas.toBlob((blob) => {\n if (blob) {\n const objectUrl = URL.createObjectURL(blob);\n resolve(objectUrl);\n } else {\n reject(new Error('Canvas to objectUrl conversion failed'));\n }\n }, mimeType, qualityArgument);\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":36,"value":"import { tweet } from \"@observablehq/tweet\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":177,"value":"import {\n loadModels,\n generateNarration,\n mutable modelsLoaded,\n mutable caption,\n mutable narration,\n mutable audio\n} from \"@ryanseddon/narrator-worker\"","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}