30:T64d,md`# Narrator Browser based implementation of the very cool [narrator project](https://github.com/cbh123/narrator). Works on your mobile device try it! I explored using open source models from Hugging Face: 1. [nlpconnect/vit-gpt2-image-captioning](https://huggingface.co/nlpconnect/vit-gpt2-image-captioning) for captioning the screenshot, this tends to not do a great job but will get us by for the demo 1. [mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) for taking that literal caption and giving it the David Attenborough flare, it's fast and great with its responses. It doesn't keep context like it would using GPT4 but it's free and possibly more accessible to folks to play around with! ## Elevenlabs I signed up for a free account with [ElevenLabs](https://elevenlabs.io/) which has a great selection of voices and gives you ${10_000} characters free/month via their API. The original author of this idea most likely used their clone service to get the David Attenborough voice. ## Getting started 1. You'll need to fill in the [Huggingface](https://huggingface.co/) APIKey and an [ElevenLabs](https://elevenlabs.io/) APIKey. 1. Hit the start capture button after 3 seconds the first screenshot will be taken every ${duration}s thereafter 1. A caption will be generated and fed into the narration model 1. Finally ElevenLabs will generate and audio clip based on the description text The screenshot is taken every ${duration}s due to response time and length of audio taking up most of that time, you can adjust this via the range slider below.`31:T633,checkMediaDevices = { let button = html` `; button.onclick = () => { if (navigator.mediaDevices.getUserMedia) { navigator.mediaDevices .getUserMedia({ video: { width: { ideal: width }, height: { ideal: width / 1.6 }, facingMode: "user" } }) .then((stream) => { video.srcObject = stream; mutable capabilities = stream.getVideoTracks()[0].getCapabilities(); }) .catch((error) => { mutable capabilities = "No Video Camera"; }); Promises.delay(3000).then(() => { // wait 3 seconds to get video feed snapshot mutable photo = snapShot(); }); return "OK"; } return "navigator does not support mediaDevices"; }; return button; }32:[] 2e:{"title":"Narrator","cells":[{"id":3,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":4,"value":"tweet(\"1724815159590293764\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":5,"value":"## Screenshot taken every ${duration}s","pinned":false,"hidden":false,"mode":"md"},{"id":74,"value":"mainView = html`\n

Webcam feed

\n ${video}\n

Snapshot

\n ${photo}\n

Note: You'll need to keep this in view otherwise the screenshot will come out as a black screen

\n \n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"{\n if (countdown) {\n return md`### Countdown to next screenshot ${countdown}s`;\n }\n\n return md`### Hit Start Capture button to begin`;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":160,"value":"#### Check the below checkbox to auto snapshot from video feed\nThis avoids churning through your 10k character limit with ElevenLabs!","pinned":false,"hidden":false,"mode":"md"},{"id":146,"value":"viewof autoSnapshot = Inputs.checkbox([\"Auto Snapshots\"])","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"viewof duration = Inputs.range([10, 120], {\n label: \"Seconds between screenshots\",\n step: 1,\n value: 45\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"## Generated audio\nWill autoplay once response comes back","pinned":false,"hidden":false,"mode":"md"},{"id":11,"value":"audio = html`\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":12,"value":"viewof narrationText = Inputs.textarea({\n placeholder: \"Narration text will appear here\",\n label: \"Description of screenshot in the style of David Attenborough\",\n //submit: \"Publish\",\n disabled: true,\n rows: 5,\n cols: 100,\n value: textGenRes ? textGenRes.generated_text.trim().replace(/\"/g, \"\") : \"\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":13,"value":"Inputs.textarea({\n disabled: true,\n placeholder: \"Image caption text will appear here\",\n value: img2textRes ? img2textRes.generated_text : \"\",\n label: \"Image caption\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":14,"value":"viewof prompt = Inputs.textarea({ label: \"Prompt\", value: textGenIn, rows: 6 })","pinned":false,"hidden":false,"mode":"ojs"},{"id":15,"value":"viewof voice = Inputs.select(voices, {\n label: \"Voice Id\",\n format: (x) => `${x.name}: ${x.voice_id}`,\n value: voices.find((t) => t.voice_id === \"7tTeJ5NyV3wWZmw2vPwv\")\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":16,"value":"viewof hfapiKey = secret(\"hfapiKey\", {\n submit: \"Set Huggingface API Key\",\n description:\n \"This is stored in localstorage feel free to use the inbuilt observable secrets\",\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":17,"value":"viewof elevenlabsapiKey = secret(\"elevenlabsKey\", {\n submit: \"Set Elevenlabs API Key\",\n description:\n \"This is stored in localstorage feel free to use the inbuilt observable secrets\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":20,"value":"

Appendix

","pinned":false,"hidden":false,"mode":"html"},{"id":6,"value":"mutable photo = html`No photo taken yet`","pinned":false,"hidden":false,"mode":"ojs"},{"id":19,"value":"video = html`\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"function snapShot() {\n const height = video.videoHeight / 4;\n const width = video.videoWidth / 4;\n const context = DOM.context2d(width, height);\n context.drawImage(video, 0, 0, width, height);\n return context.canvas;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":136,"value":"timerEvent = audio.addEventListener(\n \"timeupdate\",\n (e) => (mutable timeLeft = audio.duration - audio.currentTime)\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":132,"value":"mutable timeLeft = audio.duration || 0","pinned":false,"hidden":false,"mode":"ojs"},{"id":23,"value":"mutable capabilities = \"Waiting...\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"countdown = {\n if (capabilities !== \"Waiting...\" && !!autoSnapshot[0]) {\n let i = duration;\n while (true) {\n yield Promises.delay(1000, i === 0 ? (i = duration) : --i);\n }\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":25,"value":"{\n if (countdown === 0 && capabilities !== \"Waiting...\") {\n mutable photo = snapShot();\n return \"Photo taken\";\n } else {\n return \"Waiting...\";\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":26,"value":"{\n if (narrationText) {\n const options = {\n method: \"POST\",\n headers: {\n \"xi-api-key\": elevenlabsapiKey,\n \"Content-Type\": \"application/json\"\n },\n body: `{\"model_id\":\"eleven_monolingual_v1\",\"text\":\"${narrationText}\",\"voice_settings\":{\"similarity_boost\":1,\"stability\":0.5,\"style\":0.5}}`\n };\n\n return fetch(\n `https://api.elevenlabs.io/v1/text-to-speech/${voice.voice_id}`,\n options\n )\n .then((response) => response.blob())\n .then((blob) => {\n audio.src = URL.createObjectURL(blob);\n audio.load();\n return \"Audio loaded!\";\n })\n .catch((err) => console.error(err));\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":27,"value":"hf = new HFInference(hfapiKey)","pinned":false,"hidden":false,"mode":"ojs"},{"id":28,"value":"imgBlob = await canvasToBlob(photo).catch((err) => false)","pinned":false,"hidden":false,"mode":"ojs"},{"id":29,"value":"img2textRes = {\n if (imgBlob) {\n return hf.imageToText({\n data: imgBlob,\n model: \"nlpconnect/vit-gpt2-image-captioning\"\n });\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":30,"value":"textGenIn = `You are Sir David Attenborough. Narrate the text in quotes describing an image of the human as if it is a nature documentary. Make it snarky and funny. Don't repeat yourself. Make it extremely short. If I do anything remotely interesting, make a big deal about it!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":31,"value":"textGenRes = {\n if (img2textRes) {\n return hf.textGeneration({\n model: \"mistralai/Mistral-7B-Instruct-v0.1\",\n inputs: `${prompt}\\n\"${img2textRes.generated_text}\"`,\n parameters: { return_full_text: false, max_new_tokens: 100 }\n });\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":32,"value":"voices = {\n const options = {\n method: \"GET\",\n headers: {\n \"xi-api-key\": elevenlabsapiKey,\n \"Content-Type\": \"application/json\"\n }\n };\n\n return fetch(\"https://api.elevenlabs.io/v1/voices\", options)\n .then((response) => response.json())\n .then((json) => json.voices)\n .catch((err) => console.error(err));\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":33,"value":"function canvasToBlob(canvas, mimeType = 'image/png', qualityArgument) {\n return new Promise((resolve, reject) => {\n canvas.toBlob((blob) => {\n if (blob) {\n resolve(blob);\n } else {\n reject(new Error('Canvas to Blob conversion failed'));\n }\n }, mimeType, qualityArgument);\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":34,"value":"import { secret } from \"@tmcw/secret\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":36,"value":"import { tweet } from \"@observablehq/tweet\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":38,"value":"HFInference = (\n await import(\"https://cdn.jsdelivr.net/npm/@huggingface/inference@2.6.4/+esm\")\n).HfInference","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}