31:T5b7,md`# Soundcloud Reactive Audio Visualizer Music Video Generator This is a tool for generating an acceptable music video from any Soundcloud track. Scroll down to find customize the audio input URL and put whatever you like. Press record to generate a downloadable _webm_ for sharing. Or just chill out by going fullscreen. More options are scattered throughout the page (e.g. video resolution or customization of visuals). I hope it might help a few artists with video production. This app was developed to help me learn a few things before building a full audio visual mixing demoscene system. In particular, you can click under the hood, fork the code and create your own experiences. Its licensed as ISC so you can do what you like with the code and output, no need to credit me, though I would be honoured. This demos the following technologies 1. Using the Web Audio API to extract a waveform to drive reactive visuals 1. Using Three.js effects composer to create multi layed visuals with post processing effects 1. Importing audio from external sources (Soundcloud) 1. Setting up an high performance Web Audio API graph. 1. Extracting a combined audio and visual into a single video file. 1. Using the wake lock API to prevent the host sleeping when fullscreened. 1. Inserting HTML into a Three.js scene. These are foundational techniques for doing even cooler things in the browser! Stay tuned! Use the comment feature to get in touch. ` 32:T941,{ var controls = null; const rewind = () => { stop() source.mediaElement.currentTime = 0; } const play = () => { console.log("play") context.resume() source.connect(analyser); analyser.connect(context.destination); source.mediaElement.play(); } const stop = () => { console.log("stop") source.mediaElement.pause(); } const fullscreen = async (event) => { controls.parentElement.nextElementSibling.requestFullscreen(); await navigator.wakeLock.request('screen'); } const record = async (event) => { document.getElementById("downloadlink").innerHTML = "" var recordedChunks = []; return new Promise(function (res, rej) { const canvas = document.getElementById("visualizer"); const video = canvas.captureStream(); const audio = context.createMediaStreamDestination(); analyser.connect(audio) const stream = new MediaStream([video.getVideoTracks()[0], audio.stream.getAudioTracks()[0]]) const mediaRecorder = new MediaRecorder(stream, { mimeType: "video/webm; codecs=vp9" }); mediaRecorder.start(); play() function stopRecorder() { mediaRecorder.stop(); source.mediaElement.removeEventListener("pause", stopRecorder, true); source.mediaElement.removeEventListener('ended', stopRecorder, true); } source.mediaElement.addEventListener("pause", stopRecorder, true); source.mediaElement.addEventListener('ended', stopRecorder, true); mediaRecorder.ondataavailable = function (event) { console.log("Chunk recorded", event) recordedChunks.push(event.data); } mediaRecorder.onstop = function (event) { console.log("stopped") var blob = new Blob(recordedChunks, { type: "video/webm" }); var url = URL.createObjectURL(blob); const downloadLink = document.getElementById("downloadlink") downloadLink.href = url downloadLink.innerHTML = "download" } }) } controls = html` ` return controls; } 33:T80f,{ const width = resolution.width; const height = resolution.height; var camera = new THREE.OrthographicCamera(width / -2, width / 2, height / 2, height / -2, 0, height / -2); const renderer = new THREE.WebGLRenderer(); const renderTarget = new THREE.WebGLRenderTarget( width, height, { depthBuffer: false }); renderer.setClearColor(new THREE.Color(0x000000, 1.0)); renderer.domElement.id = "visualizer" invalidation.then(() => renderer.dispose()); renderer.setSize(width, height); const composer = new EffectComposer( renderer, renderTarget); const scene = new THREE.Scene(); scene.background = new THREE.Color(0x000000); scene.add(buffers.line); const renderScene = new RenderPass( scene, camera ); const bloomPass = new UnrealBloomPass( new THREE.Vector2( width, height), 1.5, 0.4, 0.85 ); bloomPass.threshold = 0; bloomPass.strength = 7; bloomPass.radius = 1; composer.addPass( renderScene ); composer.addPass( bloomPass ); { // Credits const texture = new THREE.TextureLoader().load(title.img); texture.needsUpdate = true; var material = new THREE.MeshBasicMaterial({ map: texture, side: THREE.DoubleSide, transparent: true }); const geometry = new THREE.PlaneBufferGeometry(title.width, title.height); const mesh = new THREE.Mesh( geometry, material ); mesh.position.set( width/2.0 - title.width/2.0, - height/2.0 + title.height / 2.0, 0) const titleScene = new THREE.Scene(); titleScene.add(mesh); const renderTitle = new RenderPass( titleScene, camera); renderTitle.clear = false; composer.addPass( renderTitle ); const copyPass = new ShaderPass( CopyShader ); copyPass.renderToScreen = true; composer.addPass( copyPass ); } const t_start = performance.now(); while (true) { mutable t = performance.now() - t_start composer.render(); yield renderer.domElement; } }34:T6fb,viewof source = render(({ useSetter }) => { const [searchInput, setSearchInput] = useState( decodeURIComponent( ( location.hash || "#https://soundcloud.com/dexcelldnb/dexcell-something-from-nothing-vip" ).substring(1) ) ); const [search, setSearch] = useState(searchInput); const [track, setTrack] = useState(null); const [media, setMedia] = useState(null); useEffect(() => { SC.resolve(search) .then((track) => setTrack(track)) .catch((e) => setTrack(null)); }, [search]); useEffect(() => { if (track === null) { setMedia(null); } else { const audio = new Audio(); audio.crossOrigin = "anonymous"; audio.src = track.stream_url + "?client_id=" + SC.client_id; setMedia(context.createMediaElementSource(audio)); } }, [track]); useEffect(() => { // change the top url if (media !== null) document.getElementById("audioviz").click(); }, [media]); const view = media || {}; view.search = search; view.track = track; useSetter(view); return jsx` setSearchInput(e.target.value)} onKeyDown=${(e) => e.key === "Enter" && setSearch(searchInput)} /> ${ track && jsx`

` } `; })35:T66a,draw = { t; // force this cell to update when time increments if (source.mediaElement.paused) return; const width = resolution.width; const height = resolution.height; const points = []; const colors = []; const point = new THREE.Vector3(); const color = new THREE.Color(); // Rotating colors color.setRGB( Math.cos(t * 0.00011) * 0.5 + 1, Math.cos(t * 0.002) * 0.5 + 1, Math.cos(t * 0.003) * 0.5 + 1 ); // Positions driven by audio analyser analyser.getByteTimeDomainData(buffers.dataArray); for (let i = 0; i < buffers.dataArray.length; i++) { var mod = 1; // Math.sin(i * Math.PI / buffers.dataArray.length) var y = ((buffers.dataArray[i] * height) / 256.0 - height / 2.0) * mod; var x = -width / 2 + (i * width) / buffers.dataArray.length; points.push(x, y, 0); colors.push(color.r, color.g, color.b); } // loop wave form round to enter the view from the left again points.push(width * 2, 0, 0); points.push(width * 2, height * 2, 0); points.push(-width * 2, height * 2, 0); points.push(-width * 2, 0, 0); // The main stratergy is to shift all the buffers down, and append new data points to the end buffers.positions.set(buffers.positions.array.slice(points.length)); buffers.positions.set(points, buffers.positions.count * 3 - points.length); buffers.positions.needsUpdate = true; // Decay the colors by a factor so the old lines fade out buffers.colors.set( buffers.colors.array.slice(colors.length).map((x) => x * 0.5), 0 ); buffers.colors.set(colors, buffers.colors.count * 3 - colors.length); buffers.colors.needsUpdate = true; }36:[{"id":"987350cd53fddf223d46539654cd6e07de3a5d182eca513c0bac27d18f25547cc61073dae7e52d9f06ca045fda5e0867fae249441dd892802f27a91673a67bfe","href":"https://static.observableusercontent.com/files/987350cd53fddf223d46539654cd6e07de3a5d182eca513c0bac27d18f25547cc61073dae7e52d9f06ca045fda5e0867fae249441dd892802f27a91673a67bfe","name":"positions.json","type":"application/json","lastModified":1600026336004.995,"size":105942},{"id":"0ca7672d7694da0962dad33a84de2e6e0c9199f8d7251a26795f167fcfaebcb672849b35f1399140cf878a9fe42160a4f709746e976f02c730f4f49901f94b7d","href":"https://static.observableusercontent.com/files/0ca7672d7694da0962dad33a84de2e6e0c9199f8d7251a26795f167fcfaebcb672849b35f1399140cf878a9fe42160a4f709746e976f02c730f4f49901f94b7d","name":"colors.json","type":"application/json","lastModified":1600026346863.502,"size":232449}] 2f:{"title":"Soundcloud Reactive Audio Visualizer Music Video Generator","cells":[{"id":0,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":79,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":183,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":1140,"value":"resolution = JSON.parse(resolution_str)","pinned":true,"hidden":false,"mode":"ojs"},{"id":1115,"value":"viewof resolution_str = select([\n {label: `screen (${width}, ${height})`, value: JSON.stringify({width:width, height:height})},\n {label: \"1080p (1920x1080)\", value: JSON.stringify({width:1920, height:1080})},\n ]);","pinned":false,"hidden":false,"mode":"ojs"},{"id":480,"value":"md`Create a global Audio Context`","pinned":false,"hidden":false,"mode":"ojs"},{"id":152,"value":"context = new AudioContext()","pinned":false,"hidden":false,"mode":"ojs"},{"id":491,"value":"md`Create an Media Element Source. We use the Sound Cloud API to get a stream URL based on the URL provided,\nthen build a media element around it. Choose your own song by pasting the link in the box, this will update the URL, so you can share this page with your favourite song already preloaded`","pinned":false,"hidden":false,"mode":"ojs"},{"id":277,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":1099,"value":"md`Its simplest to use normal HTML for the subtitles`","pinned":false,"hidden":false,"mode":"ojs"},{"id":800,"value":"titleHtml = {\n const view = html`\n \n

${source.track ? source.track.user.username: \"artist\"}

${source.track ? source.track.title: \"title\"}

`\n return view\n} ","pinned":false,"hidden":false,"mode":"ojs"},{"id":1105,"value":"md`We use a technology called dom-to-image to turn normal HTML into an image we can insert into the Three.js scene as a texture`","pinned":false,"hidden":false,"mode":"ojs"},{"id":813,"value":"title = {\n titleHtml\n const title = document.getElementById('titleHtml')\n return {\n element: title,\n width: title.offsetWidth ,\n height: title.offsetHeight,\n img: await dom2img.toPng(title)\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":791,"value":"dom2img = require('https://bundle.run/dom-to-image@2.6.0')","pinned":false,"hidden":false,"mode":"ojs"},{"id":485,"value":"md`Create an analyser, which is able to capture waveforms as they are processed on the audio graph.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":154,"value":"analyser = context.createAnalyser()","pinned":false,"hidden":false,"mode":"ojs"},{"id":365,"value":"source_debug = source","pinned":false,"hidden":false,"mode":"ojs"},{"id":501,"value":"md`Buffers are THREE.js geometry holders which are static. We will be fiddling with the geometry based on the sound waveform, but the scene graph remains static throughout`","pinned":false,"hidden":false,"mode":"ojs"},{"id":177,"value":"buffers = {\n // Buffers hold the static geometry which we manipulate elsewhere.\n const buffers = {\n positions: new THREE.Float32BufferAttribute( (MAX_SEGMENTS) * 3, 3 ),\n colors: new THREE.Float32BufferAttribute( (MAX_SEGMENTS) * 3, 3 ),\n lineGeometry: new THREE.BufferGeometry(),\n lineMaterial: new THREE.LineBasicMaterial( { color: 0xffffff, vertexColors: true} ),\n dataArray: new Uint8Array(analyser.frequencyBinCount),\n }\n buffers.positions.set(init.positions)\n buffers.colors.set(init.colors)\n buffers.lineGeometry.setAttribute('position', buffers.positions)\n buffers.lineGeometry.setAttribute('color', buffers.colors)\n buffers.line = new THREE.Line( buffers.lineGeometry, buffers.lineMaterial ) \n buffers.lineGeometry.setDrawRange(0, MAX_SEGMENTS);\n return buffers;\n};","pinned":false,"hidden":false,"mode":"ojs"},{"id":508,"value":"md`Our visualizer animation loop bumps the time variable _t_, causing dependant code to run`","pinned":false,"hidden":false,"mode":"ojs"},{"id":191,"value":"mutable t = 0.0","pinned":false,"hidden":false,"mode":"ojs"},{"id":512,"value":"md`Our core drawing code. When _t_ is bumped by rendering, the draw code runs. We use the data from the Audio Analyser to updating the line geometry`","pinned":false,"hidden":false,"mode":"ojs"},{"id":175,"value":"$35","pinned":true,"hidden":false,"mode":"ojs"},{"id":185,"value":"height = window.screen.height","pinned":false,"hidden":false,"mode":"ojs"},{"id":187,"value":"MAX_SEGMENTS = 1024*4;","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"SC = {\n const sc = await require('soundcloud');\n sc.client_id = \"95f22ed54a5c297b1c41f72d713623ef\"\n sc.initialize({\n client_id: sc.client_id,\n redirect_uri: 'https://example.com/callback'\n });\n return sc\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":181,"value":"THREE_VERSION = \"0.112.1\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":179,"value":"THREE = require(`three@${THREE_VERSION}/build/three.min.js`);","pinned":false,"hidden":false,"mode":"ojs"},{"id":189,"value":"EffectComposer = (await import(`https://unpkg.com/three@${THREE_VERSION}/examples/jsm/postprocessing/EffectComposer.js?module`)).EffectComposer","pinned":false,"hidden":false,"mode":"ojs"},{"id":195,"value":"UnrealBloomPass = (await import(`https://unpkg.com/three@${THREE_VERSION}/examples/jsm/postprocessing/UnrealBloomPass.js?module`)).UnrealBloomPass","pinned":false,"hidden":false,"mode":"ojs"},{"id":193,"value":"RenderPass = (await import(`https://unpkg.com/three@${THREE_VERSION}/examples/jsm/postprocessing/RenderPass.js?module`)).RenderPass","pinned":false,"hidden":false,"mode":"ojs"},{"id":966,"value":"ShaderPass = (await import(`https://unpkg.com/three@${THREE_VERSION}/examples/jsm/postprocessing/ShaderPass.js?module`)).ShaderPass","pinned":false,"hidden":false,"mode":"ojs"},{"id":973,"value":"CopyShader = (await import(`https://unpkg.com/three@${THREE_VERSION}/examples/jsm/shaders/CopyShader.js?module`)).CopyShader","pinned":false,"hidden":false,"mode":"ojs"},{"id":104,"value":"import {html} from \"@observablehq/htl\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":1117,"value":"import {select} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":199,"value":"import {checkbox} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":273,"value":"import {render, component, jsx, memo, forwardRef, React, ReactDOM, createElement, Children, createRef, createContext, lazy, Fragment, StrictMode, Suspense, cloneElement, useCallback, useContext, useEffect, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useRef, useState} from '@j-f1/react-16'","pinned":false,"hidden":false,"mode":"ojs"},{"id":524,"value":"init = ({\n positions: JSON.parse(await FileAttachment(\"positions.json\").text()),\n colors: JSON.parse(await FileAttachment(\"colors.json\").text())\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1179,"value":"import { footer } from \"@tomlarkworthy/footer\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":1182,"value":"footer","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W36","stdlib":"1","titling":"auto"}

${ track.user.username }

${track.title}

${source.track ? source.track.user.username: \"artist\"}

${source.track ? source.track.title: \"title\"}