imageData = { const width = streamWidth; const height = streamHeight; const frameGrab = DOM.context2d(streamWidth, streamHeight, 1); const drawCtx = DOM.context2d(streamWidth, streamHeight, 1); drawCtx.canvas.style="width:100%;height:100%;"; // horizontal contrast: sum the squares of RGB between horizontally adjacent pixels const sobel = new Float64Array(width * height); const sobelX = new Float64Array(width * height); const sobelY = new Float64Array(width * height); const width4 = width * 4; const totalSplits = spliterations; const totalRects = 4 ** totalSplits // four points times 4 to the power of totalSplits const rectangles = new Float32Array(4 * totalRects); const newRectangles = new Float32Array(4 * totalRects); // three colors times 4 to the power of totalSplits const colors = new Float32Array(3 * totalRects); const newColors = new Float32Array(3 * totalRects); // initiate rectangles as noise for(let i = 0; i < totalRects; i++){ let x = Math.random() * (width - 1); let y = Math.random() * (height - 1); rectangles[i * 4] = x; rectangles[i * 4 + 1] = y; rectangles[i * 4 + 2] = x+1; rectangles[i * 4 + 3] = y+1; colors[i * 3] = Math.random() * 255; colors[i * 3 + 1] = Math.random() * 255; colors[i * 3 + 2] = Math.random() * 255; } const totalFrames = 64; let frames = totalFrames - 1; // it's faster to multiply than to divide let frameDiv = 1 / totalFrames; while(startStream && stream){ // grab a new frame every 64 frames, interpolate otherwise frames++; if (frames >= totalFrames) { frames = 0; frameGrab.drawImage(stream, 0, 0, width, height); const {data: rgba} = frameGrab.getImageData(0, 0, width, height); for(let y = 0, lineRGBA = 0, lineSobel = 0; y < height - 1; y++) { let x4 = 0; let r0 = rgba[x4 + lineRGBA]; let g0 = rgba[x4 + lineRGBA + 1]; let b0 = rgba[x4 + lineRGBA + 2]; for (let x = 0, pdr = 0, pdg = 0, pdb = 0; x < width - 1; x++) { let i = x4 + lineRGBA + 4; let r1 = rgba[i]; let g1 = rgba[i + 1]; let b1 = rgba[i + 2]; let dr = r1 - r0; let dg = g1 - g0; let db = b1 - b0; dr *= dr; dg *= dg; db *= db; sobelX[x + lineSobel] = (dr + pdr + dg + pdg + db + pdb); pdr = dr; pdg = dg; pdb = db; r0 = r1; g0 = g1; b0 = b1; x4 += 4; } lineRGBA += width4; lineSobel += width; } for (let x = 0, x4 = 0; x < width - 1; x++) { let r0 = rgba[x4]; let g0 = rgba[x4 + 1]; let b0 = rgba[x4 + 2]; for(let y = 0, pdr = 0, pdg = 0, pdb = 0, lineRGBA = width4, lineSobel = 0; y < height - 1; y++) { let i = x4 + lineRGBA; let r1 = rgba[i + 3]; let g1 = rgba[i + 4]; let b1 = rgba[i + 5]; let dr = r1 - r0; let dg = g1 - g0; let db = b1 - b0; dr *= dr; dg *= dg; db *= db; sobelY[x + lineSobel] = (dr + pdr + dg + pdg + db + pdb); pdr = dr; pdg = dg; pdb = db; r0 = r1; g0 = g1; b0 = b1; lineRGBA += width4; lineSobel += width; } x4 += 4; } sumXSobel.update(sobelX); sumYSobel.update(sobelY); sumRGBA.update(rgba); // split rectangles repeatedly starting from a rectangle as big as the entire image newRectangles[0] = 0; // x0 newRectangles[1] = 0; // y0 newRectangles[2] = width-1; // x1 newRectangles[3] = height-1; // y1 // Yes, this is a bit nasty. Basically, every iteration we quadruple // the number of rectangles and throw away the old coordinates. // So to avoid pointless GC pressure we save the rectangle // coordinates in one big array that is initialized at the final nr // of coordinates. Then we split from back-to-front to avoid overwriting // the old coordinates before we can throw them away. for(let n = 0; n < totalSplits; n++){ for(let i = (4**n); i > 0; i--) { let j = i * 4; let k = i * 16; const y1 = newRectangles[--j]; const x1 = newRectangles[--j]; const y0 = newRectangles[--j]; const x0 = newRectangles[--j]; let cx = sumXSobel.cx(x0, y0, x1, y1) + 1; if (cx < x0 || cx > x1) cx = 0.5 * (x0 + x1); let cy1 = sumYSobel.cy(x0, y0, cx, y1) + 1; let cy2 = sumYSobel.cy(cx, y0, x1, y1) + 1; if (cy1 < y0 || cy1 > y1) cy1 = 0.5 * (y0 + y1); if (cy2 < y0 || cy2 > y1) cy2 = 0.5 * (y0 + y1); newRectangles[--k] = y1; newRectangles[--k] = x1; newRectangles[--k] = cy2; newRectangles[--k] = cx; newRectangles[--k] = y1; newRectangles[--k] = cx; newRectangles[--k] = cy1; newRectangles[--k] = x0; newRectangles[--k] = cy2; newRectangles[--k] = x1; newRectangles[--k] = y0; newRectangles[--k] = cx; newRectangles[--k] = cy1; newRectangles[--k] = cx; newRectangles[--k] = y0; newRectangles[--k] = x0; } } for(let i = 0; i < totalRects; i++) { const rIdx = i * 4; const cIdx = i * 3; const x0 = newRectangles[rIdx]; const y0 = newRectangles[rIdx+1]; const x1 = newRectangles[rIdx+2]; const y1 = newRectangles[rIdx+3]; const [r, g, b] = sumRGBA.color(x0, y0, x1, y1); newColors[cIdx] = r; newColors[cIdx+1] = g; newColors[cIdx+2] = b; } } const newWeight = 0.0625; const oldWeight = 1 - newWeight; const dx = streamWidth / (2 ** totalSplits); const dy = streamHeight / (2 ** totalSplits); for(let i = 0; i < totalRects; i++) { const rIdx = i * 4; const cIdx = i * 3; const pframes = totalFrames - frames; const x0 = rectangles[rIdx] * oldWeight + newRectangles[rIdx] * newWeight; const y0 = rectangles[rIdx + 1] * oldWeight + newRectangles[rIdx + 1] * newWeight; const x1 = rectangles[rIdx + 2] * oldWeight + newRectangles[rIdx + 2] * newWeight; const y1 = rectangles[rIdx + 3] * oldWeight + newRectangles[rIdx + 3] * newWeight; rectangles[rIdx] = x0; rectangles[rIdx + 1] = y0; rectangles[rIdx + 2] = x1; rectangles[rIdx + 3] = y1; const r = colors[cIdx] * oldWeight + newColors[cIdx] * newWeight; const g = colors[cIdx + 1] * oldWeight + newColors[cIdx + 1] * newWeight; const b = colors[cIdx + 2] * oldWeight + newColors[cIdx + 2] * newWeight; colors[cIdx] = r; colors[cIdx + 1] = g; colors[cIdx + 2] = b; drawCtx.fillStyle = `rgba(${Math.round(r)},${Math.round(g)},${Math.round(b)}, 1)`; if (forceToGrid) { // Because of the way in which we split the rectangles, we end up with an arrangement in Z-order // This wasn't planned but it is somewhat convenient I guess (if you know how Z-order works) const x = (i&1) + ((i>>>1)&2) + ((i>>>2)&4) + ((i>>>3)&8) + ((i>>>4)&16) + ((i>>>5)&32) + ((i>>>6)&64) + ((i>>>7)&128) + ((i>>>8)&256) + ((i>>>9)&512) + ((i>>>10)&1024); const y = ((i>>>1)&1) + ((i>>>2)&2) + ((i>>>3)&4) + ((i>>>4)&8) + ((i>>>5)&16) + ((i>>>6)&32) + ((i>>>7)&64) + ((i>>>8)&128) + ((i>>>9)&256) + ((i>>>10)&512) + ((i>>>11)&1024); drawCtx.fillRect(x * dx, y * dy, dx, dy); } else { drawCtx.fillRect(x0, y0, x1-x0, y1 - y0); } } yield drawCtx.canvas; } yield FileAttachment("Screenshot_20191101_174845.png").image(); }