31:T619,viewof ctx = { const ctx = DOM.context2d(250, 250, 1); ctx.canvas.classList.add("draw-canvas"); ctx.rect(0,0,250,250); ctx.fill(); const points = [[82,76],[81,78],[75,90],[71,98],[67,105],[64,112],[62,117],[61,122],[60,125],[59,128],[59,132],[59,135],[58,139],[58,144],[58,147],[58,150],[58,153],[58,156],[61,161],[64,166],[70,170],[75,174],[79,176],[85,179],[93,181],[101,182],[112,182],[122,180],[134,177],[141,174],[148,169],[154,163],[159,157],[164,149],[168,143],[172,135],[177,127],[180,121],[181,119],[184,112],[185,106],[186,100],[186,95],[186,90],[186,84],[181,75],[175,68],[171,62],[167,60],[163,58],[160,56],[157,55],[151,54],[142,53],[135,53],[130,53],[127,54],[122,55],[120,56],[117,58],[114,59],[111,61],[108,64],[104,66],[101,69],[99,71],[98,71],[98,72],[97,72],[97,72],[96,73],[96,74],[93,77],[92,78],[90,81],[90,81]]; for (let i = 1; i < points.length;i++) { ctx.beginPath(); ctx.lineWidth = 25; ctx.lineJoin = ctx.lineCap = 'round'; ctx.moveTo(points[i-1][0], points[i-1][1]); ctx.lineTo(points[i][0], points[i][1]); ctx.strokeStyle = "white"; ctx.stroke(); } const root = html`

${ctx.canvas}

`; root.value = ctx; return root; }32:T437,inputs = Generators.observe(next => { const elm = d3.select(ctx.canvas); const downsample = downsampler(ctx, { dstSize: [28, 28], targetSize: [784, 1] }); const empty = new Float32Array(784); let lastX = 0, lastY = 0; let data = downsample(); next(data) const btnClear = d3.select(viewof ctx).select('.btn-clear'); btnClear.on("click", () => { ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height); ctx.beginPath(); ctx.rect(0, 0, ctx.canvas.width, ctx.canvas.height); ctx.fillStyle = 'black'; ctx.fill(); next(data = empty); }); const onDrag = () => { ctx.beginPath(); ctx.lineWidth = 25; ctx.lineJoin = ctx.lineCap = 'round'; ctx.moveTo(lastX, lastY); ctx.lineTo(d3.event.x, d3.event.y); ctx.strokeStyle = "white"; ctx.stroke(); lastX = d3.event.x, lastY = d3.event.y; next(data = downsample()); } elm.call(d3.drag().on("start", () => { lastX = d3.event.x, lastY = d3.event.y; d3.event.on("drag", onDrag); })); return () => elm.on("drag", null); })33:T56e,downsampler = (source, opts) => { let { dstSize, targetSize } = opts; targetSize = targetSize || dstSize; const round = x => (x + 0.49) << 0 const srcW = source.canvas.width, srcH = source.canvas.height; const dest = new ImageData(targetSize[0], targetSize[1]); const dstData = new Uint32Array(dest.data.buffer); const scaleX = dstSize[0] / srcW, scaleY = dstSize[1] / srcH; const rangeX = round(1 / scaleX), rangeY = round(1 / scaleY); const rangeSqr = rangeX * rangeY; return () => { const srcData = new Uint32Array(source.getImageData(0, 0, srcW, srcH).data.buffer); for (let dr = 0; dr < dstSize[1]; dr++) { for (let dc = 0; dc < dstSize[0]; dc++) { const srcIdx = round(dc / scaleX) + round(dr / scaleY) * srcW; let r = 0, g = 0, b = 0, a = 0; for (let sr = 0; sr < rangeX; sr++) { for (let sc = 0; sc < rangeY; sc++) { const p = srcData[srcIdx + sc + sr * srcW]; r += p << 24 >>> 24; g += p << 16 >>> 24; b += p << 8 >>> 24; a += p >>> 24; } } r = round(r / rangeSqr); g = round(g / rangeSqr); b = round(b / rangeSqr); a = round(a / rangeSqr); dstData[dr * dstSize[0] + dc] = a << 24 | b << 16 | g << 8 | r; } } return dest; } }34:T425,md` ### Papers and Tutorials * [Restricted Boltzmann Machines (RBM)](http://deeplearning.net/tutorial/rbm.html) * [An Introduction to Restricted Boltzmann Machines](https://link.springer.com/chapter/10.1007/978-3-642-33275-3_2) * [A Tutorial on Energy-Based Learning](http://yann.lecun.com/exdb/publis/pdf/lecun-06.pdf) * [A Practical Guide to Training Restricted Boltzmann Machines](https://www.cs.toronto.edu/~hinton/absps/guideTR.pdf) * [Restricted Boltzmann Machines pure numpy implementation](https://colab.research.google.com/github/stwind/notebooks/blob/master/rbm-raw.ipynb) ### Prior Arts * [Erkaman/regl-cnn: Digit recognition with Convolutional Neural Networks in WebGL](https://github.com/Erkaman/regl-cnn) * [3D Visualization of a Convolutional Neural Network](https://www.cs.ryerson.ca/~aharley/vis/conv/) * [Dreaming of names with RBMs](http://colinmorris.github.io/blog/dreaming-rbms) * [Musical TensorFlow, Part 1 - How to build an RBM in TensorFlow for making music](http://danshiebler.com/2016-08-10-musical-tensorflow-part-one-the-rbm/) `35:[{"id":"8d875be65c0693a6b106856389699eb7f3987415e24ce24d060f03bf21803fe597c2393ace888214e4bf796ebfd4c11f55528c98894a9de94f4b2ad6d569a89e","href":"https://static.observableusercontent.com/files/8d875be65c0693a6b106856389699eb7f3987415e24ce24d060f03bf21803fe597c2393ace888214e4bf796ebfd4c11f55528c98894a9de94f4b2ad6d569a89e","name":"group1-shard1of1@1.bin","type":"application/octet-stream","lastModified":1595990206918.747,"size":204096},{"id":"86ae29b0621f97bc5769b23f1917a98507476eda68f0309add6c0a8f7681bb7f7f92555da4053df4382d8f550ae8a8d486fafc99bb64739e51aa8edd0507d198","href":"https://static.observableusercontent.com/files/86ae29b0621f97bc5769b23f1917a98507476eda68f0309add6c0a8f7681bb7f7f92555da4053df4382d8f550ae8a8d486fafc99bb64739e51aa8edd0507d198","name":"rbm.png","type":"image/png","lastModified":1596333737614.523,"size":63238}] 2f:{"title":"Realtime Restricted Boltzmann Machines","cells":[{"id":0,"value":"md`# Realtime Restricted Boltzmann Machines`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2783,"value":"md`**Restricted Boltzmann Machines (RBM)** is a simplistic [generative model](https://en.wikipedia.org/wiki/Generative_model) that has only one hidden layer, which also is usually Bernoulli distributed (${tex`\\mathbf{h} \\in \\{0,1\\}^n`}). This simple structure makes it suitable for fast realtime inferences in browsers with WebGL.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3016,"value":"html`

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3022,"value":"md`I [trained](https://colab.research.google.com/github/stwind/notebooks/blob/master/rbm-tf.ipynb) a simple RBM with 64 hidden units on the MNIST 784 dataset with Tensorflow 2, and exported the model to [tfjs](https://www.tensorflow.org/js/) format. Since the model is simple enough, I can load and run it without using tfjs, instead just by reading the weights and doing all the computation in WebGL (with the handy [@thi.ng/webgl](http://thi.ng/webgl)).\n\nYou can draw in following canvas, see how the model detects hidden features and reconstructs (or metaphorically, dreams about) the image. It is not an effective model due to the simplicity, but hopefully still fun to play with.\n\n_(currently only desktop Chrome and Firefox are supported)_`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2688,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":2404,"value":"viewof viz = {\n const glCanvas = DOM.canvas(250, 250);\n const ctx = DOM.context2d(8 * 80, 8 * 25);\n const cap = html`

▲ Hidden features and the possibilities they are detected with

◀ Reconstructed image based on hidden features

`;\n \n const gl = glCanvas.getContext('webgl2');\n gl.getExtension('EXT_color_buffer_float');\n \n const root = html`

\n ${glCanvas}\n

\n ${ctx.canvas}\n ${cap}\n

`;\n root.value = { gl, ctx };\n return root;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2675,"value":"md`---\n## Code\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2595,"value":"md`### Runtime Variables`","pinned":false,"hidden":false,"mode":"ojs"},{"id":141,"value":"gl = viz.gl","pinned":false,"hidden":false,"mode":"ojs"},{"id":1870,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":1912,"value":"activations = {\n textures.V.bind();\n gl.texImage2D(textures.V.target, 0, textures.V.format, textures.V.size[0], textures.V.size[1], 0, gl.RED, textures.V.type, inputs);\n textures.V.unbind();\n \n fbos.hp.bind();\n gl.viewport(0, 0, textures.hp.size[0], textures.hp.size[1]);\n forward.textures = [textures.V, textures.W, textures.c];\n webgl.draw(forward);\n fbos.hp.unbind();\n\n return readActication();\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1973,"value":"readActication = texReader(textures.hp, fbos.hp)","pinned":false,"hidden":false,"mode":"ojs"},{"id":2587,"value":"md`### Render`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1131,"value":"renderImage = {\n activations;\n \n // sample\n fbos.h.bind();\n gl.viewport(0, 0, textures.h.size[0], textures.h.size[1]);\n bernoulli.uniforms.time = now % 1e6 * 1e-3;\n bernoulli.textures = [textures.hp];\n webgl.draw(bernoulli);\n fbos.h.unbind();\n \n // backward\n fbos.vp.bind();\n gl.viewport(0, 0, textures.vp.size[0], textures.vp.size[1]);\n backward.textures = [textures.h, textures.W, textures.b];\n webgl.draw(backward);\n fbos.vp.unbind();\n \n gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);\n gl.clearColor(0.0, 0.0, 0.0, 1.0);\n gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);\n \n show.textures = [textures.vp];\n webgl.draw(show);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2424,"value":"renderFeatures = {\n const w = 80, h = 25, s0 = 28, s1 = 20;\n const ctx = viz.ctx;\n ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);\n for (let i = 0; i < 8; i++) {\n for (let j = 0; j < 8; j++) {\n const p = activations[i * 8 + j];\n \n ctx.globalAlpha = Math.max(p, 0.2);\n ctx.drawImage(neurons.canvas, i * s0, j * s0, s0, s0, i * w, j * h, s1, s1);\n ctx.globalAlpha = Math.max(p, 0.3);\n ctx.fillStyle = \"hsl(0,0%,20%)\";\n ctx.font = \"10px 'Courier'\";\n ctx.fillText(p.toFixed(4), i * w + h, j * h + h / 2 + 1);\n ctx.globalAlpha = 1;\n }\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2600,"value":"md`### WebGL`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2644,"value":"md`#### Textures and FBOs`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1342,"value":"defTexture = (width, height, data = null) => webgl.defTexture(gl, {\n width, height,\n format: gl.R32F,\n type: gl.FLOAT,\n filter: gl.NEAREST,\n wrap: gl.CLAMP_TO_EDGE,\n image: data\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1466,"value":"textures = ({\n // input\n V: defTexture(784, 1),\n \n // variables\n W: defTexture(64, 784, weights.W),\n b: defTexture(784, 1, weights.b),\n c: defTexture(64, 1, weights.c),\n \n // output\n hp: defTexture(64, 1),\n h: defTexture(64, 1),\n vp: defTexture(784, 1)\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":225,"value":"fbos = ({\n hp: webgl.defFBO(gl, { tex: [textures.hp] }),\n h: webgl.defFBO(gl, { tex: [textures.h] }),\n vp: webgl.defFBO(gl, { tex: [textures.vp] }),\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1891,"value":"md`#### Shaders`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2925,"value":"defModel = (fs, uniforms) => webgl.compileModel(gl, {\n ...webgl.defQuadModel(),\n shader: webgl.defShader(gl, {\n vs: `\nvoid main() {\n v_uv = uv;\n gl_Position = vec4(position, 0.0, 1.0);\n}`,\n declPrefixes: { v: \"v_\" },\n attribs: {\n position: \"vec2\",\n uv: \"vec2\",\n },\n varying: { uv: \"vec2\" },\n fs, uniforms\n })\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1792,"value":"bernoulli = defModel(`\n${frags.random}\n${frags.bernoulli}\n\nvoid main() {\n float v = random(vec3(v_uv, time));\n float r = bernoulli(texture(P, v_uv).x, v);\n fragColor = vec4(r, 0.0, 0.0, 1.0);\n}`, {\n P: [\"sampler2D\", 0],\n time: \"float\",\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1098,"value":"forward = defModel(`\nconst float step = 1.0 / 784.0;\n\n${frags.sigmoid}\n\nvoid main() {\n float res = 0.0;\n\n${d3.range(784).map(\n i => `res += texture(X, vec2(step * ${i}., v_uv.y)).x * texture(W, vec2(v_uv.x, step * ${i}.)).x;`\n).join('\\n')}\n\n res += texture(B, vec2(v_uv.x, 0.0)).x;\n res = sigmoid(res);\n\n fragColor = vec4(res, 0.0, 0.0, 1.0);\n}`, {\n X: [\"sampler2D\", 0],\n W: [\"sampler2D\", 1],\n B: [\"sampler2D\", 2],\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1634,"value":"show = defModel(`\nconst float size = 28.0;\nconst float size2 = 1.0 / 784.0;\nconst float large = 200.0;\n\nfloat getV(vec2 uv) {\n float x = uv.x + uv.y * size;\n return texture(V, vec2(x, 0.0) * size2).x;\n}\n\nvoid main() {\n vec2 uv1 = floor(vec2(v_uv.x, 1. - v_uv.y) * large);\n vec2 uv2 = uv1 * size / large;\n vec2 frac = fract(uv2);\n vec2 uv3 = floor(uv2);\n\n vec2 tl = uv3;\n vec2 tr = min(vec2(tl.x + 1., tl.y), size - 1.);\n vec2 bl = min(vec2(tl.x, tl.y + 1.), size - 1.);\n vec2 br = min(tl + 1., size - 1.);\n\n // bilinear interpolation\n vec2 vl = vec2(getV(tl), getV(bl));\n vec2 vr = vec2(getV(tr), getV(br));\n vec2 vh = mix(vl, vr, frac.x);\n float v = mix(vh.x, vh.y, frac.y);\n\n fragColor = vec4(vec3(v), 1.0);\n}`, {\n V: [\"sampler2D\", 0],\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1308,"value":"backward = defModel(`\nconst float step = 1.0 / 64.0;\n\n${frags.sigmoid}\n\nvoid main() {\n float res = 0.0;\n\n${d3.range(64).map(\n i => `res += texture(X, vec2(step * ${i}., v_uv.y)).x * texture(W, vec2(step * ${i}., v_uv.x)).x;`\n).join('\\n')}\n\n res += texture(B, vec2(v_uv.x, 0.0)).x;\n res = sigmoid(res);\n\n fragColor = vec4(res, 0.0, 0.0, 1.0);\n}`, {\n X: [\"sampler2D\", 0],\n W: [\"sampler2D\", 1],\n B: [\"sampler2D\", 2],\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":541,"value":"copy = webgl.compileModel(gl, {\n ...webgl.defQuadModel(),\n shader: webgl.defShader(gl, webgl.FX_SHADER_SPEC_UV)\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1286,"value":"frags = ({\n sigmoid: `\nfloat sigmoid(float x) {\n\treturn 1./(1. + exp(-x));\n}`,\n relu: `\nfloat relu(float x) {\n return max(x, 0.0);\n}`,\n bernoulli: `\nfloat bernoulli(float p, float v) {\n return max(sign(p - v), 0.0);\n}`,\n random: `\nuint hashInt( uvec3 v )\n{\n uint x = v.x, y = v.y, z = v.z;\n x += x >> 11;\n x ^= x << 7;\n x += y;\n x ^= x << 3;\n x += z ^ ( x >> 14 );\n x ^= x << 6;\n x += x >> 15;\n x ^= x << 5;\n x += x >> 12;\n x ^= x << 9;\n return x;\n}\n\nfloat rand(uint h) {\n const uint mantissaMask = 0x007FFFFFu;\n const uint one = 0x3F800000u;\n\n h &= mantissaMask;\n h |= one;\n \n float r2 = uintBitsToFloat( h );\n return r2 - 1.0;\n}\n\nfloat random( vec3 f ) {\n return rand(hashInt(floatBitsToUint(f)));\n}`\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":2395,"value":"md`### Assets`","pinned":false,"hidden":false,"mode":"ojs"},{"id":53,"value":"neurons = {\n const data = weights.W;\n const dim = [8, 8], size = 28;\n const ctx = DOM.context2d(size * dim[0], size * dim[1], 1); \n const p = ctx.createImageData(size, size);\n \n for (let i = 0; i < dim[1]; i++) {\n for (let j = 0; j < dim[0];j++) {\n const k = i * dim[1] + j;\n const scale = d3.scaleLinear(d3.extent(d3.range(784),i=>data[i*64+k]), [0, 1]);\n \n for (let l = 0; l < 784; l++) {\n const d = scale(data[l * 64 + k]) * 255;\n p.data[l * 4 + 0] = p.data[l * 4 + 1] = p.data[l * 4 + 2] = d;\n p.data[l * 4 + 3] = 255;\n }\n ctx.putImageData(p, j * size, i * size);\n }\n }\n \n return ctx\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":134,"value":"md`### Model Weights`","pinned":false,"hidden":false,"mode":"ojs"},{"id":42,"value":"weights = {\n const buf = await FileAttachment(\"group1-shard1of1@1.bin\").arrayBuffer();\n \n let offset = 0;\n const specs = [[\"c\",[1,64]],[\"W\",[784,64]],[\"b\",[1,784]]];\n const res = {};\n for (const [name, shape] of specs) {\n const size = shape[0] * shape[1] * 4;\n res[name] = new Float32Array(buf.slice(offset, offset + size));\n offset += size;\n }\n return res;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2398,"value":"md`### Utilities`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1970,"value":"texReader = (tex, fbo) => {\n const [w, h] = tex.size;\n const n = w * h;\n const tmp = new Float32Array(n * 4);\n const res = new Float32Array(n);\n \n return () => {\n fbo.bind();\n gl.readPixels(0, 0, w, h, gl.RGBA, gl.FLOAT, tmp);\n fbo.unbind();\n for (let i = 0; i < n; i++)\n res[i] = tmp[i * 4]\n \n return res;\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1867,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":1776,"value":"fps = {\n let t = performance.now(), delta = 0;\n const last = [t];\n return cell => {\n last.length >= 10 && last.shift();\n t = performance.now();\n delta = (t - last[0]) / last.length;\n last.push(t);\n return Math.round(1000 / delta);\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":5,"value":"md`### Appendix`","pinned":false,"hidden":false,"mode":"ojs"},{"id":13,"value":"d3 = require(\"d3@5\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"webgl = require('https://bundle.run/@thi.ng/webgl@3.0.4')","pinned":false,"hidden":false,"mode":"ojs"},{"id":149,"value":"idRef = md`---\n## References`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2654,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}