31:T684,{ // TODO draw head vertex // TODO pick most accurate starting frame var image = frame.image; const context = DOM.canvas(width, image.height).getContext('2d'); if (userConfig.includes("drawgif")) { context.drawImage(image, 0, 0, image.width, image.height); } else { context.fillStyle = "black"; context.fillRect(0, 0, image.width, image.height); } yield context.canvas; context.fillStyle = context.strokeStyle = "yellow"; let pose = poses[frame.index]; if (userConfig.includes("smoothing")) { pose = smoothedPoses[frame.index]; } if (typeof pose !== 'undefined') { pose.keypoints.forEach((d, i) => { // TODO get average of head and draw one neck/head if (i <= 4) return; context.beginPath(); context.arc(d.position.x, d.position.y, config.size, 0, 2 * Math.PI); context.fill(); }); var adjacentKeyPoints = posenet.getAdjacentKeyPoints(pose.keypoints, 0); adjacentKeyPoints.forEach(keypoints => { context.beginPath(); context.moveTo(keypoints[0].position.x, keypoints[0].position.y); context.lineTo(keypoints[1].position.x, keypoints[1].position.y); context.lineWidth = config.size; context.stroke(); }); } context.fillStyle = "white"; context.font = 'bold 25px monospace'; if (loadingFrames) { context.fillText('Loading Gif Frames.', 5, 25); } else if (loadingPosenet) { context.fillText('Loading PoseNet....', 5, 25); } else if (loadingPoses) { context.fillText('Generating Poses...', 5, 25); } else if (loadingSmoothedPoses) { context.fillText('Smoothing Poses....', 5, 25); } yield context.canvas; }32:Tae7,{ var height = 1200; // TODO: why does the last frame have such bad tracking compared to the first frame? const canvas = DOM.canvas(width, height); const ctx = canvas.getContext('2d'); function render() { ctx.setTransform(1, 0, 0, 1, 0, 0); ctx.clearRect(0, 0, width, height); frames.forEach((f, i) => { let image = f.image; ctx.translate(i == 0 ? 0 : image.width / 2, 0); if (i % 6 == 0 && i != 0) { ctx.translate(-image.width * 3, image.height / 2); } ctx.drawImage( image, 0, 0, config.thumbnail.width, config.thumbnail.height ); let pose = undefined; if (userConfig.includes("smoothing")) { pose = smoothedPoses[i]; } // just draw the default pose if the smoothedPose isn't generated yet if (typeof pose === 'undefined') { pose = poses[i]; } var transform = ctx.getTransform(); if (typeof pose !== 'undefined') { pose.transform = { x: transform.e, y: transform.f }; ctx.fillStyle = ctx.strokeStyle = "yellow"; var adjacentKeyPoints = posenet.getAdjacentKeyPoints(pose.keypoints, 0); adjacentKeyPoints.forEach(keypoints => { ctx.beginPath(); ctx.moveTo( keypoints[0].position.x * config.thumbnail.scale, keypoints[0].position.y * config.thumbnail.scale ); ctx.lineTo( keypoints[1].position.x * config.thumbnail.scale, keypoints[1].position.y * config.thumbnail.scale ); ctx.lineWidth = config.size * config.thumbnail.scale; ctx.stroke(); }); // draw keypoints on top of links pose.keypoints.forEach((d, ki) => { // TODO get average of head and draw one neck/head if (ki <= 4) return; let override = overrides[`${i}:${ki}`]; let position = {}; if (typeof override !== 'undefined') { console.log('overriding', override); d.position.x = override.x; d.position.y = override.y; } if (userConfig.includes("error") && d.score < 0.25) { ctx.fillStyle = ctx.strokeStyle = "red"; } ctx.beginPath(); ctx.arc( d.position.x * config.thumbnail.scale, d.position.y * config.thumbnail.scale, config.size * config.thumbnail.scale, 0, 2 * Math.PI ); ctx.stroke(); ctx.fill(); }); } }); } d3.select(ctx.canvas).call( drag(userConfig.includes("smoothing") ? smoothedPoses : poses).on( "start.render drag.render end.render", render ) ); render(); yield ctx.canvas; }33:T4ca,drag = poses => { let transform = {}; // Choose the circle that is closest to the pointer for dragging. function dragsubject() { let subject = null; posesLoop: for (const [i, p] of poses.entries()) { for (const [ki, kp] of p.keypoints.entries()) { if (ki <= 4) continue; let d = Math.hypot( d3.event.x - p.transform.x - kp.position.x * config.thumbnail.scale, d3.event.y - p.transform.y - kp.position.y * config.thumbnail.scale ); if (d < 10) { subject = `${i}:${ki}`; transform = p.transform; break posesLoop; } } } return subject; } function dragged() { let override = { x: Math.max( 0, Math.min( config.thumbnail.width / config.thumbnail.scale, (d3.event.x - transform.x) / config.thumbnail.scale ) ), y: Math.max( 0, Math.min( config.thumbnail.height / config.thumbnail.scale, (d3.event.y - transform.y) / config.thumbnail.scale ) ) }; mutable overrides[d3.event.subject] = override; } return d3 .drag() .subject(dragsubject) .on("drag", dragged); }34:T422,smoothedPoses = { // yield early so the canvas drawing isn't blocked and loads progressively yield []; mutable loadingSmoothedPoses = true; console.log("smoothing poses"); let sp = []; yield sp; // min poses length must be greater than 2*size+1 if (poses.length <= 5) { yield sp; return sp; } sp = JSON.parse(JSON.stringify(poses)); yield sp; for (let i = 0; i < poses[0].keypoints.length; i++) { let keypoint = poses[0].keypoints[i]; let xs = []; let ys = []; for (let i2 = 0; i2 < poses.length; i2++) { xs.push(poses[i2].keypoints[i].position.x); ys.push(poses[i2].keypoints[i].position.y); } let smoothXs = Taira.smoothen(xs, Taira.ALGORITHMS.GAUSSIAN, 0, 0, true); let smoothYs = Taira.smoothen(ys, Taira.ALGORITHMS.GAUSSIAN, 0, 0, true); for (let i2 = 0; i2 < poses.length; i2++) { sp[i2].keypoints[i].position.x = smoothXs[i2]; sp[i2].keypoints[i].position.y = smoothYs[i2]; } yield sp; } mutable loadingSmoothedPoses = false; yield sp; }35:[] 2f:{"title":"PoseNet Spinning Dancer","cells":[{"id":0,"value":"md`# PoseNet Spinning Dancer \n\nGenerates poses for the [Spinning Dancer](https://en.wikipedia.org/wiki/Spinning_Dancer) ([gif source](https://en.wikipedia.org/wiki/File:Spinning_Dancer_Optical_Illusion_%282040_frames%29.ogv)) using the TensorFlow.js [PoseNet](https://github.com/tensorflow/tfjs-models/tree/master/posenet) MobileNetV1 model and smoothes the animation using [taira](https://github.com/petoem/taira).\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":1459,"value":"viewof userConfig = checkbox({\n options: [\n { value: \"drawgif\", label: \"Draw Gif\" },\n {\n value: \"smoothing\",\n label: \"Smooth out keypoint movement between frames\"\n },\n { value: \"error\", label: \"Highlight low confidence keypoints\" }\n ],\n value: [\"drawgif\", \"smoothing\"]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1436,"value":"md`The verticies below can be dragged to update the animation above`","pinned":false,"hidden":false,"mode":"ojs"},{"id":596,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":753,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":1585,"value":"md`## Asynchronously Load the Gif Frames`","pinned":false,"hidden":false,"mode":"ojs"},{"id":249,"value":"frame = {\n let i = 0;\n while (true) {\n let index = i % frames.length;\n if (i != 0) await Promises.delay(25);\n ++i;\n yield frames[index];\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":232,"value":"frames = {\n mutable loadingFrames = true;\n var f = [];\n var gf = await gifFrames({\n url:\n \"https://upload.wikimedia.org/wikipedia/commons/2/21/Spinning_Dancer.gif\",\n frames: 'all',\n outputType: 'canvas',\n cumulative: true\n });\n for (let i = 0; i < gf.length; i++) {\n f.push({ index: i, image: gf[i].getImage() });\n yield f;\n }\n\n mutable loadingFrames = false;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1361,"value":"md`## Poses Loaded Asynchronously\n\nhttps://observablehq.com/@observablehq/introduction-to-asynchronous-iteration`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1321,"value":"poses = {\n // yield early so the canvas drawing isn't blocked and loads progressively\n yield [];\n\n mutable loadingPoses = true;\n\n console.log(\"estimating poses\");\n\n let poses = [];\n\n for (let f of frames) {\n const pose = await posenetModel.estimateSinglePose(f.image, {\n maxDetections: 1\n });\n poses.push(pose);\n yield poses;\n }\n\n mutable loadingPoses = false;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1652,"value":"posenetModel = {\n // yield early so the canvas drawing isn't blocked and loads progressively\n yield null;\n\n mutable loadingPosenet = true;\n\n console.log(\"loading posenet model\");\n\n yield await posenet\n .load({\n architecture: 'MobileNetV1',\n outputStride: 16,\n inputResolution: 513,\n multiplier: 0.75\n })\n .then(d => {\n mutable loadingPosenet = false;\n return d;\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":439,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":1417,"value":"md`## Variables`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1168,"value":"mutable loadingPosenet = true","pinned":false,"hidden":false,"mode":"ojs"},{"id":1374,"value":"mutable loadingPoses = true","pinned":false,"hidden":false,"mode":"ojs"},{"id":1468,"value":"mutable loadingSmoothedPoses = true","pinned":false,"hidden":false,"mode":"ojs"},{"id":1600,"value":"mutable loadingFrames = true","pinned":false,"hidden":false,"mode":"ojs"},{"id":841,"value":"mutable overrides = []","pinned":false,"hidden":false,"mode":"ojs"},{"id":617,"value":"config = {\n let tscale = .5;\n let image = frames[0].image;\n return {\n size: 5,\n thumbnail: {\n width: image.width * tscale,\n height: image.height * tscale,\n scale: tscale\n }\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1113,"value":"md`## Imports`","pinned":false,"hidden":false,"mode":"ojs"},{"id":6,"value":"tf = require(\"@tensorflow/tfjs@2.0.0\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":1684,"value":"tf_version = tf.version","pinned":false,"hidden":false,"mode":"ojs"},{"id":2,"value":"posenet = require(\"@tensorflow-models/posenet\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":74,"value":"gifFrames = require('https://bundle.run/gif-frames@1.0.1')","pinned":false,"hidden":false,"mode":"ojs"},{"id":493,"value":"Taira = require('https://bundle.run/taira@3.2.1')","pinned":false,"hidden":false,"mode":"ojs"},{"id":471,"value":"import { slider, checkbox } from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":756,"value":"d3 = require('d3@5')","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}