30:T539,function mapGradually2(iterable, mapper, batch = defaultBatch) { const length = iterable.length; const progress = DOM.element( "progress", length ? { max: length } : undefined ); const label = DOM.element("label"); const text1 = DOM.text("0"); const text2 = DOM.text("/" + (iterable.length ?? "?")); label.appendChild(progress); label.appendChild(text1); label.appendChild(text2); const view = { value: label, done: false }; const output = []; const doneResult = { value: output, done: true }; let i = 0; const it = iterable[Symbol.iterator](); const ourIt = { next: start, // functions declared below return: () => ((ourIt.next = finished), it.return?.() ?? doneResult) }; return ourIt; function start() { ourIt.next = process; return view; } function process() { let value, done; while ((({ value, done } = it.next()), !done)) { output.push(mapper ? mapper(value) : value); if (++i % batch === 0) { if (length) progress.value = i; text1.data = i; return view; } } // Done! text1.data = i; text2.data = "/" + i; progress.value = i; progress.max = i; label.value = output; ourIt.next = finished; return view; // one last time } function finished() { return doneResult; } }31:[] 2e:{"title":"mapGradually","cells":[{"id":0,"value":"# mapGradually\n\n`mapGradually(iterable,mapper)`\n\nA helper to perform heavy computation gradually without freezing the notebook, and showing a nice progress bar to boot.\n","pinned":false,"hidden":false,"mode":"md"},{"id":196,"value":"## As a generator function, but only arrays (or something with `.length`)","pinned":false,"hidden":false,"mode":"md"},{"id":13,"value":"function* mapGradually(iterable, mapper, batch = defaultBatch) {\n const progress = document.createElement(\"progress\");\n progress.max = iterable.length;\n const label = document.createElement(\"label\");\n const text1 = document.createTextNode(\"0\");\n const text2 = document.createTextNode(`/${iterable.length ?? \"?\"}`);\n const final = document.createElement(\"div\");\n label.appendChild(progress);\n label.appendChild(text1);\n label.appendChild(text2);\n final.appendChild(label);\n const { promise, resolve } = Promise.withResolvers();\n final.value = promise;\n yield final;\n const output = [];\n let i = 0;\n while (i < iterable.length) {\n let result = iterable[i];\n if (mapper) result = mapper(iterable[i]);\n output.push(result);\n ++i;\n if (i % batch === 0) {\n if (iterable.length) progress.value = i;\n text1.data = i;\n yield final;\n }\n }\n resolve(output);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":15,"value":"viewof test = mapGradually(_.range(testSize), (x) => x * 2)","pinned":true,"hidden":false,"mode":"ojs"},{"id":18,"value":"test","pinned":true,"hidden":false,"mode":"ojs"},{"id":202,"value":"## As a generator function using iterable protocol and `for (x of y)`","pinned":false,"hidden":false,"mode":"md"},{"id":69,"value":"function* mapGradually1(iterable, mapper, batch = defaultBatch) {\n const length = iterable.length;\n const progress = document.createElement(\"progress\");\n if (length) progress.max = length;\n const label = document.createElement(\"label\");\n const text1 = document.createTextNode(\"0\");\n const text2 = document.createTextNode(\"/\" + (iterable.length ?? \"?\"));\n label.appendChild(progress);\n label.appendChild(text1);\n label.appendChild(text2);\n const { promise, resolve } = Promise.withResolvers();\n yield label;\n\n const output = [];\n let i = 0;\n for (const it of iterable) {\n output.push(mapper ? mapper(it) : it);\n ++i;\n if (i % batch === 0) {\n if (length) progress.value = i;\n text1.data = i;\n yield label;\n }\n }\n text2.data = \"/\" + i;\n\n progress.max = i;\n progress.value = i;\n label.value = output;\n yield label;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":74,"value":"viewof test1 = mapGradually1(_.range(testSize), (x) => x * 2)","pinned":true,"hidden":false,"mode":"ojs"},{"id":92,"value":"test1","pinned":true,"hidden":false,"mode":"ojs"},{"id":78,"value":"viewof test2 = mapGradually1(\n (function* () {\n let i = 0;\n while (i < testSize) yield i++;\n })(),\n (x) => x * 2\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":90,"value":"test2","pinned":true,"hidden":false,"mode":"ojs"},{"id":96,"value":"viewof test3 = mapGradually1(_(_.range(testSize)), (x) => x * 2)","pinned":true,"hidden":false,"mode":"ojs"},{"id":211,"value":"## As a custom iterator protocol implementation","pinned":false,"hidden":false,"mode":"md"},{"id":134,"value":"$30","pinned":true,"hidden":false,"mode":"ojs"},{"id":160,"value":"viewof test4 = mapGradually2(_.range(testSize), (x) => x * 2)","pinned":true,"hidden":false,"mode":"ojs"},{"id":171,"value":"test4","pinned":true,"hidden":false,"mode":"ojs"},{"id":176,"value":"viewof test5 = mapGradually2(\n (function* () {\n let i = 0;\n while (i < testSize) yield i++;\n })(),\n (x) => x * 2\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":183,"value":"test5","pinned":true,"hidden":false,"mode":"ojs"},{"id":218,"value":"viewof test6 = mapGradually2(_(_.range(testSize)), (x) => x * 2)","pinned":true,"hidden":false,"mode":"ojs"},{"id":246,"value":"test6","pinned":true,"hidden":false,"mode":"ojs"},{"id":273,"value":"## Variant: Providing a proxy iterable\nThe idea is to have a function that takes a source iterable and a handler function that will receive a wrapped version of that source. The wrapper will pause between batches automatically so that the notebook stays responsive, and like before returns a view with a progress bar tracking items yielded from the source and completes with the return value of the handler. \n\nAnd to be able to do that, we either need to go async (and so would everything consuming the iterable), or we need to go push-based instead of pull-based.\n\n### Enter RxJS.","pinned":false,"hidden":false,"mode":"md"},{"id":297,"value":"rx = import(\"https://unpkg.com/rxjs@7.8.1/dist/esm/index.js?module\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":362,"value":"// Full Rx\nrx.lastValueFrom(\n rx.from(_.range(testSize)).pipe(\n rx.map((x) => 2 * x),\n rx.toArray()\n )\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":369,"value":"{\n // Manual subscription\n const arr = [];\n rx.from(_.range(testSize))\n .pipe(rx.map((x) => 2 * x))\n .subscribe((x) => arr.push(x));\n return arr;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":375,"value":"// Manual notification\nrx.lastValueFrom(\n new rx.Observable(function subscribe(subscriber) {\n const data = _.range(100); // lower number since only one is processed each frame\n const it = data.values();\n\n // Keep track of the interval resource\n const intervalId = setInterval(() => {\n const { value, done } = it.next();\n if (done) {\n subscriber.complete();\n unsubscribe();\n } else {\n subscriber.next(value);\n }\n }, 0);\n\n // Provide a way of canceling and disposing the interval resource\n function unsubscribe() {\n clearInterval(intervalId);\n }\n // Auto-cancel when cell input is invalidated\n invalidation.then(unsubscribe);\n return unsubscribe;\n }).pipe(\n rx.map((x) => 2 * x),\n rx.toArray()\n )\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":381,"value":"### The variant","pinned":false,"hidden":false,"mode":"md"},{"id":455,"value":"// Custom observable to reduce buffering and scheduler overhead\nfunction staggeredFrom(iterable, notify = () => {}, timeBudgetMs = 15) {\n return new rx.Observable(function subscribe(subscriber) {\n const it = iterable[Symbol.iterator]();\n let count = 0;\n function produce() {\n const deadline = Date.now() + timeBudgetMs;\n while (Date.now() < deadline) {\n const { value, done } = it.next();\n if (done || subscriber.closed) {\n cleanup();\n return;\n }\n subscriber.next(value);\n count++;\n }\n notify(count);\n requestAnimationFrame(produce);\n }\n requestAnimationFrame(produce);\n function cleanup(reason) {\n notify(count);\n if (!subscriber.closed) {\n if (reason) subscriber.error(reason);\n else subscriber.complete();\n }\n }\n invalidation.then(cleanup);\n return cleanup;\n });\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":389,"value":"// Helper to make a view\nfunction progressView() {\n const el = htl.html`\n \n 0/?\n `;\n const [t1, t2] = el.getElementsByTagName(\"b\");\n const progress = el.firstElementChild;\n return {\n el,\n updateCount(value) {\n progress.value = value;\n t1.textContent = value;\n },\n set total(value) {\n progress.max = value;\n t2.textContent = value;\n },\n set value(value) {\n el.value = value;\n el.dispatchEvent(new Event(\"input\", { bubbles: true }));\n }\n };\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":266,"value":"function stagger(source, ...operators) {\n const length = source.length;\n\n const view = progressView();\n if (length) view.total = length;\n const producer = staggeredFrom(source, view.updateCount).pipe(...operators);\n\n const output = [];\n const unsub = producer.subscribe({\n next: (value) => output.push(value),\n error: (err) => (view.value = err),\n complete: () => (view.value = output.length === 1 ? output[0] : output)\n });\n invalidation.then(unsub);\n return view.el;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":496,"value":"viewof test7 = stagger(\n _.range(testSize * 100),\n rx.map((x) => 2 * x)\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":607,"value":"test7","pinned":true,"hidden":false,"mode":"ojs"},{"id":688,"value":"## Duplex Iterators/Generators\n\nMaybe it's possible to utilize how the iterator protocol lets you send values into the iterator when calling next()?","pinned":false,"hidden":false,"mode":"md"},{"id":690,"value":"{\n function* duplexMap(f) {\n let value = yield;\n while (true) {\n value = yield f(value);\n }\n }\n const gen = duplexMap((x) => 2 * x);\n gen.next(); // advance to first yield\n return [\n gen,\n gen.next(1),\n gen.next(2),\n gen.next(),\n gen.return(3),\n gen.next(4)\n ];\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":706,"value":"{\n function* duplexMap(f) {\n let value = yield;\n try {\n while (value != 42) {\n value = yield f(value);\n }\n } finally {\n return f(value);\n }\n }\n const gen = duplexMap((x) => 2 * x);\n gen.next();\n const gen2 = duplexMap((x) => 2 * x);\n gen2.next();\n\n return [\n [gen, gen.next(1), gen.next(2), gen.return(3), gen.next()],\n [gen2.next(1), gen2.next(2), gen2.next(42)]\n ];\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":712,"value":"Maybe custom generator","pinned":false,"hidden":false,"mode":"md"},{"id":714,"value":"{\n const f = (x) => 2 * x;\n const gen = {\n next(input) {\n return f(input);\n }\n };\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":753,"value":"## Fully custom API","pinned":false,"hidden":false,"mode":"md"},{"id":755,"value":"custom = ({\n mapper: {},\n filterer: {},\n reducer: {}\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":214,"value":"## Experiment: Proxying `Array.length`","pinned":false,"hidden":false,"mode":"md"},{"id":185,"value":"Reflect.getOwnPropertyDescriptor([1], \"length\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":187,"value":"new Proxy([1], {\n get(it, key) {\n if (key === \"length\") return 10;\n return it[key];\n }\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":194,"value":"({ length: 5, 1: 1 })","pinned":true,"hidden":false,"mode":"ojs"},{"id":248,"value":"## Parameters","pinned":false,"hidden":false,"mode":"md"},{"id":250,"value":"defaultBatch=1000","pinned":true,"hidden":false,"mode":"ojs"},{"id":252,"value":"testSize = 10000","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}