31:T5a1,viewof sourceMakeup = { let data = (await FileAttachment("file-extensions.json").json()).filter(file=> !['png','ini','cfg','md','txt','toml'].includes(file.ext)) let rx = 10 let plot = addTooltips(Plot.plot({ inset: 15, color: { legend: mobile ? false : true, type: "categorical", range: [0.75,-0.6], scheme: "warm", reverse: false, }, x: { label: null,padding:0.25}, y: { label: null /*"Total file length"*/ }, marginRight: 32, marginLeft: 72, marks: [ Plot.rect([0], { rx, fill: "#f8f8f8", opacity: 0.35}), Plot.ruleX(data, { x: "ext", strokeOpacity: 1, stroke: "#FFF" }), Plot.ruleY([...Array(10).keys()].map((d) => d * 1000000), { strokeOpacity: 1, stroke: "#FFF" }), Plot.barY(data,Plot.groupX({ y: "sum" ,title: e=> d3.format(".2%")(e.reduce((pool,item)=> pool+item.length,0)/data.reduce((pool,item)=>pool+item.length,0)) },{ x: "ext", y: "length", fill: "ext", /*sort: { x: "y", reverse: true }*/})), Plot.rect([0], { rx, stroke: "#ccc", strokeOpacity: 1, }), ], }),{stroke:"#ddd", "stroke-width":4}) plot.value = '' plot.querySelectorAll('[aria-label=bar] rect').forEach( (bar,i) => { bar.addEventListener('click', (event)=>{ plot.value = plot.querySelectorAll('[aria-label="x-axis tick label"] text')[i].textContent plot.dispatchEvent(new Event('input')); }) }) return plot }32:T2a90,viewof treeview = { const width = parseInt(4096*1.5) const svg = d3.create("svg") .attr("viewBox", [0,0, 4*840, 4*640]) .attr("filter", "url(#filter)") // only needed when rendering to png/svg .classed("treeview",true) let cache = [{x:0, y:0}]; const defs = svg.append("defs") const style = (` ${filthy('sepia(5%) invert(96%) brightness(225%) sepia(25%)')}`) let init = performance.now() let res = 4 let nodes = root.descendants() let links = root.links(); const steps = root.height //*root.descendants().length; const interp = d3.interpolateTurbo; const colorMap = {}; for (const [i, node] of nodes.entries()) { colorMap[node.id] = interp(0.675 - (((node.data.size || node.value )**0.75 ) / (steps) )) } const g = svg.append("g") .attr("font-family", "sans-serif") .attr("font-size", 11) const disp = svg.append("g") .attr("font-family", "sans-serif") .attr("font-size", 84) .append("text") .attr("x",-3448).attr("y",1840) .attr("style",mobile ? '' : "display:none;") .attr("paint-order","stroke fill") .attr("stroke-width",1) .attr("stroke", "white") .classed('disp',true) let prepLinks = performance.now(); let lod = ""; let grads = []; const link = g.append("g") .attr("id","links") .attr("fill", "none") //.attr("stroke", "#555") .attr("stroke-width", 3.75) .selectAll("path") .data(links) .join("path") .attr("d", d3.linkRadial() .angle(d => d.x) .radius(d => d.y) .digits(0)) .attr("stroke-linecap", "round") // .attr("vector-effect" , "non-scaling-stroke") .classed('stem', function({source,target}){ return target.children?.length || source.parent == undefined ? true : (source.children.at(0) == target || source.children.at(-1) == target) ? nodes.length < 2048+512 ? true : Math.random() > 0.5 : false }) .classed('coarse', function({source,target}){ return target.value > 3 || source.parent == undefined ? true : false }) .attr('stroke', function ({source, target}) { // get src/tgt from computed path let d = this.getAttribute('d') lod += d //this.classList.contains('stem') ? d : ''; let path = d.split(/[MC,]/).slice(1) let src = {x:path[0],y:path[1]} let tgt = {x:path[6],y:path[7]} cache.push(tgt) // encodeURIComponents when using strings // ids NEED be asci compatible // make sure id has no `--` in it const gradId = (`${source.index}--${target.index}`) this.setAttribute('id','link_'+gradId) /*const grad = defs.append('linearGradient') .attr('id', gradId) .attr('gradientUnits', 'userSpaceOnUse') .attr('x1', src.x) .attr('x2', tgt.x) .attr('y1', src.y) .attr('y2', tgt.y); grad.append('stop') .attr('offset', '0%') .attr('stop-color', colorMap[source.id]); grad.append('stop') .attr('offset', '100%') .attr('stop-color', colorMap[target.id]);*/ grads.push( ` `.replaceAll('\n','')) return `url(#${gradId})`; }) g.append("path") .attr("d", lod) .attr("stroke", Object.values(colorMap).at(~~(nodes.length/3)))//d3.interpolateHcl(mode(Object.values(colorMap)),'darkgreen')(0.15)) .attr('fill',"none") // .attr('opacity',0) .classed('lod',true).lower() defs.html(style + '\n\n' + grads.join('')) let prepNodes = performance.now() const node = g.append("g") .attr("id","nodes") .attr("stroke-linejoin", "round") .attr("stroke-width", 3) .selectAll("g") .data(nodes) .join("g") .attr("id", function(d){ // include direct DOM node reference in data d.node = this; return d.index }) .attr("transform", d => `rotate(${parseFloat((d.x * 180 / Math.PI - 90).toPrecision(res))}) translate(${parseFloat((d.y).toPrecision(res))},0)`); node.append("circle") .attr("fill", "white") .attr('stroke', n => colorMap[n.id]) .attr('stroke-width', 1.5) .attr("r", d=> Math.round(Math.sqrt(d.value))) /*.classed('stem', function(){ let dat = d3.select(this).data()[0] return dat.children?.length ? true : dat.parent.children[0] == dat || dat.parent.children.at(-1) == dat })*/ .classed('coarse', function(sel){ return d3.select(this).data()[0].value > 3 ? true : false }) svg.select('circle').attr('id','root') let maxLength = 24; let prepText = performance.now(); function addText(node) { node.append("text") .attr("transform", d => `rotate(${d.x >= Math.PI ? 180 : 0})`) .attr("dy", "0.32em") .attr("x", d => d.x < Math.PI === !d.children ? 8 : -8) .attr("text-anchor", d => d.x < Math.PI === !d.children ? "start" : "end") .attr("fill", "#777" ) .text(d => d.id.split('/').filter(Boolean).at(-1)?.slice(0,maxLength) ) .classed('stem', function(){ let dat = d3.select(this).data()[0] return dat.children?.length ? true : false }) .attr("paint-order","stroke fill") .attr("stroke-width",1) .attr("stroke", "white") } addText(node) let area = g.append("circle") .attr('id','area') .attr('r',span * 1.5) .attr('fill','white') .attr('opacity',0) .attr('will-change','opacity') .attr('style','will-change:opacity;') .attr('mask', "url(#activePath)") let clip = svg.append("mask") .attr('id',"activePath") clip.append("circle") .attr('r',span *1.5) .attr('fill', 'white') // ----- highlighter let prepCoords = performance.now(); let coords; function updateCoords() { coords = cache //consolidateCoords(node.selectAll('circle').nodes(),{absolute:true}) } updateCoords() let prepFinished = performance.now(); let bounceOver = null; g.on("pointermove", function(event) { clearTimeout(bounceOver); const m = d3.pointer(event); // d3.select(event.target).data()[0].source || d3.select(event.target).data()[0]; let leaf = nodes[d3.scan(coords.map(d => dist2([d.x, d.y], m)))] if(leaf) { let context = leaf.node.ownerSVGElement disp.text(leaf.data.name.replace(/.$/,'').replaceAll('/',' > ')) const data = leaf.ancestors() const path = data.map(d=>d.data) clip.selectAll('path').remove() // vanilla selectors for better performance context.querySelectorAll('.show').forEach(node=>(node.classList.remove("show"),node.classList.remove("leaf"))) context.querySelectorAll(':is(#nodes, #id):not(.blur)').forEach(group => group.classList.add('blur')) context.querySelector('#area').classList.add("veil") for (let i = 0; i < data.length; ++i) { data[i].node.classList.add("show") } leaf.node.classList.add("leaf") //d3.selectAll(data.map(d=>d.node)).call(addText) d3.pairs(data).forEach(([target,source])=>{ let el = document.getElementById(`link_${source.index}--${target.index}`); el.classList.add("show") let cl = el.cloneNode() cl.setAttribute('stroke','black') clip.node().appendChild(cl) }) bounceOver = setTimeout( function(){ context.value = path; context.dispatchEvent(new Event("input")) }, 11) } } ); // -------- debounce on leave // https://stackoverflow.com/questions/69116998 // a-concise-simpler-way-to-debounce-the-onclick-event let bounceOut = null; g.on('transitionend touchend pointerleave', function(event) { clearTimeout(bounceOut); bounceOut = setTimeout(()=> { event.target.style.setProperty('pointer-events', 'none'); node . classed('show',null); link . classed('show',null); area . classed("veil", null); node . classed("leaf", null); g . selectAll('#nodes, #links') . classed('blur',null) event . target.style . setProperty('pointer-events', 'auto')}, 550)}) // ------- styling let initRender = performance.now() let view = focus(svg . attr('transform-origin','center'), /*{ end: ()=>updateCoords()}*/) . attr('viewBox',[-2048,-2048,4096,4096].map(d=>d*0.9)), gap = 0; view . insert('rect','g').attr('id','bg') . attr('x','-1000%') . attr('y','-1000%') . attr('width','2000%') . attr('height','2000%') . attr('fill','white'); view . attr('style',` padding:${gap}px; width: ${width-(gap*2)}; max-width: ${width-(gap*2)}; border: 1px solid #ddd; border-radius: 8px; overflow:hidden; transform-origin: center; user-select: none; max-height: ${parseInt((width-(gap*2))*0.55)}px; /* 620px;*/ filter: sepia(5%) invert(96%) brightness(225%) sepia(25%);`) let out = view.node() let pubRender = performance.now() console.log("Init: ", parseInt(prepLinks - init)) console.log("Build links: ", parseInt(prepNodes - prepLinks)) console.log("Build nodes: ", parseInt(prepText - prepNodes)) console.log("Build label: ", parseInt(prepCoords - prepText)) console.log("Apply coords: ", parseInt(prepFinished - prepCoords)) console.log("Prep render: ", parseInt(pubRender - initRender)) return out }33:Tc4a,function focus(svg, {end = ()=>{}}= {}) { //var svg = d3.select(svg); var g = svg.select("g"), pointer = {x:0,y:0}; g.attr('will-change','transform') let [x, y, w, h] = svg.attr("tabindex",0).attr('focusable', 'true').attr("viewBox").split(/\,| /); let res = 10 ** 3, div = 10**-3; let zoom = d3.zoom().interpolate(d3.interpolate); svg . call( zoom . extent([ [ 0, 0 ], [ w, h ]]) . scaleExtent([0.5, 32]) . on("start", ({sourceEvent})=>{}) . on("zoom", ({sourceEvent,transform }) =>{ svg.classed('zooming',sourceEvent?.type !== 'touchstart'); // ad-hoc mobile fix //if(transform.k.toExponential) { // transform.k = transform.k.toExponential(2) //} // transform.k = fastRound(transform.k, res, div) // transform.x = ~~(transform.x) // transform.y = ~~(transform.y) g.attr("transform", transform)}) . on("end", ()=>(svg.classed('zooming',null),end))) . call(zoom.scaleBy, 1); svg . on("dblclick.zoom blur", resetZoom) svg . on("pointermove",(e)=>{ pointer = d3.pointer(e,g.node())}) function resetZoom(event) { event . stopPropagation(); svg . transition().duration(450) . call(zoom.transform, d3.zoomIdentity) } // d3.select("body") svg.on("keydown", function (event) { //if(event.target == svg.node()) { event.preventDefault() && event.stopPropagation() let s = svg; //svg.transition('f').duration(200) s.interrupt(); console.log(pointer) var step = 16; var key = event.key || event.keyCode; // safari doesn't know .key switch (key) { case "Escape": case "Backspace": case "Esc": case 27: s.transition().duration(1250).call(zoom.interpolate(d3.interpolateZoom).transform, d3.zoomIdentity.translate(0, 0)) break; case "0": s.transition().duration(750).call( zoom.interpolate(d3.interpolate).transform, d3.zoomIdentity.translate(w / 2, h / 2).scale(6).translate(pointer[0]*-1/1.125,pointer[1]*-1/1.125) ); break; case "+": case "=": case 187: s.transition().duration(450).call(zoom.scaleBy, 2, [w/2,0]); break; case '-': case 189: s.transition().duration(450).call(zoom.scaleBy, 0.5, [w/2,0]); break; case "ArrowLeft": case "Left": case 37: s.call(zoom.translateBy, step, 0); break; case "ArrowRight": case "Right": case 39: s.call(zoom.translateBy, -step, 0); break; case "ArrowUp": case "Up": case 38: s.call(zoom.translateBy, 0, step); break; case "ArrowDown": case "Down": case 40: s.call(zoom.translateBy, 0, -step); break; case "Enter": this.parentElement.requestFullscreen(); //default: // debug('key pressed: \'' + key + '\''); } //} }); return svg; }34:[{"id":"94391671951064e879926464b311331e2a4481d4e728fd037475928b69f292c03222cfa7597f58075519f775c92cdca6bf3a3ce437c1fd589d0d945a99acafe1","href":"https://static.observableusercontent.com/files/94391671951064e879926464b311331e2a4481d4e728fd037475928b69f292c03222cfa7597f58075519f775c92cdca6bf3a3ce437c1fd589d0d945a99acafe1","name":"index.txt","type":"text/plain","lastModified":1680317428330.54,"size":640913},{"id":"8571fbda993f58039028ee881bfdc1811e5f84317aa0815453ae5315ecb565ef5fde718fdd033fb309adb16be2f89fa9be9e9dfa355f16e78de1ba968e55f044","href":"https://static.observableusercontent.com/files/8571fbda993f58039028ee881bfdc1811e5f84317aa0815453ae5315ecb565ef5fde718fdd033fb309adb16be2f89fa9be9e9dfa355f16e78de1ba968e55f044","name":"file-extensions.json","type":"application/json","lastModified":1680323170454.502,"size":139147}] 2f:{"title":"Exploring the Twitter ‘Algorithm’","cells":[{"id":5,"value":"# Exploring the Twitter ‘Algorithm’\n## GitHub repo code analysis","pinned":false,"hidden":false,"mode":"md"},{"id":192,"value":"___","pinned":false,"hidden":false,"mode":"md"},{"id":598,"value":"#### Source code makeup\n`combined length per file extension | click to select subset` ","pinned":false,"hidden":false,"mode":"md"},{"id":289,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":822,"value":"___\n#### File tree\n`depth < ${depth} | pinch to zoom | click and enter for fullscreen | backspace to reset`","pinned":false,"hidden":false,"mode":"md"},{"id":1081,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":1172,"value":"```\n${mobile\n ? treeview.map(d => d.name.split(\"/\").at(-1) || d.name.split(\"/\").at(-2)).reverse().at(-1).replace('.','root')\n : treeview.map(d => d.name.split(\"/\").at(-1) || d.name.split(\"/\").at(-2)).reverse().join(' > ').replace('.','root')}\n```","pinned":false,"hidden":false,"mode":"md"},{"id":4636,"value":"viewof layout = Inputs.select(['tree','cluster'],{label:\"Layout\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4356,"value":"viewof search = Inputs.text({label:'Search', placeholder:'regex',submit:true})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1028,"value":"depth = mobile ? 11 : 16","pinned":false,"hidden":false,"mode":"ojs"},{"id":973,"value":"___\n**`Implementation:`**","pinned":false,"hidden":false,"mode":"md"},{"id":3333,"value":"span = {\n let max = 0;\n root.each(d=> {if(d.y> max) max = d.y})\n return max\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2295,"value":"root = { \n let empty = {name: './',contents:\"\",size:0}\n \n let i = 0; \n let root = d3.stratify().path(file=>file.name)(filestem)\n .eachBefore(d => { \n if(d.data == null) {\n d.data = structuredClone(empty);\n d.data.name = d.id;\n d.data.size = 1;\n }\n d.index = i++ \n }).sum(d => 1)\n\n let width = 1440;\n let height = width;\n let margin = 0; \n let marginTop = margin; \n let marginRight = margin;\n let marginBottom = margin; \n let marginLeft = margin;\n let radius = Math.min(width - marginLeft - marginRight, height - marginTop - marginBottom) / (2-Math.min(depth/16,1.5));\n \n root.dx = 0;\n root.dy = 0; //Math.max(256, width / (root.height + 1));\n root.data = empty\n root.id = 'root'\n return d3[layout]().size([2 * Math.PI, radius]).separation( (a, b) => (a.parent == b.parent ? 1 : 1) / a.depth)(root)\n \n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":986,"value":"counts = files.map(d=>d.name.split('/').filter((d,i,f)=> d && (i < f.length -1)).slice(0,depth).join('/')).filter(Boolean)\n.reduce( (pool,item) => {\n item = item + '/'\n pool[item]? pool[item]+= 1 : pool[item] = 1\n return pool \n}, {})","pinned":false,"hidden":false,"mode":"ojs"},{"id":2368,"value":"filetree = files\n .filter(\n (file) =>\n (file.name.endsWith(\"/\") || file.name.endsWith(\".\"+sourceMakeup)) &&\n file.name.split(\"/\").length < depth\n //&& (file.contents.length < 1000 || file.contents.length == 0)\n )\n .map((file) => ({ ...file, size: counts[file.name] }))","pinned":false,"hidden":false,"mode":"ojs"},{"id":2533,"value":"// compress singleton paths \n\nfilestem = filetree.map(file => {\n\n let match = null, n = 0;\n while (!match && n <16) { n++\n\n let parent = file.name.split('/').filter((d,i,f)=> i < f.length-n).join('/')+'/'\n match = singlets.find(e => e==parent)\n \n }\n let name = file.name;\n if (match) name = name.replace(match, match.replaceAll('/','_'))\n return {...file, name: name }\n \n})//.filter(file=> file.contents.includes(\".\") )\n.filter(file=>new RegExp(search).test(file.contents))","pinned":false,"hidden":false,"mode":"ojs"},{"id":2526,"value":"singlets = Object.entries(counts).flatMap(([path,size])=> size == 1 && path.split('/').length < 0 ? path : [] ).sort((a,b)=>b.length - a.length)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1084,"value":"function autoBox(margin = 32) {\n document.body.appendChild(this);\n const {x, y, width, height} = this.getBBox();\n document.body.removeChild(this);\n\n console.log(x,y,width,height)\n return [x*1.6-margin, y-margin, width*1.6+(margin*2), height+(margin*2)]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1097,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":2247,"value":"mobile = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent) ","pinned":false,"hidden":false,"mode":"ojs"},{"id":2827,"value":"{ \n d3.select(viewof treeview).select('.temp').remove()\n \n d3.select(viewof treeview).append('g').classed('temp',true)\n\n /* .selectAll(\"circle\")\n .data([{x:0,y:0}])\n .join(\"circle\")\n \n //.append(\"circle\")\n .attr(\"fill\", 'black')\n .attr('stroke-width', 1.5)\n\n // .attr(\"opacity\", d => d.children ? 1 : 1 )\n\n .attr(\"r\", 3)\n .attr(\"cx\", d=>d?.x)\n .attr(\"cy\", d=>d?.y)*/\n return 'drawfunction'\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2729,"value":" function dist2(a, b) {\n return (a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2;\n }","pinned":false,"hidden":false,"mode":"ojs"},{"id":2725,"value":"function consolidateCoords(elems, { absolute = true } = {}) {\n // based on transform on first group\n \n let init = elems[0], offset;\n let svg = init.ownerSVGElement\n let orphan = !svg.isConnected\n\n if(orphan) {\n document.body.appendChild(svg)\n }\n \n if(absolute) {\n offset = svg.querySelector(\"g\").getScreenCTM().inverse();\n } else {\n offset = svg.getScreenCTM().inverse();\n }\n\n let s = svg.createSVGPoint();\n let coords = elems.map((shape) => {\n let bbox = shape.getBoundingClientRect();\n s.x = bbox.x;\n s.y = bbox.y;\n\n let { x, y } = s.matrixTransform(offset);\n\n return { x, y };\n });\n\n if(orphan) {\n document.body.removeChild(svg)\n }\n return coords\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":5322,"value":"// safeish fast rounding of first 4-6 floats\n\nfunction fastRound(num,n = 0,div = 0){\n\n // let int = (num + (num >0?0.5:-0.5) << 0)\n // (num << 0) or (~~num)\n \n return (~~num) + (( num*n + ( num >0?0.5:-0.5) << 0 ) - ((~~num)*n)) * div\n \n // num > 0 ? \n // ( num*n + ( num*n >0?0.5:-0.5) << 0 ) - ((num << 0)*n) \n // : ((num << 0)*n) - ( num*n + ( num*n >0?0.5:-0.5) << 0 ) \n\n }","pinned":false,"hidden":false,"mode":"ojs"},{"id":5418,"value":"function mode(arr){\n return arr.sort((a,b) =>\n arr.filter(v => v===a).length\n - arr.filter(v => v===b).length\n ).pop();\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":789,"value":"____","pinned":false,"hidden":false,"mode":"md"},{"id":124,"value":"extensions = files\n .filter(\n (file) =>\n file.name.includes(\".\") &&\n !file.name.includes(\"gitignore\") &&\n !file.name.includes(\"README\")\n )\n .map((file) => ({\n name: file.name.split('/').at(-1),\n ext: file.name.split(\".\")[1],\n length: file.contents.split('\\n').length,\n }))\n .filter((file) => file.ext.length < 7)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1039,"value":"___\n#### Search tree","pinned":false,"hidden":false,"mode":"md"},{"id":111,"value":"___\n### Data","pinned":false,"hidden":false,"mode":"md"},{"id":20,"value":"index = FileAttachment(\"index.txt\").text().then(d=>d.split(/\\r\\n/).filter(d=>d.includes('.')))","pinned":false,"hidden":false,"mode":"ojs"},{"id":77,"value":"files = fetchp(\"https://github.com/twitter/the-algorithm/archive/refs/heads/main.zip\")\n .then((d) => d.arrayBuffer())\n .then((buffer) => fflate.unzipSync(new Uint8Array(buffer)))\n .then((root) =>\n \n Object.entries(root)\n .map(([file, buf]) => ({\n name:file.split(\"the-algorithm-main/\")[1],\n contents:fflate.strFromU8(buf)\n }))\n .filter( file => file.name.length > 1)\n \n );","pinned":false,"hidden":false,"mode":"ojs"},{"id":16,"value":"___\n### Imports","pinned":false,"hidden":false,"mode":"md"},{"id":85,"value":"import {fetchp} from '@tomlarkworthy/fetchp'","pinned":false,"hidden":false,"mode":"ojs"},{"id":1165,"value":"import {filthy} from \"@dleeftink/svg-workbench\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":697,"value":"import {addTooltips} from \"@mkfreeman/plot-tooltip\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"ghRaw = (await import('https://cdn.skypack.dev/gh-raw@0.1.2?min')).default","pinned":false,"hidden":false,"mode":"ojs"},{"id":79,"value":"fflate = import(\"https://cdn.skypack.dev/fflate?min\")","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W34","stdlib":"1","titling":"auto"}