30:T1d1f,function atomise({ mode = 'BFS', depth = 5, slots = 2 ** 15, dist = false, cycle = false , tree = false, split = 2, query = (``), filter = (k,c,p) => true, take = Infinity, separator = '.', meta, format = (v)=> v , parse = (v) => v, leaves = null, prefix = '@', ... rest } = {}) { /* -------- */ let iters = -1; let object = rest; let outer = [], queue = []; let t1 = performance.now(); let maxDepth = depth; let prior = new Set(), cache = new Map(), nodes = new Map(), types = new Map(); /* ---------------- */ let its = (prop) => Object.prototype.toString.call(prop); let any = (prop, ...types) => types.some((type) => its(prop) === its(type)); let not = (prop, ...types) => !any(prop,...types) ; let see = (prop) => ext(prop) ? 'Class' : its(prop).slice(1,-1).split(' ')[1] let ext = (prop) => its(prop).includes('Func') && Function.prototype.toString.call(prop).startsWith('class') let end = (prop) => leaves === null ? ~leaves : leaves ^ typeof prop === typeof {}; let bin = (item) => Object.fromEntries(Object.entries(item).map(d=>[d[0],''])); let get = (path) => path.slice(1).reduce((path,item)=> path && path[item] || null, (object)) /* ---------------- */ let sum = (list, all = {}) => { if ( list.length ) { for ( let [ key, val ] of list) { let type = types.get(val) ?? see(val?.value || val); types.set(val,type); all[type.concat('s')] = (all[type.concat('s')] || 0) + 1}} return [all,list.length] } /* ---------------- */ let sep = (types, left = '', right = '') => { for ( let [key,val] of types) { if ( key.length > split && !isFinite(key)) { left += key + '|' } else { right += key + '|' }} return [ left.slice(0,-1), right.slice(0,-1) ] } /* ---------------- */ let add = (item) => { let node = document.createElement(item.own); Object.defineProperty(node,'out',{ get:()=>decodeURI(format(node.outerHTML))}) Object.entries(item) .filter(([key, val], i, f) => ["key", "cid","sub","dim","sig","all","ext"].includes(key) && val) .forEach(([key, val]) => any(val, Object()) ? Object.entries(val).forEach(([key,val])=> node.setAttribute(key,val)) : node.setAttribute(key, val)); if(not(item.val, Array(),Object(),Symbol())) { node.setAttribute('content', encodeURI(item.val.toString().replaceAll(/\n+/g,'')))} return node }; /* ---------------- */ function getProp(object, depth = 0) { let chain = [], forward; queue.push([object, chain, depth]); switch (mode) { case "BFS": forward = true , mode = "shift" ; break; case "DFS": forward = false , mode = "pop" ; break;} while ( queue.length ) {[ object , chain , depth ] = queue[mode](); // console.log(chain); for ( let [ key, val ] of forward ? spy(object) : spy(object).reverse().sort((a,b)=>parseInt(a[0]) - parseInt(b[0]))) { ++iters; if ( outer.length > take - 1 ) continue let own = types.get(val) ?? see(val) , set, sig, all, dim, child ; types.set(val,own); let entry = { key, own, val, cid: (val?.name ?? val?.constructor?.name)?.toString().replace // escape reserved rgx chars (new RegExp(`^${[own,key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')].join('|')}$`),''), set, sig, sub:depth, dim, pad:chain,all }; if( typeof meta == 'function' ) { Object.assign(entry, { ext:meta({...entry},iters)}) } Object.defineProperties(entry , { parse: { value:()=>not(val, Object(),Symbol()) ? parse(val.toString()) : {}}, path: { get:()=>(prop)=>entry.pad.map(d=>d[prop||'key'])}}) if ( dist ) { let counts = cache.get(val) if( counts === undefined ) { counts = spy(val); [ set , sig , all, dim ] = sep(counts).concat(sum(counts)); cache.set(val , [ set , sig , all, dim ])} else {[ set , sig , all , dim ] = counts}; Object.assign(entry, { set , sig , all, dim })} if ( tree || query.length > 0 ) { let parentPath = chain.map(d=>d.key), childsPath = parentPath.concat(key), parent = nodes.get(parentPath.join(separator)) ; child = nodes.get(childsPath.join(separator)) ?? add(entry); nodes.set(childsPath.join(separator),child) parent?.appendChild(child) } if ( filter(key, entry, chain.at(-1) ?? bin(entry)) && end(val)) { outer.push( Object.defineProperties( { [[prefix].concat(entry.path()).join(separator)]: key, above:chain.at(-1), below:entry, chain, child, iters,depth }, { out: { get:()=>child.out}, path:{ get:()=>entry.path }})) if ( tree || query.length > 0 ){ child.setAttribute('include', true)}} if ((typeof val == 'object' ? prior.has(val) : prior.has(key)) && !cycle || prior.size > slots - 1 ) { continue } if ((typeof val == 'object' || typeof val == 'function') && depth < maxDepth) { prior.add(val); queue.push([val, [...chain, entry], depth + 1]) } else { prior.add(key); } } } } getProp(object); let root = nodes.values().next().value; if ( query.length > 0 ) { let result = root.querySelectorAll(query); for ( let i = 0; i < result.length; ++i) { result[i].setAttribute('select', true) } outer = outer.filter(d=>{ let inc = d.child.getAttribute('select') ; return inc })} Object.defineProperties(outer,{ ticks: { get:()=> performance.now() - t1 }, root: { get:()=> root }, crop: { get:()=> { nodes.forEach(node=> { if (!node.getAttribute('include') || (!node.getAttribute('select') && query.length > 0) ) { node.replaceWith(...node.children) } else { node.removeAttribute('include'); node.removeAttribute('select') }}); root.removeAttribute('include');root.removeAttribute('select'); return root?.out }}, out: { get:()=> root?.out } }) return outer }31:T74c,code = ps.parseSource(`package com.twitter.ann.annoy import com.twitter.ann.common.RuntimeParams import com.twitter.ann.common.thriftscala.AnnoyIndexMetadata import com.twitter.bijection.Injection import com.twitter.mediaservices.commons.codec.ThriftByteBufferCodec import com.twitter.ann.common.thriftscala.{AnnoyRuntimeParam, RuntimeParams => ServiceRuntimeParams} import scala.util.{Failure, Success, Try} object AnnoyCommon { private[annoy] lazy val MetadataCodec = new ThriftByteBufferCodec(AnnoyIndexMetadata) private[annoy] val IndexFileName = "annoy_index" private[annoy] val MetaDataFileName = "annoy_index_metadata" private[annoy] val IndexIdMappingFileName = "annoy_index_id_mapping" val RuntimeParamsInjection: Injection[AnnoyRuntimeParams, ServiceRuntimeParams] = new Injection[AnnoyRuntimeParams, ServiceRuntimeParams] { override def apply(scalaParams: AnnoyRuntimeParams): ServiceRuntimeParams = { ServiceRuntimeParams.AnnoyParam( AnnoyRuntimeParam( scalaParams.nodesToExplore ) ) } override def invert(thriftParams: ServiceRuntimeParams): Try[AnnoyRuntimeParams] = thriftParams match { case ServiceRuntimeParams.AnnoyParam(annoyParam) => Success( AnnoyRuntimeParams(annoyParam.numOfNodesToExplore) ) case p => Failure(new IllegalArgumentException(s"Expected AnnoyRuntimeParams got $p")) } } } case class AnnoyRuntimeParams( /* Number of vectors to evaluate while searching. A larger value will give more accurate results, but will take longer time to return. * Default value would be numberOfTrees*numberOfNeigboursRequested */ nodesToExplore: Option[Int]) extends RuntimeParams { override def toString: String = s"AnnoyRuntimeParams( nodesToExplore = $nodesToExplore)" }`)32:T5e8,// from // stackoverflow.com/a/9927697 function hash(o, l = 2) { /* o: object , l : length */ var i, c, r = []; for (i = 0; i < l; i++) r.push(i * 268803292); function toHex(number) { var ret = ((number < 0 ? 0x8 : 0) + ((number >> 28) & 0x7)).toString(16) + (number & 0xfffffff).toString(16); while (ret.length < 8) ret = "0" + ret; return ret; } let seen = new Set(); function stringify(o) { var i, r; if (o === null) return "n"; if (o === true) return "t"; if (o === false) return "f"; if (o instanceof Date) return "d:" i = typeof o; if (i === "string") return "s:" + o.replace(/([\\\\;])/g, "\\$1"); if (i === "number") return "n:" + o; if (o instanceof Function) return "m:" + o.toString().replace(/([\\\\;])/g, "\\$1"); if(o instanceof HTMLElement) { return "h:" + o.outerHTML } if (o instanceof Array) { r = []; for (i = 0; i < o.length; i++) r.push(stringify(o[i])); return "a:" + r.join(";"); } r = []; for (i in o) { if(seen.has(i)) {continue} seen.add(i); r.push(i + ":" + stringify(o[i])); } return "o:" + r.join(";"); } o = stringify(o); for (i = 0; i < o.length; i++) { for (c = 0; c < r.length; c++) { r[c] = (r[c] << 13) - (r[c] >> 19); r[c] += o.charCodeAt(i) << r[c] % 24; r[c] = r[c] & r[c]; } } for (i = 0; i < r.length; i++) { r[i] = toHex(r[i]); } return r.join(""); } 33:[{"id":"f9ff95ba7a27e3a77aa082c7e5d89a742765f26af1698f8d7dad40c43bd7519420c8468cd07dd174548f30bc9686feb387deebb6fb606a24d57ef4d495a3216f","href":"https://static.observableusercontent.com/files/f9ff95ba7a27e3a77aa082c7e5d89a742765f26af1698f8d7dad40c43bd7519420c8468cd07dd174548f30bc9686feb387deebb6fb606a24d57ef4d495a3216f","name":"chunk-it@2.png","type":"image/png","lastModified":1674794719082.573,"size":324765}] 2e:{"title":"Metatree","cells":[{"id":0,"value":"# Metatree\n## Extracting deeply nested structures","pinned":false,"hidden":false,"mode":"md"},{"id":87,"value":"___","pinned":false,"hidden":false,"mode":"md"},{"id":34,"value":"$${await FileAttachment(\"chunk-it@2.png\").image({style: 'width:640px; max-width:width'})}","pinned":false,"hidden":false,"mode":"md"},{"id":38,"value":"___","pinned":false,"hidden":false,"mode":"md"},{"id":145,"value":"How would you go about pinpointing deeply nested data, especially in a codebase not your own? \nMany libraries and strategies have been devised to traverse JS object trees,\nincluding retrofitting XPath and CSS selectors to target object properties. \nWhile these might not be as performant as pure JS property selection, \nit is worthwhile to revisit this idea now that the CSS `:has` selector makes it possible \nto select parent elements based on the contents of their children. ","pinned":false,"hidden":false,"mode":"md"},{"id":158,"value":"```js\nimport { atomise } from \"@dleeftink/metatree\"\n```","pinned":false,"hidden":false,"mode":"md"},{"id":154,"value":"___","pinned":false,"hidden":false,"mode":"md"},{"id":164,"value":"We start out with a simple object containing various data and object types.\nImagine this to be a single record, an imported NPM package or some other type of data you want to match \nincluding cyclical references.\n___","pinned":false,"hidden":false,"mode":"md"},{"id":43,"value":"source = ({ \n a: cycle, \n b:[{ sub:{ tab:{ B:[{d:3},[1,2],2,3,4,5], cycle } } },{},new Map()],\n c: { c: 2, d: {b: 3, e: {f: function*(e){return e}, g: Symbol('desc'),h:Object.assign(htl.html`

Some content

`,{a:()=>{}})}, cycle}}\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1051,"value":"cycle = ({t:performance.now(), ID:DOM.uid()})","pinned":false,"hidden":false,"mode":"ojs"},{"id":2997,"value":"atom2 = atomise({\n \n source, format,\n mode:'DFS', // breadth or depth first\n dist: true, // get leaf distributions \n tree: true, // build an XML-like tree\n\n depth: 7, // maximum iteration depth\n cycle: false, // ignore nested cycles\n\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":169,"value":"___","pinned":false,"hidden":false,"mode":"md"},{"id":174,"value":"We can `atomise` this source tree into individual values using the function provided here, \ntaking various options and a `meta` callback function:","pinned":false,"hidden":false,"mode":"md"},{"id":45,"value":"atom = atomise({\n \n source, format,\n mode:'DFS', // breadth or depth first\n dist: true, // get leaf distributions \n tree: true, // build an XML-like tree\n\n depth: 7, // maximum iteration depth\n cycle: false, // ignore nested cycles\n\n query: (`Object:has([content*=http])`), // typed cssselector query\n meta: ()=> ({time:performance.now()}) // callback for each entry\n\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":1264,"value":"___\n\nFor each node or leaf on the tree, a record is returned with the following structure:\n\n```javascript\nreturn { \n named: String (Key)\n above: Object (Parent)\n entry: Object (Current)\n chain: Array (Full path)\n child: DOM (XML Element)\n iters: Number (Iterations)\n depth: Number (Nesting)\n}\n```","pinned":false,"hidden":false,"mode":"md"},{"id":1385,"value":"___","pinned":false,"hidden":false,"mode":"md"},{"id":1298,"value":"Each entry in turn, contains the following metadata:\n\n``` js\nreturn {\n key: String (Key)\n own: String (Type)\n val: Any (Value)\n pad: Array (Path)\n all: Object (Counts) => ({dist: true})\n sét: String (Long props) => ({dist: true, length > split })\n sig: String (Short props) => ({dist: true, length < split })\n cid: String (Constructor name)\n sub: Number (Depth)\n dim: Number (Children)\n ext: Object (Callback)\n}\n```","pinned":false,"hidden":false,"mode":"md"},{"id":1360,"value":"___\n\nAny of these entry values can be used to filter an initial source tree, including libraries as `d3`. \nSet `{dist:true}` for distributional metadata:","pinned":false,"hidden":false,"mode":"md"},{"id":1366,"value":"filtered = atomise({\n \n d3, format, // optional DOM formatter\n dist: true, // get leaf distributions \n // tree: true,\n \n depth: 5,\n filter: (key, child, parent)=> (parent.own =='Class' && child.own == 'Function'),\n // query: (`Class[sub=\"1\"]:has([sub=\"2\"])`), \n\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":2970,"value":"atomise({d3,depth:3,mode:'BFS',filter:key=>key.includes('x')})","pinned":true,"hidden":false,"mode":"ojs"},{"id":5,"value":"$30","pinned":true,"hidden":false,"mode":"ojs"},{"id":655,"value":"function spy(object) { /*\n* ------ enumerate keys names, methods\n*/ var list = [], props, \n current = object;\n \n do { \n try { props = Object.entries( current?.prototype ? \n Object.getOwnPropertyDescriptors(current.prototype) : typeof current === 'string' ? [] : current )} \n catch { props = [] }\n \n for ( let [ key , item ] of props ) \n { if ( item !== undefined && list.indexOf([key, item]) === -1\n &&! [ \"constructor\", \"__proto__\",\"writable\",\"enumerable\",\"configurable\",].includes(key) \n &&! ( item?.location && item.location == window.location) \n &&! ( item?.value?.toString && item?.value?.toString().includes(\"native code\") || item?.toString && item?.toString().includes(\"native code\")))\n { list.push([key, item?.value ?? item ]) }}} \n \n while ( current && \n ( current = Object.getPrototypeOf(current))); return list }","pinned":true,"hidden":false,"mode":"ojs"},{"id":2958,"value":"___\n\n### Playground","pinned":false,"hidden":false,"mode":"md"},{"id":2904,"value":"atomise({\n format,\n root:JSON.parse(JSON.stringify(code)) ,\n mode:'BFS', // breadth or depth first\n dist: true, // get leaf distributions \n tree: true, // build an XML-like tree\n\n depth: 2, // maximum iteration depth\n cycle: false, // ignore nested cycles\n meta: (entry,i)=>{\n entry.own = 'o'+entry.key\n mutable temp.push(entry), mutable temp = mutable temp, i+1\n }\n}).out","pinned":true,"hidden":false,"mode":"ojs"},{"id":2938,"value":"mutable temp = []","pinned":true,"hidden":false,"mode":"ojs"},{"id":2908,"value":"$31","pinned":true,"hidden":false,"mode":"ojs"},{"id":20,"value":"___\n\n### Imports","pinned":false,"hidden":false,"mode":"md"},{"id":12,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"format = import('https://cdn.skypack.dev/xml-formatter@2.6.1?min').then(i=>i.default)","pinned":false,"hidden":false,"mode":"ojs"},{"id":952,"value":"acorn = import('https://unpkg.com/acorn@8.8.2/dist/acorn.mjs?module')","pinned":false,"hidden":false,"mode":"ojs"},{"id":2902,"value":"ps = (await import(\"https://esm.sh/scalameta-parsers@4.7.6\")).default","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"}