2f:T128c,chart = { const sw = 0.4; const so = 0.95; const k = 2; // 2^k colors segments per curve const myColor = blackWhite ? (d) => (darkMode ? "white" : "black") : colorFunc(colorNumber, colorRotation, colorSeed, darkenFactor); // Specify the chart’s dimensions. const width = 1500; const height = width; const cx = width * 0.5; // adjust as needed to fit const cy = height * 0.5; // adjust as needed to fit const radius = Math.min(width, height) / 2 - 135; // Create a radial cluster layout. The layout’s first dimension (x) // is the angle, while the second (y) is the radius. const tree = d3 .cluster() .size([2 * Math.PI, radius]) .separation((a, b) => (a.parent == b.parent ? 1 : 1)); // Sort the tree and apply the layout. const root = tree(hierarchie); root.leaves().forEach((leaf) => { leaf.outgoing = []; leaf.incoming = []; const link = links.find((node) => node.file === leaf.data.name); if (link) { link.links.forEach((targetLink) => { const targetNode = root.find((d) => d.data.name === targetLink.file); if (targetNode) { leaf.outgoing.push([leaf, targetNode]); console.log(targetNode); } }); } links.forEach((node) => { node.links.forEach((link) => { if (link.file === leaf.data.name) leaf.incoming.push([node, leaf]); }); }); leaf.degree = leaf.incoming.length + leaf.outgoing.length; }); console.log(root); // Creates the SVG container. const svg = d3 .create("svg") .attr("width", width) .attr("height", height) .attr("viewBox", [-cx, -cy, width, height]) .attr("style", "width: 100%; height: auto; font: 10px sans-serif;"); if (darkMode) svg .append("rect") .attr("x", -width / 2) .attr("y", -height / 2) .attr("width", width) .attr("height", height) .attr("fill", "black"); // Append links. if (structure) svg .append("g") .attr("fill", "none") .attr("stroke", "#555") .attr("stroke-opacity", 0.4) .attr("stroke-width", 1.5) .selectAll() .data(root.links()) .join("path") .attr( "d", d3 .linkRadial() .angle((d) => d.x) .radius((d) => d.y) ) .attr("stroke", (d) => myColor( d.target.ancestors().find((ancestor) => ancestor.depth == colorDepth) ) ); // Append nodes. svg .append("g") .selectAll() .data(root.leaves()) .join("circle") .attr( "transform", (d) => `rotate(${(d.x * 180) / Math.PI - 90}) translate(${d.y},0)` ) .attr("fill", (d) => (d.children ? "#555" : "#999")) .attr("r", (d) => 1 + 0.05 * d.degree) .attr("fill", (d) => { const col = myColor( d.ancestors().find((ancestor) => ancestor.depth == colorDepth) ); return d.children ? col : darkMode ? darken(col, 0.8) : lighten(col, 0); // return d.children ? col : darkMode ? darken(col, 0.8) : lighten(col, 0.8); }); // Append labels. svg .append("g") .attr("stroke-linejoin", "round") .attr("stroke-width", 3) .selectAll() .data(root.leaves()) .join("text") .attr( "transform", (d) => `rotate(${(d.x * 180) / Math.PI - 90}) translate(${d.y},0) rotate(${ d.x >= Math.PI ? 180 : 0 })` ) .attr("dy", "0.31em") .attr("x", (d) => (d.x < Math.PI === !d.children ? 6 : -6)) .attr("text-anchor", (d) => d.x < Math.PI === !d.children ? "start" : "end" ) .attr("font-size", (d) => 9.5 + 0.13 * d.degree) .attr("paint-order", "stroke") .attr("stroke", darkMode ? "black" : "white") .attr("fill", "currentColor") .attr("fill", (d) => { const col = myColor( d.ancestors().find((ancestor) => ancestor.depth == colorDepth) ); return darkMode ? lighten(col, 0.4) : col; }) .text((d) => d.height == 0 ? links.find((node) => node.file == d.data.name).title : d.depth ); if (edge) svg .append("g") .attr("fill", "none") .attr("stroke-width", sw) .attr("stroke-opacity", so) .selectAll("path") .data(root.leaves().flatMap((leaf) => leaf.outgoing)) .join("path") .style("mix-blend-mode", darkMode ? "normal" : "multiply") .style("opacity", so) .attr("d", ([i, o]) => line(i.path(o))) .each(function (d) { d.path = this; }) .attr("stroke", (d) => lighten( myColor( d[1].ancestors().find((ancestor) => ancestor.depth == colorDepth) ), LighenFactor ) ); return svg.node(); }30:Te2f,html`

Click to expand

${md` \`\`\`js const fs = require('fs-extra'); const path = require('path'); const { JSDOM } = require('jsdom'); function toTitleCase(str) { return str .toLowerCase() // Convert the entire string to lowercase .split(' ') // Split the string into words .map(word => word.charAt(0).toUpperCase() + word.slice(1)) // Capitalize the first letter of each word .join(' '); // Join the words back into a string } // Example usage const input = "hello world, THIS is a test!"; const titleCased = toTitleCase(input); console.log(titleCased); // Output: "Hello World, This Is A Test!" // Function to check if a URL is internal (relative to the root) function isInternalLink(url) { return !/^https?:\/\//i.test(url) && !url.startsWith('//') && !url.startsWith('data:'); } // Function to filter links based on additional conditions function shouldIncludeLink(href) { // Exclude links starting with "dic" or "table" return !(href.startsWith('../dic') || href.startsWith('../table')); } // Recursive function to traverse directories function traverseDirectory(dir, callback) { fs.readdirSync(dir).forEach((file) => { const filePath = path.join(dir, file); if (fs.statSync(filePath).isDirectory()) { traverseDirectory(filePath, callback); } else { callback(filePath); } }); } // Function to parse HTML files and extract links and title function extractLinksAndTitleFromHTML(filePath, basePath) { const content = fs.readFileSync(filePath, 'utf8'); const dom = new JSDOM(content); const document = dom.window.document; const links = []; const title = document.querySelector('title')?.textContent.trim() || 'Untitled'; const firstH2 = document.querySelector('h2')?.textContent.trim() || 'No H2 found'; document.querySelectorAll('a[href]').forEach((link) => { const parent = link.closest('p[align="center"]'); if (parent) { return; // Ignore links inside

} const href = link.getAttribute('href'); if (isInternalLink(href) && shouldIncludeLink(href)) { const absoluteHref = path.resolve(basePath, href); links.push({ file: href.split('/').pop(), //absoluteHref: absoluteHref, // title: link.textContent.trim(), // attributes: Array.from(link.attributes).reduce((acc, attr) => { // acc[attr.name] = attr.value; // return acc; // }, {}) }); } }); return { title, firstH2, links }; } // Main script function main() { const websiteFolder = './wtb'; // Replace with your folder path const outputJSON = './links.json'; const basePath = path.resolve(websiteFolder); const allPages = []; traverseDirectory(basePath, (filePath) => { if (filePath.endsWith('.html') || filePath.endsWith('.htm')) { const relativePath = path.relative(basePath, filePath); const { title, firstH2, links } = extractLinksAndTitleFromHTML(filePath, basePath); allPages.push({ file: relativePath.split('/').pop(), // title: title, title: toTitleCase(firstH2), links: links, }); } }); fs.writeFileSync(outputJSON, JSON.stringify(allPages, null, 2), 'utf8'); console.log(\`Links, firstH2 and titles extracted to \${outputJSON}\`); } main(); \`\`\` `}

`31:T16f4,html`

Click to expand

${md` \`\`\`py import networkx as nx import json import os # Load your JSON data with open('links.json', 'r') as file: data = json.load(file) # Create a directed graph directed_graph = nx.DiGraph() # Add nodes and edges for entry in data: source_file = entry["file"] source_title = entry["title"] directed_graph.add_node(source_file, title=source_title) for link in entry["links"]: target_file = link["file"] directed_graph.add_edge(source_file, target_file) # Convert to an undirected graph and merge multiple edges into one undirected_graph = nx.Graph() # Add edges while merging for u, v in directed_graph.edges(): if undirected_graph.has_edge(u, v): # If the edge already exists, you could sum weights or leave as a single edge. undirected_graph[u][v]["weight"] += 1 else: undirected_graph.add_edge(u, v, weight=1) # Add node attributes to the undirected graph # for node, data in directed_graph.nodes(data=True): # undirected_graph.nodes[node].update(data) # Calculate metrics for the entire undirected graph print("Metrics for the undirected graph:") print(f"Number of nodes: {undirected_graph.number_of_nodes()}") print(f"Number of edges: {undirected_graph.number_of_edges()}") print(f"Density: {nx.density(undirected_graph):.4f}") # Degree distributions degrees = dict(undirected_graph.degree()) average_degree = sum(degrees.values()) / len(degrees) print(f"Average degree: {average_degree:.2f}") # Identify the largest connected component largest_component = max(nx.connected_components(undirected_graph), key=len) main_component = undirected_graph.subgraph(largest_component).copy() # Metrics for the main component print("\nMetrics for the main component:") print(f"Number of nodes: {main_component.number_of_nodes()}") print(f"Number of edges: {main_component.number_of_edges()}") print(f"Density: {nx.density(main_component):.4f}") ## SPLIT GRAPH ## # Append the split_graph function def split_graph(graph, log_file_path="graph_split_log.txt", output_dir="split_results"): # Ensure the output directory exists if not os.path.exists(output_dir): os.makedirs(output_dir) removed_edges = [] remaining_subgraphs = [] communities = [] # To store the hierarchical structure with open(log_file_path, "w") as log_file: def recursive_split(subgraph, level=0): log_file.write(f"[Level {level}] Processing subgraph with {subgraph.number_of_nodes()} nodes and {subgraph.number_of_edges()} edges.\n") print(f"[Level {level}] Nodes: {subgraph.number_of_nodes()}, Edges: {subgraph.number_of_edges()}") # Calculate edge betweenness centrality without normalization edge_bc = nx.edge_betweenness_centrality(subgraph, normalized=False) # Filter edges based on the given conditions valid_edges = [ (edge, bc) for edge, bc in edge_bc.items() if bc > 1 and subgraph.degree(edge[0]) > 1 and subgraph.degree(edge[1]) > 1 ] if not valid_edges: # Save the subgraph to file subgraph_filename = os.path.join(output_dir, f"subgraph_level_{level}_{len(remaining_subgraphs)}.graphml") nx.write_graphml(subgraph, subgraph_filename) remaining_subgraphs.append(subgraph) # Save the subgraph as a community community = { "parent": f"Community_Level_{level}_{len(remaining_subgraphs)}", "children": list(subgraph.nodes) } communities.append(community) log_file.write(f"[Level {level}] Subgraph saved to {subgraph_filename}.\n") return # Find and remove the edge with the highest betweenness centrality edge_to_remove, max_bc = max(valid_edges, key=lambda x: x[1]) log_file.write(f"[Level {level}] Removing edge {edge_to_remove} with betweenness centrality {max_bc}.\n") removed_edges.append(edge_to_remove) # Remove the edge and split the graph into connected components subgraph.remove_edge(*edge_to_remove) components = [subgraph.subgraph(c).copy() for c in nx.connected_components(subgraph)] for component in components: recursive_split(component, level + 1) # Start the recursive splitting recursive_split(graph) # Save the list of removed edges edges_file = os.path.join(output_dir, "removed_edges.json") with open(edges_file, "w") as f: json.dump(removed_edges, f, indent=4) # Save the communities to a JSON file communities_file = os.path.join(output_dir, "communities.json") with open(communities_file, "w") as f: json.dump(communities, f, indent=4) log_file.write(f"\nSplitting process complete. Removed edges saved to {edges_file}\n") log_file.write(f"Communities saved to {communities_file}\n") print(f"\nSplitting process complete. Removed edges saved to {edges_file}\n") print(f"Communities saved to {communities_file}") return removed_edges, remaining_subgraphs, communities # Apply the split function edges_removed, subgraphs, communities = split_graph(main_component) # Output summary print("\nEdges removed during the splitting process:") for edge in edges_removed: print(edge) print(f"\nNumber of remaining subgraphs: {len(subgraphs)}") for i, sg in enumerate(subgraphs): print(f"Subgraph {i + 1}: {sg.number_of_nodes()} nodes, {sg.number_of_edges()} edges") print("\nHierarchical Communities:") for community in communities: print(community) \`\`\` `}

`32:T885,hierarchie = { let communities = communitiesH; const links = removed_edges; let r = 0; links.reverse().forEach((link) => { // Finding Communities let a = link[0], b = link[1]; let comA = communities.find((com) => com.find((node) => node.data.name == a) ); let comB = communities.find((com) => com.find((node) => node.data.name == b) ); // Merginig if coms are different if (comA != comB) { let heightA = comA.find((node) => node.depth == 0).height; let heightB = comB.find((node) => node.depth == 0).height; if (heightB != heightA) { if (heightB > heightA) { const comTemp = comB, temp = b, heightTemp = heightB; comB = comA; comA = comTemp; b = a; a = temp; heightB = heightA; heightA = heightTemp; } // comA is heigher than comB // we have to merge at heightB + 1 // we need to find the sub tree of comA that contains the node a const nodeA = comA.find((node) => node.data.name == a); const ancestorA = nodeA .ancestors() .find((ancestor) => ancestor.height == heightB + 1); comB.parent = ancestorA; ancestorA.children.push(comB); comA = d3.hierarchy(comA); // recompute the hierarchy comA.each((d) => (d.data = { ...d.data.data, children: d.children })); // fixes the glitch communities = communities.filter((com) => com != comB); } else { // we need to add a common new ancestor to both communities let comC = d3.hierarchy({ name: "C-" + heightA + "-" + r }); comC.children = [comA, comB]; comC = d3.hierarchy(comC); comC.each((d) => (d.data = { ...d.data.data, children: d.children })); // fixes the glitch communities.push(comC); communities = communities.filter((com) => com != comA && com != comB); r++; } } console.table(communities.length); }); let comF = d3.hierarchy(communities[0]); comF.each((d) => (d.data = { ...d.data.data, children: d.children })); // fixes the glitch return comF; }33:T7a1,// Function that give a scale of # (colorNum) according to rainbow. It can rotate the array, shuffle them with a seed or darken // shuffleSeed = require("https://bundle.run/shuffle-seed@1.1.6") colorFunc = (colorNum, rotateNum = 0, seed = false, dark = false) => { function darken(color, k = 1) { const { l, c, h } = d3.lch(color); return d3.lch(l - 30 * k, c, h); } const rotate = (arr, d) => { for (let i = d; i < arr.length; i++) arr.unshift(arr.pop()); return arr; }; const colors = rotate( d3.quantize(d3.interpolateRainbow, colorNum + 1).slice(0, -1), rotateNum ); const colorsDark = colors.map((color) => darken(color, dark).toString()); const colorsCopy = JSON.parse(JSON.stringify(colors)); const colorsCopyDark = JSON.parse(JSON.stringify(colorsDark)); // const shuffeledColors = colorsCopy.sort(() => { // old methode doesn't required shuffleSeed // const randomTrueOrFalse = Math.random() > 0.5; // return randomTrueOrFalse ? 1 : -1; // }); // return shuffeled ? d3.scaleOrdinal(shuffeledColors) : d3.scaleOrdinal(colors); // if (seed != false) { // if (dark == false) { // return d3.scaleOrdinal(shuffleSeed.shuffle(colorsCopyDark, seed)); // } else { // return d3.scaleOrdinal(shuffleSeed.shuffle(colorsCopy, seed)); // } // } else { // if (dark != false) { // return d3.scaleOrdinal(colorsDark); // } else { // return d3.scaleOrdinal(colors); // } // } if (dark != false) { // either true or is a number if (seed != false) { // either true or is a number return d3.scaleOrdinal(shuffleSeed.shuffle(colorsCopyDark, seed)); } else { // return d3.scaleOrdinal(colorsCopyDark); } } else { if (seed != false) { // either true or is a number return d3.scaleOrdinal(shuffleSeed.shuffle(colorsCopy, seed)); } else { return d3.scaleOrdinal(colors); } } }34:[{"id":"699eb84f511adab5942830483a9d2896faa7b7329ae609077d596b4aef6d102420394e4b4c9989c9f1938c4744e6d2b23e401dd4a3b817afab39d5b1edf6fa75","href":"https://static.observableusercontent.com/files/699eb84f511adab5942830483a9d2896faa7b7329ae609077d596b4aef6d102420394e4b4c9989c9f1938c4744e6d2b23e401dd4a3b817afab39d5b1edf6fa75","name":"untitled.json","type":"application/json","lastModified":1735463196985.317,"size":4179},{"id":"537f2f60b6b16fb0c7750baab4359d3bca54cb5945781d4213d6feb1c4fce2046d029ee831ca3028a702b9a2a74ad3660fd6e81979745e5454b4b551c386d978","href":"https://static.observableusercontent.com/files/537f2f60b6b16fb0c7750baab4359d3bca54cb5945781d4213d6feb1c4fce2046d029ee831ca3028a702b9a2a74ad3660fd6e81979745e5454b4b551c386d978","name":"untitled (1).json","type":"application/json","lastModified":1735463201613.45,"size":4191},{"id":"ed8a661fce55fc7fd1adb3526f759699e0ef8a459289f427a4e120dabeec89076200a7d996f74b5c32562545023e5ae252f5569feed602357e376f83309e2327","href":"https://static.observableusercontent.com/files/ed8a661fce55fc7fd1adb3526f759699e0ef8a459289f427a4e120dabeec89076200a7d996f74b5c32562545023e5ae252f5569feed602357e376f83309e2327","name":"untitled (2).json","type":"application/json","lastModified":1735463205276.302,"size":4233},{"id":"0ad24c24de88563b47b1584da3370751301acb720d41c58f3b28fc664eb40952fa53b0c6b40a01defda4434d500c6735984d98cb8672d70d5a102e32cb7a5477","href":"https://static.observableusercontent.com/files/0ad24c24de88563b47b1584da3370751301acb720d41c58f3b28fc664eb40952fa53b0c6b40a01defda4434d500c6735984d98cb8672d70d5a102e32cb7a5477","name":"links2.json","type":"application/json","lastModified":1733383097192.273,"size":75894},{"id":"9d4829fab7f8e4f750080be70b39cdc329c5cfa60204ce827616be81e159c5eeea1a96a5477aa77f0e0d3d5f64ebf796160cd635c2388eb629ec4e465599f76c","href":"https://static.observableusercontent.com/files/9d4829fab7f8e4f750080be70b39cdc329c5cfa60204ce827616be81e159c5eeea1a96a5477aa77f0e0d3d5f64ebf796160cd635c2388eb629ec4e465599f76c","name":"links2@1.json","type":"application/json","lastModified":1733409534700.106,"size":75861},{"id":"722c5ad3f39f99f5a5cc29cafd79b0a190e6092e1dd0eb34d13def8adfa1832f6e156d4055eb8e1eb15443011398c9dcad79a51b56301a8c987b4f8962f87c27","href":"https://static.observableusercontent.com/files/722c5ad3f39f99f5a5cc29cafd79b0a190e6092e1dd0eb34d13def8adfa1832f6e156d4055eb8e1eb15443011398c9dcad79a51b56301a8c987b4f8962f87c27","name":"cover.jpg","type":"image/jpeg","lastModified":1733421787571.25,"size":53910},{"id":"e0cc6a4d543df42a274d0583957df6a9642813b299a7b50b3a743d879a8650f852a10c595d5cea3cce7a3d041fcaf54d9e1d4700bacc87ad4f15c7a089d30cdf","href":"https://static.observableusercontent.com/files/e0cc6a4d543df42a274d0583957df6a9642813b299a7b50b3a743d879a8650f852a10c595d5cea3cce7a3d041fcaf54d9e1d4700bacc87ad4f15c7a089d30cdf","name":"removed_edges.json","type":"application/json","lastModified":1733323408240.935,"size":35207},{"id":"720ab1aab59d3eedb7d65143746519242626c2fb6462c75334d2b59a37c1a951dbf64cbcff88b9a462e59bc0bb26dacd3567a7ac9615ce4f5b17856e4aff4c15","href":"https://static.observableusercontent.com/files/720ab1aab59d3eedb7d65143746519242626c2fb6462c75334d2b59a37c1a951dbf64cbcff88b9a462e59bc0bb26dacd3567a7ac9615ce4f5b17856e4aff4c15","name":"communities.json","type":"application/json","lastModified":1733323408339.059,"size":18605}] 2d:{"title":"The Illustrated Manual of Buddhist Terms and Doctrines (IMBTD)","cells":[{"id":0,"value":"# The Illustrated Manual of Buddhist Terms and Doctrines (IMBTD)\n\nThe goal of this project is to explore the graph created by mapping the links and references of the book: **Buddhist Dictionary: A Manual of Buddhist Terms and Doctrines**\nBy Nyanatiloka Thera\n\n","pinned":false,"hidden":false,"mode":"md"},{"id":425,"value":"cover = FileAttachment(\"cover.jpg\").image({ width: 300 })","pinned":false,"hidden":false,"mode":"ojs"},{"id":430,"value":"# References\n\n## PDF\n\n- [Buddhist Library PDF](https://www.buddhistelibrary.org/buddhism-online/e-books/palidictionary.pdf)\n- [Buddhist University PDF](https://buddhistuniversity.net/content/reference/buddhist-dictionary_nyanatiloka)\n\n## Online Version\n\n- [Pali Kanon](https://www.palikanon.com/english/wtb/dic_idx.html)\n\n## Bonus\n\nThe Illustrated History of Buddhism\nBy Ashin Janaka Bhivamsa (Aggamahapandita)\n\nArtist: U Ba Kyi\n\n- [Saraniya Website](https://saraniya.com/buddhism/buddhist-stories/the-illustrated-history-of-buddhism/)\n\n# Chart\n \nThe chart below explore the connections of the differents entries in the dictionnary. The text size is according to the number of entries the entry quotes or get quoted. The structure is revealed via the Hierarchical Edge Bundles for General Graphs Methode explained below. ","pinned":false,"hidden":false,"mode":"md"},{"id":218,"value":"viewof blackWhite = Inputs.toggle({ label: \"blackWhite\", value: false })","pinned":false,"hidden":false,"mode":"ojs"},{"id":224,"value":"viewof darkMode = Inputs.toggle({ label: \"darkMode\", value: false })","pinned":false,"hidden":false,"mode":"ojs"},{"id":249,"value":"viewof structure = Inputs.toggle({ label: \"Show Structure\", value: false })","pinned":false,"hidden":false,"mode":"ojs"},{"id":251,"value":"viewof edge = Inputs.toggle({ label: \"Show Edges\", value: true })","pinned":false,"hidden":false,"mode":"ojs"},{"id":216,"value":"viewof colorNumber = Inputs.range([0, 15], {\n label: \"colorNumber\",\n step: 1,\n value: 9\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":220,"value":"viewof colorSeed = Inputs.range([0, 100], {\n label: \"colorSeed\",\n step: 1,\n value: darkMode ? 0 : 0\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":222,"value":"viewof colorRotation = Inputs.range([0, 10], {\n label: \"colorRotation\",\n step: 1,\n value: 0\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":226,"value":"viewof darkenFactor = Inputs.range([0, 3], {\n label: \"darkenFactorNode\",\n step: 0.1,\n value: darkMode ? 0 : 1.2 // 1.4\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":306,"value":"viewof LighenFactor = Inputs.range([0, 3], {\n label: \"LightenFactorEdge\",\n step: 0.1,\n value: darkMode ? 0 : 0.3\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":228,"value":"viewof colorDepth = Inputs.range([0, 3], {\n label: \"colorDepth\",\n step: 1,\n value: 2\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":570,"value":"viewof beta = Inputs.range([0.6, 1.2], {\n label: \"beta curve\",\n step: 0.05,\n value: 0.95 // 0.85 default, 0.9 looks good as well\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":388,"value":"viewof PNGprecision = Inputs.range([1, 12], {\n label: \"PNG resolution factor\",\n step: 1,\n value: 3\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":404,"value":"DOM.download(() => rasterizeWhite(chart), undefined, \"Save as PNG\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":409,"value":"DOM.download(() => serialize(chart), undefined, \"Save as SVG\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":74,"value":"$2f","pinned":false,"hidden":false,"mode":"ojs"},{"id":414,"value":"# Processus Outline\n\n## Data Scraping\n\nThe online version above already contains all the main entries of the dictionnary as individual web page. We can collect all the from that website and build a network of relationship. \n\n### Making a local Copy\n\n\n\nWith httrack or wget we get a local copy of the website. You will need a local http-server to preview correctly the local site if you want to.\n\n## Mapping Nodes and Links\n\nUsing the following **node** script, we can extract the data from the website and get a json\n","pinned":false,"hidden":false,"mode":"md"},{"id":461,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":213,"value":"links = FileAttachment(\"links2@1.json\").json()","pinned":false,"hidden":false,"mode":"ojs"},{"id":520,"value":"From there, there are multiple ways to represent and analyse the network. We will follow a graph visualization approach that extracts the community structure of a network and organizes it into a more balanced and meaningful hierarchy so that its edge bundle rendering better indicates its structure from [_Yuntao Jia, Michael Garland, J. Hart_](https://www.semanticscholar.org/paper/Hierarchical-Edge-Bundles-for-General-Graphs-Jia-Garland/31f3ae917eaf2c1562274356cf7759d6a547cb69) \n\n## Hierarchical Edge Bundles for General Graphs\n\nin 2009, researchers have develop a [technique](https://www.semanticscholar.org/paper/Hierarchical-Edge-Bundles-for-General-Graphs-Jia-Garland/31f3ae917eaf2c1562274356cf7759d6a547cb69) to vizualise a graph of relationships by generating a hierarchy between the nodes and then using the well known Edge Bundling technic in a radial graph. We need first to break down the main network into subgraph with the **python** code below\n","pinned":false,"hidden":false,"mode":"md"},{"id":536,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"communitiesData = FileAttachment(\"communities.json\").json()","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"removed_edges = FileAttachment(\"removed_edges.json\").json()","pinned":false,"hidden":false,"mode":"ojs"},{"id":15,"value":"### Create D3 hierarchies","pinned":false,"hidden":false,"mode":"md"},{"id":11,"value":"communitiesH = communitiesData.map((com) =>\n d3.hierarchy({\n name: com.parent,\n children: com.children.map((value) => ({ name: value }))\n })\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"### Merging Communities\n\nOnce we have everything, we can merge the communities into a giant structure with D3","pinned":false,"hidden":false,"mode":"md"},{"id":27,"value":"$32","pinned":true,"hidden":false,"mode":"ojs"},{"id":581,"value":"## Extract community name list","pinned":false,"hidden":false,"mode":"md"},{"id":588,"value":"groupLeavesAtDepth(hierarchie, 3)","pinned":false,"hidden":false,"mode":"ojs"},{"id":583,"value":"function groupLeavesAtDepth(hierarchy, depth) {\n // Helper function to collect leaves for a node\n function collectLeaves(node) {\n return node\n .leaves()\n .map((leaf) => links.find((node) => node.file == leaf.data.name).title);\n }\n\n // Traverse the hierarchy and stop at the specified depth\n const result = [];\n hierarchy.each((node) => {\n if (node.depth === depth && node.children) {\n result.push(collectLeaves(node));\n }\n });\n\n return result;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":610,"value":"With this function, we can create list of the names at different root division and ask Chat GPT to come up with categories","pinned":false,"hidden":false,"mode":"md"},{"id":598,"value":"level1 = FileAttachment(\"untitled.json\").json()","pinned":false,"hidden":false,"mode":"ojs"},{"id":599,"value":"level2 = FileAttachment(\"untitled (1).json\").json()","pinned":false,"hidden":false,"mode":"ojs"},{"id":600,"value":"level3 = FileAttachment(\"untitled (2).json\").json()","pinned":false,"hidden":false,"mode":"ojs"},{"id":563,"value":"### Other Usefull things","pinned":false,"hidden":false,"mode":"md"},{"id":166,"value":"color = (t) => d3.interpolateRdBu(1 - t)","pinned":false,"hidden":false,"mode":"ojs"},{"id":114,"value":"line = d3\n .lineRadial()\n .curve(d3.curveBundle.beta(beta))\n .radius((d) => d.y)\n .angle((d) => d.x)","pinned":false,"hidden":false,"mode":"ojs"},{"id":118,"value":"function path([source, target]) {\n const p = new Path();\n line.context(p)(source.path(target));\n return p;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":120,"value":"class Path {\n constructor(_) {\n this._ = _;\n this._m = undefined;\n }\n moveTo(x, y) {\n this._ = [];\n this._m = [x, y];\n }\n lineTo(x, y) {\n this._.push(new Line(this._m, (this._m = [x, y])));\n }\n bezierCurveTo(ax, ay, bx, by, x, y) {\n this._.push(\n new BezierCurve(this._m, [ax, ay], [bx, by], (this._m = [x, y]))\n );\n }\n *split(k = 0) {\n const n = this._.length;\n const i = Math.floor(n / 2);\n const j = Math.ceil(n / 2);\n const a = new Path(this._.slice(0, i));\n const b = new Path(this._.slice(j));\n if (i !== j) {\n const [ab, ba] = this._[i].split();\n a._.push(ab);\n b._.unshift(ba);\n }\n if (k > 1) {\n yield* a.split(k - 1);\n yield* b.split(k - 1);\n } else {\n yield a;\n yield b;\n }\n }\n toString() {\n return this._.join(\"\");\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":122,"value":"class Line {\n constructor(a, b) {\n this.a = a;\n this.b = b;\n }\n split() {\n const { a, b } = this;\n const m = [(a[0] + b[0]) / 2, (a[1] + b[1]) / 2];\n return [new Line(a, m), new Line(m, b)];\n }\n toString() {\n return `M${this.a}L${this.b}`;\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":124,"value":"BezierCurve = {\n const l1 = [4 / 8, 4 / 8, 0 / 8, 0 / 8];\n const l2 = [2 / 8, 4 / 8, 2 / 8, 0 / 8];\n const l3 = [1 / 8, 3 / 8, 3 / 8, 1 / 8];\n const r1 = [0 / 8, 2 / 8, 4 / 8, 2 / 8];\n const r2 = [0 / 8, 0 / 8, 4 / 8, 4 / 8];\n\n function dot([ka, kb, kc, kd], { a, b, c, d }) {\n return [\n ka * a[0] + kb * b[0] + kc * c[0] + kd * d[0],\n ka * a[1] + kb * b[1] + kc * c[1] + kd * d[1]\n ];\n }\n\n return class BezierCurve {\n constructor(a, b, c, d) {\n this.a = a;\n this.b = b;\n this.c = c;\n this.d = d;\n }\n split() {\n const m = dot(l3, this);\n return [\n new BezierCurve(this.a, dot(l1, this), dot(l2, this), m),\n new BezierCurve(m, dot(r1, this), dot(r2, this), this.d)\n ];\n }\n toString() {\n return `M${this.a}C${this.b},${this.c},${this.d}`;\n }\n };\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":240,"value":"shuffleSeed = require(\"https://bundle.run/shuffle-seed@1.1.6\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":238,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":242,"value":"darken = (color, k = 1) => {\n const { l, c, h } = d3.lch(color);\n return d3.lch(l - 30 * k, c, h);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":244,"value":"lighten = (color, k = 1) => {\n const { l, c, h } = d3.lch(color);\n return d3.lch(l + 30 * k, c, h);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":392,"value":"import { serialize } from \"@mbostock/saving-svg\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":390,"value":"function rasterizeWhite(svg) {\n let resolve, reject;\n const promise = new Promise((y, n) => ((resolve = y), (reject = n)));\n const image = new Image();\n image.onerror = reject;\n image.onload = () => {\n const rect = svg.getBoundingClientRect();\n const context = DOM.context2d(\n rect.width * PNGprecision,\n rect.height * PNGprecision\n );\n context.fillStyle = \"white\";\n context.fillRect(\n 0,\n 0,\n rect.width * PNGprecision,\n rect.height * PNGprecision\n );\n context.drawImage(\n image,\n 0,\n 0,\n rect.width * PNGprecision,\n rect.height * PNGprecision\n );\n context.canvas.toBlob(resolve, \"image/png\", 1);\n };\n image.src = URL.createObjectURL(serialize(svg));\n return promise;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":400,"value":"function rasterize(svg) {\n let resolve, reject;\n const promise = new Promise((y, n) => ((resolve = y), (reject = n)));\n const image = new Image();\n image.onerror = reject;\n image.onload = () => {\n const rect = svg.getBoundingClientRect();\n const context = DOM.context2d(\n rect.width * PNGprecision,\n rect.height * PNGprecision\n );\n context.drawImage(\n image,\n 0,\n 0,\n rect.width * PNGprecision,\n rect.height * PNGprecision\n );\n context.canvas.toBlob(resolve, \"image/png\", 1);\n };\n image.src = URL.createObjectURL(serialize(svg));\n return promise;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":559,"value":"","pinned":true,"hidden":false,"mode":"html"}],"files":"$W34","stdlib":"1","titling":"auto"}