31:T105d,chart2 = { const height = 100; var extra = CORES * 2 + 3*15 + 80; const color = d3.scaleOrdinal().domain(PHASES.slice(1).map(d => d.name)).range(d3.schemeCategory10); const xScale = d3.scaleLinear() .domain([0, d3.max(PHASES, d => d.end)]) .range([0, width]); var svg = d3.create("svg") .attr('width', width) .attr('height', height) .style("font-family", "sans-serif") .style("font-size", "12px"); const groups = svg.selectAll("g") .data(PHASES.slice(1)) .join("g") .attr("transform", d => `translate(${xScale(d.start)},10)`); groups.append('line') .attr("y1", height) .attr("y2", height+extra-30) .attr("height", 1000) .attr('stroke', d => color(d.name)); groups.append('rect') .attr("width", computeWidth) .attr("height", height) .attr('fill', d => color(d.name)) .attr('fill-opacity', '0.1') .append('title').text(d => d.summary); groups.append('rect') .attr("width", computeWidth) .attr("y", d => height - height * (d.cpu / d.wall) / CORES) .attr("height", d => height * (d.cpu / d.wall) / CORES) .attr('fill', d => color(d.name)) .attr('fill-opacity', '0.7') .append('title').text(d => d.summary); groups.append("text") .attr("y", d => height - height * (d.cpu / d.wall) / CORES - 5) .attr("x", d => computeWidth(d)/2) .attr("text-anchor", "middle") .text(d => computeWidth(d) > 20 ? Math.round(10 * d.cpu / d.wall)/10 : ""); groups.filter(d => computeWidth(d) > 10 && computeWidth(d) < 50) .append('text') .attr('x', height + 4) .attr('y', -4) .attr('transform', `rotate(90)`) .style('font-weight', 'bold') .text(d => d.name); var wideGroups = groups.filter(d => computeWidth(d) >= 50).append('g').attr('transform', `translate(0,${height})`); wideGroups .append('text') .attr('x', 3) .attr('y', 12) .style('font-weight', 'bold') .text(d => d.name); var threadGroups = wideGroups .selectAll('g.thread') .data(d => d.threads.reduce((prev, next) => [...prev, {...next, prevWorkers: prev[prev.length - 1]?.workers || 0}], [])) .join('g') .attr('class', 'thread') .attr('transform', (d, i) => `translate(0,${i * 15 + 25 + d.prevWorkers * 2})`) threadGroups.append('text') .text(d => d.name) .attr('x', 3) .attr('y', 0); var threadBars = threadGroups .selectAll('g') .data((d, i) => replicate({...d, threadGroup: i}, d.workers)) .join('g') .attr('transform', (d, i) => `translate(1,${3+i*2})`); threadBars.append('rect') .attr('height', d => d.workers == 1 ? 3 : 1) .attr('width', d => computeSecondWidth(d.cpu)) .attr('fill', 'steelBlue') .append('title').text(d => formatTime(d.cpu - d.sys) + " user"); // overlay threadBars.append('rect') .attr('height', d => d.workers == 1 ? 3 : 1) .attr('width', d => computeSecondWidth(d.sys)) .attr('fill', '#66FF00') .append('title').text(d => formatTime(d.sys) + " sys"); threadBars.append('rect') .attr('height', d => d.workers == 1 ? 3 : 1) .attr('width', d => computeSecondWidth(d.block)) .attr('x', d => computeSecondWidth(d.cpu)) .attr('fill', 'red') .append('title').text(d => formatTime(d.block) + " blocking"); threadBars.append('rect') .attr('height', d => d.workers == 1 ? 3 : 1) .attr('width', d => computeSecondWidth(d.wait)) .attr('x', d => computeSecondWidth(d.cpu + d.block)) .attr('fill', '#b6cee2') .attr('fill-opacity', 0.4) .append('title').text(d => formatTime(d.wait) + " waiting"); svg.attr('height', height + Math.max(extra, 150)); svg.append('g') .attr('transform', `translate(0,${height + extra - 20})`) .call(d3.axisBottom(d3.scaleLinear().domain([0, d3.max(PHASES, d => d.end)/60]).range([0, width])).tickFormat(d => d + ":00")); function computeWidth(item) { return xScale(item.end) - xScale(item.start); } function computeSecondWidth(seconds) { return xScale(seconds) - xScale(0); } function replicate(value, times) { var result = []; for (let i = 0; i < times; i++) { result.push(value); } return result; } return svg.node(); }32:T11b7,chart = { var sectionBuffer = 20; var workerHeight = 3; var queueHeight = 20; var topBuffer = 40; const progressLines = PROGRESS_LINES.map(d => ({ ...d, topology: d.topology.reduce((prev, curr, i) => [...prev, { ...curr, height: curr.type === 'worker' ? curr.workers.length * workerHeight : queueHeight }], []) })); var height = 0; for (const line of progressLines) { line.topology[0].top = 0; for (let i = 1; i < line.topology.length; i++) { const last = line.topology[i - 1].top; const lastHeight = line.topology[i - 1].height; line.topology[i].top = last + lastHeight + sectionBuffer; height = Math.max(height, line.topology[i].top + line.topology[i].height); } } height += topBuffer * 1.75; const xScale = d3.scaleLinear().domain([0, 20 + d3.max(progressLines, d => d.time)]).range([17, width]); const rectOffset = xScale(15) - xScale(0); const color = d3.scaleLinear().domain([0, 80, 100]).range(['#eee', 'black', 'red']); const svg = d3.create('svg') .attr('width', width) .attr('height', height) .style("font-family", "sans-serif") .style("font-size", "12px"); const times = svg.selectAll('g') .data(progressLines) .join('g') .attr('transform', d => `translate(${xScale(d.time)},0)`); const topologyComponents = times.selectAll('g') .data(d => d.topology || []) .join('g') .attr('transform', d => `translate(0,${topBuffer + d.top})`); // workers function renderWorkers(selection, height) { return selection .selectAll('rect') .data(d => (d.workers || [])) .join('rect') .attr('y', (d, i) => i * workerHeight) .attr('width', xScale(20) - xScale(0)) .attr('height', height) .style('shape-rendering', 'crispEdges') .attr('fill', d => typeof d === 'number' ? color(d) : 'white'); } topologyComponents.filter(d => d.type === 'worker' && d.workers.length === 1) .call(renderWorkers, workerHeight * 2); topologyComponents.filter(d => d.type === 'worker' && d.workers.length > 1) .call(renderWorkers, workerHeight); // GC const gcScale = d3.scaleLinear().domain([0, 20, 60]).range(['#eee', 'black', 'red']); svg.append('text') .attr('x', xScale(0) - 1) .attr('y', height - 1) .attr('text-anchor', 'end') .style('font-size', 10) .text('GC'); times.append('rect') .attr('y', height - workerHeight * 2) .attr('width', xScale(20) - xScale(0)) .attr('height', workerHeight * 2) .style('shape-rendering', 'crispEdges') .attr('fill', d => typeof d.gc === 'number' ? color(d.gc) : 'white'); // queues limit topologyComponents.filter(d => d.type === 'queue') .append('rect') .attr('width', xScale(20) - xScale(0)) .attr('height', queueHeight) .attr('fill', '#eee'); // queue used topologyComponents.filter(d => d.type === 'queue') .append('rect') .attr('width', xScale(20) - xScale(0)) .attr('y', d => queueHeight * (1 - d.used / d.limit)) .attr('height', d => queueHeight * (d.used / d.limit)) .attr('fill', 'steelBlue'); // phases const firsts = progressLines .filter((d, i, l) => i === 0 || d.phase !== l[i - 1].phase); for (let i = 0; i < firsts.length; i++) { if (i === firsts.length - 1) { firsts[i].width = width - xScale(firsts[i].time); } else { firsts[i].width = xScale(firsts[i + 1].time) - xScale(firsts[i].time); } } const phaseStarts = svg.selectAll('g.label') .data(firsts) .join('g') .attr('class', 'label') .attr('transform', d => `translate(${xScale(d.time)},0)`); phaseStarts .filter(d => d.width > 30) .selectAll('text') .data(d => (d.topology || []).filter(d => d.name)) .join('text') .attr('x', 2) .attr('y', d => topBuffer + d.top - 5) .text(d => d.name + '(' + d.workers.length + ')'); phaseStarts.append('line') .attr('y1', 0) .attr('y2', height) .attr('stroke', 'black'); phaseStarts.filter(d => d.width > 30) .append('text') .attr('x', 2) .attr('y', 10) .style('font-weight', 'bold') .text(d => d.phase); svg.selectAll('line.phaseend') .data([...OSM_PHASE_ENDS, ...MBTILES_ZOOM_ENDS]) .join('line') .attr('class', 'phaseend') .attr('x1', d => rectOffset + xScale(d.time)) .attr('x2', d => rectOffset + xScale(d.time)) .attr('y1', topBuffer) .attr('y2', height) .attr('stroke', 'black') .attr('stroke-width', 0.5) .attr('stroke-dasharray', '3 1'); return svg.node(); }33:T414,PROGRESS_LINES = LOGS .replace(/[\r\n]/g, '__LINEBREAK__') .replace(/([0-9]),([0-9])/g, '$1.$2') .replace(/__LINEBREAK__([0-9]{1,2}:[0-9]{2}:[0-9]{2})/g, "\n$1") .split(/[\r\n]/g) .filter(d => / -> /.test(d)) .map(line => { const [,time, phase, gc, topologyString] = /([0-9:]*) .*? \[(.*?)\] .*?gc:\s+([0-9]+)[ ]*% .*?([a-z]*$.*)/.exec(line); const topology = topologyString .split(' -> ') .map(d => d.trim()) .map(part => { const queue = /\(([0-9\.kmbtq ]+)\/([0-9\.kmbtq ]+)$/i.exec(part); const worker = /(.*?)$(.*?)$/.exec(part); return queue ? { type: 'queue', used: parseNum(queue[1]), limit: parseNum(queue[2]) } : worker ? { type: 'worker', name: worker[1], workers: worker[2].split(/[\s\%]+/).filter(d => d).map(d => d === '-' ? undefined : Number(d)) } : (() => { throw new Error("Unrecognized topology part: " + part) })(); }); return { time: parseLogTime(time), gc: Number(gc), phase, topology } })34:T4f7,PHASES = SUMMARY.trim().split("\n").map(line => { return line.split("\t")[1]; }).filter(d => d).reduce((accumulator, currentValue) => { if (currentValue.startsWith(" ")) { const currentPhase = accumulator[accumulator.length - 1]; const data = currentValue.trim(); const [,name,workers,cpu] = /\s+([^\s]+)\s+([0-9]+)x.*? ([0-9\.msh]+)/.exec(currentValue) || []; const [,sys] = /sys:(.*?) /.exec(currentValue) || []; const [,block] = /block:(.*?) /.exec(currentValue) || []; const [,wait] = /wait:(.*?) /.exec(currentValue) || []; const [,done] = /done:(.*?) /.exec(currentValue) || []; currentPhase.threads.push({ name, workers: Number(workers), cpu: parseTime(cpu), sys: parseTime(sys), block: parseTime(block), wait: parseTime(wait), done: parseTime(done) }); return accumulator; } else { const [name, wall, cpu] = currentValue.split(/ +/); const lastEnd = accumulator.length > 1 ? accumulator[accumulator.length - 1].end : 0; const data = { name, start: lastEnd, end: lastEnd + parseTime(wall), wall: parseTime(wall), cpu: parseTime(cpu), threads: [], summary: currentValue }; return [...accumulator, data]; } }, []);35:[{"id":"6a62b94bcd995e3dd84e8e0affe35e09672e10710de0d97f4a9ec6cf87b604846062b467c975d38e5bbf3494849c4966b9e6c69775b812e5b071c870be66af56","href":"https://static.observableusercontent.com/files/6a62b94bcd995e3dd84e8e0affe35e09672e10710de0d97f4a9ec6cf87b604846062b467c975d38e5bbf3494849c4966b9e6c69775b812e5b071c870be66af56","name":"v0.3.0-planet-c6gd-128gb.txt","type":"text/plain","lastModified":1648687537003.554,"size":241692}] 2f:{"title":"Planetiler Log Visualizer","cells":[{"id":0,"value":"# Planetiler Log Visualizer\nThis notebook visualizes logs from a [planetiler](https://github.com/onthegomap/planetiler) run. Copy/paste your logs into the text box below to view locally, or fork this notebook and attach your logs to share.","pinned":false,"hidden":false,"mode":"md"},{"id":500,"value":"## Summary\nThis chart shows a summary of the CPU utilization of each phase (top) and each thread within each phase (bottom).","pinned":false,"hidden":false,"mode":"md"},{"id":181,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":741,"value":"■ system time ■ user time ■ blocked ■ waiting","pinned":false,"hidden":false,"mode":"md"},{"id":716,"value":"## Worker Threads\nThis chart shows the CPU utilization of each worker thread (white/black/red) and each queue (blue/grey) throughout the planetiler run.","pinned":false,"hidden":false,"mode":"md"},{"id":524,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":761,"value":"■ 0% ■ 80% ■ 100%","pinned":false,"hidden":false,"mode":"md"},{"id":460,"value":"## Logs\nFork and attach your own log file to share a link, or just copy/paste your logs into the text area below to see a breakdown.","pinned":false,"hidden":false,"mode":"md"},{"id":289,"value":"viewof _LOGS = {\n function stripAnsi(text) {\n return text.replace(/[\\\\u001B\\\\u009B][[\\\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\\\d\\\\/#&.:=?%@~_]+)*|[a-zA-Z\\\\d]+(?:;[-a-zA-Z\\\\d\\\\/#&.:=?%@~_]*)*)?\\\\u0007)|(?:(?:\\\\d{1,4}(?:;\\\\d{0,4})*)?[\\\\dA-PR-TZcf-nq-uy=><~]))/g, '');\n }\n var defaultValue = stripAnsi(await FileAttachment(\"v0.3.0-planet-c6gd-128gb.txt\").text());\n return Inputs.textarea({ monospace: true, value: defaultValue, width: '100%', rows: 10 })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":876,"value":"LOGS = {\n function stripAnsi(text) {\n return text.replace(/\\u001b[^m]*?m/g,\"\").replace(/[\\\\u001B\\\\u009B][[\\\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\\\d\\\\/#&.:=?%@~_]+)*|[a-zA-Z\\\\d]+(?:;[-a-zA-Z\\\\d\\\\/#&.:=?%@~_]*)*)?\\\\u0007)|(?:(?:\\\\d{1,4}(?:;\\\\d{0,4})*)?[\\\\dA-PR-TZcf-nq-uy=><~]))/g, '');\n }\n return stripAnsi(_LOGS)\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":505,"value":"## Utilities and data processing ...\nYou can ignore below this","pinned":false,"hidden":false,"mode":"md"},{"id":323,"value":"SUMMARY = LOGS.replace(/^.*?---------*/s, '').replace(/^.*?---------*/s, '').replace(/------.*$/s, '')","pinned":true,"hidden":false,"mode":"ojs"},{"id":474,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":722,"value":"OSM_PHASE_ENDS = LOGS.split(/[\\r\\n]+/g)\n .filter(line => /Finished (nodes|ways|relations)/.test(line))\n .map(line => {\n const [,time, phase, type] = /^([0-9:]+)[ a-z]*\\[([^:\\]]*).*?Finished ([a-z]+)/i.exec(line);\n return { time: parseLogTime(time), phase, type };\n });","pinned":false,"hidden":false,"mode":"ojs"},{"id":767,"value":"MBTILES_ZOOM_ENDS = LOGS.split(/[\\r\\n]+/g)\n .filter(line => /Finished (z[0-9]+)/.test(line))\n .map(line => {\n console.log(line);\n console.log(/^.../i.exec(line.trim()));\n const [,time, level] = /^([0-9:]+).*?Finished z([0-9]+)/i.exec(line.trim());\n return { time: parseLogTime(time), level: Number(level) };\n });","pinned":false,"hidden":false,"mode":"ojs"},{"id":88,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":358,"value":"CORES = Number(/threads=([0-9]+)/.exec(LOGS)[1])","pinned":false,"hidden":false,"mode":"ojs"},{"id":99,"value":"function parseTime(string) {\n if (!string) return 0;\n var [,hours = '0'] = /([0-9\\.]+)h/.exec(string) || [];\n var [,minutes = '0'] = /([0-9\\.]+)m/.exec(string) || [];\n var [,seconds = '0'] = /([0-9\\.]+)s/.exec(string) || [];\n return Number(hours) * 60*60 + Number(minutes) * 60 + Number(seconds);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":490,"value":"function parseLogTime(string) {\n if (!string) return 0;\n var [,hours = '0', minutes = '0', seconds = '0'] = /([0-9]+):([0-9]+):([0-9]+)/.exec(string) || [];\n return Number(hours) * 60*60 + Number(minutes) * 60 + Number(seconds);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":241,"value":"function formatTime(seconds) {\n var result = '';\n const hours = Math.floor(seconds / 3600);\n const minutes = Math.floor((seconds - hours * 3600) / 60);\n seconds = seconds % 60;\n return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds)}` :\n minutes > 0 ? `${minutes}:${pad(seconds)}` : seconds;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":261,"value":"function pad(num) {\n return num < 10 ? '0' + num : num;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":606,"value":"function parseNum(string) {\n let [,num,prefix] = /([0-9\\\\.]+)([a-z]*)/i.exec(string.trim());\n num = Number(num);\n prefix = prefix.toLowerCase();\n switch (prefix) {\n case 'q': num *= 1000;\n case 't': num *= 1000;\n case 'b': num *= 1000;\n case 'm': num *= 1000;\n case 'k': num *= 1000;\n }\n return num;\n}","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}