30:T72d,circleTextWithLine = (svg, radius, lineDist, x, y, d, i, perRow, total) => { const yside = 2 * radius * Math.sin(Math.PI / 4) const xside = 2 * radius * Math.cos(Math.PI / 4) const dx = radius - xside / 2; const dy = radius - yside / 2; var global = svg.append('g') .attr('class', 'state-instance') .attr('transform', 'translate(' + [ radius, radius] + ')'); const xPos = (i) => x+(i%perRow)*(radius*2+lineDist) const yPos = (i) => y+Math.floor(i/perRow)*(radius*2 + lineDist/2) global.append('g') .attr('transform',`translate(${xPos(i)}, ${yPos(i)})`) .append('path') .attr('d', circle(radius, 2)) .attr('stroke', 'black') global.append('foreignObject') .attr('x',xPos(i) - (xside/2)) .attr('y', yPos(i) - (yside/2)) .attr('width', xside) .attr('height', yside) .attr('color', 'black') .append('xhtml:div') .attr('style', 'height: 100%;') .append('xhtml:p') .html(`${d.count} at ${d.stage.split(/(?=[A-Z])/).join(" ")}`) .attr('style', 'position: absolute;left: 50%;top: 50%;transform:translate(-50%,-50%);text-align:center;font-size:15px;margin:2px;'); if (i != total -1) { global.append('path') .attr('d', d3.line()([[xPos(i)+radius, yPos(i)], [xPos(i)+radius+lineDist-markerWidth, yPos(i)]])) .attr('stroke', 'black') .attr('marker-end', 'url(#arrow)') .attr('fill', 'none'); global.append('foreignObject') .attr('x',xPos(i) + radius + lineDist/2 - xside/2) .attr('y', yPos(i) - yside - markerHeight) .attr('width', xside) .attr('height', yside) .attr('color', 'black') .append('xhtml:div') .attr('style', 'height: 100%;') .append('xhtml:p') .html(`${d.remainderPercentage.toFixed(2)*100} %`) .attr('style', 'position: absolute; left: 50%; bottom: 0px;transform: translateX(-50%); text-align:center;padding:2px;margin:2px;'); } }31:[] 2e:{"title":"Deals Dashboard","cells":[{"id":0,"value":"md`# Deals Dashboard`","pinned":false,"hidden":false,"mode":"ojs"},{"id":531,"value":"html`

WARNING: THE DATA HERE IS NOT RELIABLE YET!

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":14,"value":"viewof url = Text({label: 'GraphQL URL', value: \"https://lb.mainnet-us-east-1.filops.net/mainnet-dealbot-graphql/\", placeholder: \"Enter url for running controller Graph QL endpoint\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":368,"value":"viewof startDate = Text({type: \"date\", label: 'Starting On', value: startDateDefault})","pinned":false,"hidden":false,"mode":"ojs"},{"id":372,"value":"viewof endDate = Text({type: \"date\", label: 'Going Until', value: endDateDefault})","pinned":false,"hidden":false,"mode":"ojs"},{"id":36,"value":"viewof miner = Text({label: 'Filter By Miner', placeholder: \"Miner ID\", value: minerIDDefault})","pinned":false,"hidden":false,"mode":"ojs"},{"id":586,"value":"viewof clientVersionFilter = Inputs.checkbox(clientVersions, {label: \"Filter By Client Version\", format: versionString})","pinned":false,"hidden":false,"mode":"ojs"},{"id":590,"value":"viewof minerVersionFilter = Inputs.checkbox(minerVersions, {label: \"Filter By Miner Version\", format: versionString})","pinned":false,"hidden":false,"mode":"ojs"},{"id":632,"value":"md `*Not checking any versions means show all*`","pinned":false,"hidden":false,"mode":"ojs"},{"id":142,"value":"md`### Average Storage Deal Success Rate: ${(dealSuccessRate(filteredStorageTasks)*100).toFixed(2)} %`","pinned":false,"hidden":false,"mode":"ojs"},{"id":439,"value":"md`### Average Retrieval Deal Success Rate: ${(dealSuccessRate(filteredRetrievalTasks)*100).toFixed(2)} %`","pinned":false,"hidden":false,"mode":"ojs"},{"id":486,"value":"md`### % Deals Failed Because Of Undialable Miners: ${(undialableRate(versionFilteredTasks)*100).toFixed(2)} %`","pinned":false,"hidden":false,"mode":"ojs"},{"id":517,"value":"md`### % Average Time To First Byte (retrieval deals): ${avgTimeToFirstByte(filteredRetrievalTasks).toFixed(2)} MS`","pinned":false,"hidden":false,"mode":"ojs"},{"id":953,"value":"md`### Storage Deal Stage Diagram`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1068,"value":"md`Total attempted deals: ${filteredStorageTasks.length}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":954,"value":"stageDiagram(filteredStorageTasks)","pinned":false,"hidden":false,"mode":"ojs"},{"id":946,"value":"md`### Retrieval Deal Stage Diagram`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1053,"value":"md`Total attempted deals: ${filteredRetrievalTasks.length}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":940,"value":"stageDiagram(filteredRetrievalTasks)","pinned":false,"hidden":false,"mode":"ojs"},{"id":187,"value":"md`### Storage Deal Errors: Histogram`","pinned":false,"hidden":false,"mode":"ojs"},{"id":62,"value":"failureBoard(filteredStorageTasks)","pinned":false,"hidden":false,"mode":"ojs"},{"id":455,"value":"md`### Retrieval Deal Errors: Histogram`","pinned":false,"hidden":false,"mode":"ojs"},{"id":460,"value":"failureBoard(filteredRetrievalTasks)","pinned":false,"hidden":false,"mode":"ojs"},{"id":57,"value":"md `## Appendix`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1015,"value":"md`### Derived Data`","pinned":false,"hidden":false,"mode":"ojs"},{"id":17,"value":"tasks = fetch(url +\"graphql?query=query {Tasks { All {UUID Status Stage ErrorMessage StartedAt StorageTask { Miner } RetrievalTask { Miner }}}}\").then((response) => response.json()).then((tasks) => tasks.data.Tasks.All)","pinned":false,"hidden":false,"mode":"ojs"},{"id":32,"value":"statusNames = [\"\", \"Available\", \"InProgress\", \"Successful\", \"Failed\"]","pinned":false,"hidden":false,"mode":"ojs"},{"id":397,"value":"endDateUnix = (endDate == formatDate(new Date()) ? (new Date()).getTime() : (new Date(endDate)).getTime())+1000*24*60*60","pinned":false,"hidden":false,"mode":"ojs"},{"id":386,"value":"startDateUnix = (new Date(startDate)).getTime()","pinned":false,"hidden":false,"mode":"ojs"},{"id":23,"value":"filteredTasks = tasks\n .filter(task => miner === \"\" ? true : taskMiner(task) === miner)\n .filter(task => (task.StartedAt / 1000000) >= startDateUnix && (task.StartedAt / 1000000) <= endDateUnix)\n .filter(task => statusNames[task.Status] != \"InProgress\" && statusNames[task.Status] != \"Available\")\n .filter((task) => !dropError(task))","pinned":false,"hidden":false,"mode":"ojs"},{"id":299,"value":"uuids = filteredTasks.map(task => task.UUID)","pinned":false,"hidden":false,"mode":"ojs"},{"id":283,"value":"filteredFinishedTasks = fetch(url+\"graphql\", { \n method: 'POST',\n headers: {\n 'Content-Type': 'application/json'\n },\n body: JSON.stringify({\n query: `query {FinishedTasks(UUIDs: ${JSON.stringify(uuids)}) { All { MinerLatencyMS TimeToFirstByteMS TimeToLastByteMS ClientVersion MinerVersion }}}`})\n}).then((response) => response.json())\n.then((tasks) => tasks.data.FinishedTasks.All.reduce((acc, task, n) => Object.assign(acc, {[uuids[n]]: task}), {}))","pinned":false,"hidden":false,"mode":"ojs"},{"id":563,"value":"clientVersions = sortBy(filteredTasks, clientVersion, \"version\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":584,"value":"minerVersions = sortBy(filteredTasks, minerVersion, \"version\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":625,"value":"versionFilteredTasks = filteredTasks.filter(matchesClientVersion).filter(matchesMinerVersion)","pinned":false,"hidden":false,"mode":"ojs"},{"id":409,"value":"filteredStorageTasks = versionFilteredTasks.filter(matchesTaskType('Storage'))","pinned":false,"hidden":false,"mode":"ojs"},{"id":416,"value":"filteredRetrievalTasks = versionFilteredTasks.filter(matchesTaskType('Retrieval'))","pinned":false,"hidden":false,"mode":"ojs"},{"id":783,"value":"commonStages = [\n\t\"MinerOnline\",\n\t\"QueryAsk\",\n\t\"CheckPrice\",\n\t\"ClientImport\",\n\t\"ProposeDeal\",\n]","pinned":false,"hidden":false,"mode":"ojs"},{"id":785,"value":"retrievalStages = [\n\t\"DealAccepted\",\n\t\"FirstByteReceived\",\n\t\"DealComplete\",\n]\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":787,"value":"storageDealStages = [\n\"StorageDealUnknown\",\n\"StorageDealValidating\",\n\"StorageDealProposalNotFound\",\n\"StorageDealProposalRejected\",\n\"StorageDealProposalAccepted\",\n\"StorageDealAcceptWait\",\n\"StorageDealReserveClientFunds\",\n\"StorageDealClientFunding\",\n\"StorageDealFundsReserved\",\n\"StorageDealStartDataTransfer\",\n\"StorageDealTransferring\",\n\"StorageDealWaitingForData\",\n\"StorageDealCheckForAcceptance\",\n\"StorageDealVerifyData\",\n\"StorageDealReserveProviderFunds\",\n\"StorageDealProviderFunding\",\n\"StorageDealPublish\",\n\"StorageDealPublishing\",\n\"StorageDealStaged\",\n\"StorageDealAwaitingPreCommit\",\n\"StorageDealSealing\",\n\"StorageDealActive\",\n\"StorageDealExpired\",\n\"StorageDealSlashed\",\n\"StorageDealRejecting\",\n\"StorageDealFailing\",\n\"StorageDealError\",\n\"StorageDealFinalizing\",\n\"StorageDealClientTransferRestart\",\n\"StorageDealProviderTransferAwaitRestart\",\n]","pinned":false,"hidden":false,"mode":"ojs"},{"id":261,"value":"params = new URLSearchParams(location.search.substring(1))","pinned":false,"hidden":false,"mode":"ojs"},{"id":354,"value":"startDateDefault = params.get(\"start_date\") || formatDate(oneWeekAgo())","pinned":false,"hidden":false,"mode":"ojs"},{"id":363,"value":"endDateDefault = params.get(\"end_date\") || formatDate(new Date())","pinned":false,"hidden":false,"mode":"ojs"},{"id":278,"value":"minerIDDefault = params.get(\"miner_id\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":982,"value":"md`### Helper Functions`","pinned":false,"hidden":false,"mode":"ojs"},{"id":52,"value":"matchesTaskType = (taskType) => (task) => taskType === \"All\" ? true : taskType === \"Storage\" ? task.StorageTask : task.RetrievalTask","pinned":false,"hidden":false,"mode":"ojs"},{"id":146,"value":"failures = (tasks, failureSort) => sortBy(\n tasks.filter((task) => statusNames[task.Status] === \"Failed\"), failureName(failureSort), 'status')","pinned":false,"hidden":false,"mode":"ojs"},{"id":548,"value":"sortBy = (tasks, sort, keyName) => Object.entries(\n tasks.reduce((acc, task) => Object.assign(acc, {[sort(task)]: acc[sort(task)] ? acc[sort(task)] + 1 : 1 }), {}))\n .map((statusEntry) => ({ [keyName]: statusEntry[0], count: statusEntry[1]}))","pinned":false,"hidden":false,"mode":"ojs"},{"id":789,"value":"stageIndex = (stage) => {\n return stage == null ? 0 : \n commonStages.includes(stage) ? commonStages.indexOf(stage) : \n storageDealStages.includes(stage) ? commonStages.length + storageDealStages.indexOf(stage) :\n retrievalStages.includes(stage) ? commonStages.length + retrievalStages.indexOf(stage) : -1\n}\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":811,"value":"orderByStage = (tasks) => sortBy(tasks, (task) => task.Stage, 'stage').sort(\n (a, b) => stageIndex(b.stage) < stageIndex(a.stage) ? 1 : -1)","pinned":false,"hidden":false,"mode":"ojs"},{"id":840,"value":"withRemainderPercentages = (stages, total) => stages.reduce((acc, stage) => ({\n stages : acc.stages.concat([Object.assign({remainderPercentage: (acc.remaining-stage.count)/acc.sum}, stage)]),\n sum: acc.sum,\n remaining: acc.remaining - stage.count\n}), { stages: [], remaining: total, sum: total }).stages","pinned":false,"hidden":false,"mode":"ojs"},{"id":40,"value":"taskMiner = (task) => task.StorageTask ? task.StorageTask.Miner : task.RetrievalTask.Miner","pinned":false,"hidden":false,"mode":"ojs"},{"id":572,"value":"prefixVersion = (base) => {\n if (base.includes(\"+\")) {\n return base.substring(0, base.indexOf(\"+\"))\n }\n return base\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":571,"value":"version = (base) => prefixVersion(base).replace(\"lotus-\", \"\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":644,"value":"versionString = (bucket) => bucket.version","pinned":false,"hidden":false,"mode":"ojs"},{"id":544,"value":"clientVersion = (task) => {\n const finishedTask = filteredFinishedTasks[task.UUID]\n if (!(finishedTask && finishedTask.ClientVersion)) {\n return \"\"\n }\n return version(finishedTask.ClientVersion)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":612,"value":"matchesClientVersion = (task) => clientVersionFilter.length > 0 ? clientVersionFilter.map(versionString).includes(clientVersion(task)) : true","pinned":false,"hidden":false,"mode":"ojs"},{"id":577,"value":"minerVersion = (task) => {\n const finishedTask = filteredFinishedTasks[task.UUID]\n if (!(finishedTask && finishedTask.MinerVersion)) {\n return \"\"\n }\n return version(finishedTask.MinerVersion)\n}\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":618,"value":"matchesMinerVersion = (task) => minerVersionFilter.length > 0 ? minerVersionFilter.map(versionString).includes(minerVersion(task)) : true","pinned":false,"hidden":false,"mode":"ojs"},{"id":121,"value":"failureName = (failureSort) => (task) => failureSort == \"stage\" ? task.Stage : cleanupError(task.ErrorMessage)","pinned":false,"hidden":false,"mode":"ojs"},{"id":462,"value":"dropError = (task) => {\n if ((task.ErrorMessage || \"\").includes(\"there is an active retrieval deal with\")) {\n return true\n }\n if ((task.ErrorMessage || \"\").includes(\"blockstore: block not found\")) {\n return true\n } \n if ((task.ErrorMessage || \"\").includes(\"missing permission to invoke\")) {\n return true\n }\n return false\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":200,"value":"cleanupError = (errorMessage) => {\n if (errorMessage.includes(\"failed to dial\")) {\n return \"failed to dial\"\n }\n if (errorMessage.includes(\"piece size less than minimum required size\")) {\n return \"piece size less than minimum required size\"\n }\n if (errorMessage.includes(\"storage price per epoch less than asking price\")) {\n return \"storage price per epoch less than asking price\"\n }\n if (errorMessage.includes(\"deal rejected\")) {\n return \"deal rejected -- other\"\n }\n if (errorMessage.includes(\"Unexpected deal response status: DealStatusErrored\")) {\n return \"Unexpected deal response status: DealStatusErrored\"\n }\n if (errorMessage.includes(\"timed out waiting 2m0s for Accept message from remote peer\")) {\n return \"data transfer failed: timed out waiting 2m0s for Accept message\"\n }\n return errorMessage\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":158,"value":"successes = (tasks) => tasks.filter(task => statusNames[task.Status] === \"Successful\")\n ","pinned":false,"hidden":false,"mode":"ojs"},{"id":163,"value":"dealSuccessRate = (tasks) => tasks.length == 0 ? 0 : (successes(tasks).length / tasks.length)","pinned":false,"hidden":false,"mode":"ojs"},{"id":482,"value":"undialable = (tasks) => tasks.filter((task) => task.Stage == \"MinerOnline\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":481,"value":"undialableRate = (tasks) => tasks.length == 0 ? 0 : undialable(tasks).length / tasks.length","pinned":false,"hidden":false,"mode":"ojs"},{"id":357,"value":"oneWeekAgo = () => { \n let date = new Date()\n date.setDate(date.getDate() - 7)\n return date\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":349,"value":"function formatDate(date) {\n return [\n (date.getFullYear()).toString().padStart(4, \"0\"),\n (date.getMonth() + 1).toString().padStart(2, \"0\"),\n (date.getDate()).toString().padStart(2, \"0\")\n ].join(\"-\")\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":511,"value":"timeToFirstByte = (task) => filteredFinishedTasks[task.UUID].TimeToFirstByteMS","pinned":false,"hidden":false,"mode":"ojs"},{"id":510,"value":"avgTimeToFirstByte = (tasks) => {\n let withTimeToByte = tasks.filter(hasTimeToFirstByte)\n return withTimeToByte.length == 0 ? 0 : withTimeToByte.reduce((acc, task) => acc + timeToFirstByte(task), 0) / withTimeToByte.length}","pinned":false,"hidden":false,"mode":"ojs"},{"id":522,"value":"hasTimeToFirstByte = (task) => filteredFinishedTasks[task.UUID] && filteredFinishedTasks[task.UUID].TimeToFirstByteMS != null","pinned":false,"hidden":false,"mode":"ojs"},{"id":992,"value":"md`### Visual Logic`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4,"value":"import {Text, Radio} from \"@observablehq/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":995,"value":"md`#### Histograms`","pinned":false,"hidden":false,"mode":"ojs"},{"id":672,"value":"height = 240","pinned":false,"hidden":false,"mode":"ojs"},{"id":55,"value":"import { vl } from \"@vega/vega-lite-api\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":446,"value":"failureBoard = (tasks) => vl.markBar()\n\n .data(failures(tasks, \"error\"))\n.transform(vl.calculate().calculate(\"split(datum.status, ' ')\").as('Status'))\n .width(width - 220)\n .height(width/3-20)\n.encode(\n vl.x().fieldN(\"Status\").axis({ title: 'Status', labelAngle: 0}).sort(\"count\"),\n vl.y().fieldQ(\"count\").axis({ title: 'Count'}),\n vl.color().field(\"Status\")\n )\n .render()","pinned":false,"hidden":false,"mode":"ojs"},{"id":998,"value":"md`#### Stage Diagrams`","pinned":false,"hidden":false,"mode":"ojs"},{"id":689,"value":"d3 = require(\"d3@6\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":879,"value":"arrowPoints = [[0, 0], [0, 20], [20, 10]];","pinned":false,"hidden":false,"mode":"ojs"},{"id":885,"value":"markerBoxWidth = 20","pinned":false,"hidden":false,"mode":"ojs"},{"id":886,"value":"markerBoxHeight = 20;","pinned":false,"hidden":false,"mode":"ojs"},{"id":888,"value":"refX = markerBoxWidth / 2","pinned":false,"hidden":false,"mode":"ojs"},{"id":899,"value":"refY = markerBoxHeight / 2","pinned":false,"hidden":false,"mode":"ojs"},{"id":900,"value":"markerWidth = markerBoxWidth / 2;","pinned":false,"hidden":false,"mode":"ojs"},{"id":902,"value":"markerHeight = markerBoxHeight / 2;","pinned":false,"hidden":false,"mode":"ojs"},{"id":705,"value":"circle = (radius, w) => d3.arc().innerRadius(radius-w).outerRadius(radius).startAngle(0).endAngle(2*Math.PI)()","pinned":false,"hidden":false,"mode":"ojs"},{"id":676,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":1019,"value":"plotStages = (stagesToPlot, radius, lineDist, x, y, perRow) => {\n const svg = d3.create('svg')\n .attr('width', width)\n .attr('height', y + (Math.ceil(stagesToPlot.length/perRow)*radius*2) + (Math.floor(stagesToPlot.length/perRow)*lineDist/2))\n \n svg.append('defs')\n .append('marker')\n .attr('id', 'arrow')\n .attr('viewBox', [0, 0, markerBoxWidth, markerBoxHeight])\n .attr('refX', refX)\n .attr('refY', refY)\n .attr('markerWidth', markerBoxWidth)\n .attr('markerHeight', markerBoxHeight)\n .attr('orient', 'auto-start-reverse')\n .append('path')\n .attr('d', d3.line()(arrowPoints))\n .attr('stroke', 'black');\n \n stagesToPlot.forEach((item, index) => \n circleTextWithLine(svg, radius, lineDist, x, y, item, index, perRow, stagesToPlot.length))\n \n return svg.node()\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":694,"value":"stageDiagram = (tasks) => plotStages(withRemainderPercentages(orderByStage(tasks), tasks.length), 90, 50, 5, 5, 4) ","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}