31:T58b,I want to understand how the mortality rate of covid is changing over time. This is a second attempt at the work I started in *[Charting a Moving Case Fatality Rate](https://observablehq.com/@harrislapiroff/charting-a-moving-case-fatality-rate)*. This time, instead of trying to chart CFR as a continuous value, I want to break an epidemic up into successive waves and calculate a CFR for each wave. The **case fatality rate** is the number of deaths divided by the number of documented infections ("cases"). Before we get started, let's acknowledge that the CFR is an imperfect metric in ways that we are *not* going to attempt to adjust for: * It will necessarily be higher than the **infection fatality rate** (IFR) because a certain number of infections will go unrecorded, since tests may not be available, people may opt not to get tested, or people may get tested in ways that do not get recorded in the public record (for example by a private institution or using an at home antigen test). * While we're most interested in seeing if the case fatality rate changes over time due to vaccinations and improved clinical treatments, it might also change over time as test availability/practices change (for example the CDC advised vaccinated people not to get tested); and these different reasons for a changing CFR may be difficult to separate! Let's look at one state in particular. I'll let you pick the state:32:Tc93,annotateStateData = (singleStateData) => { const blurredCases = blur(singleStateData.map(d => d['actuals.newCases'])) const blurredDeaths = blur(singleStateData.map(d => d['actuals.newDeaths'])) // A container to store state during map iterations const memo = { vaccinesInitiatedRatioCursor: 0, vaccinesCompletedRatioCursor: 0, caseWaveCursor: 0, deathWaveCursor: 0, } return singleStateData.map((d, i, arr) => ({ date: d.date, newCases: d['actuals.newCases'], newDeaths: d['actuals.newDeaths'], newCasesSmooth: blurredCases[i], newDeathsSmooth: blurredDeaths[i], caseMinimum: blurredCases[i - 1] > blurredCases[i] && blurredCases[i + 1] > blurredCases[i], deathMinimum: blurredDeaths[i - 1] > blurredDeaths[i] && blurredDeaths[i + 1] > blurredDeaths[i], vaccinesInitiatedRatio: d['metrics.vaccinationsInitiatedRatio'], vaccinesCompletedRatio: d['metrics.vaccinationsCompletedRatio'], state: d.state, })) .map((d, i, all) => { // This function fills empty vaccine data with the previous value memo.vaccinesInitiatedRatioCursor = d.vaccinesInitiatedRatio ? d.vaccinesInitiatedRatio : memo.vaccinesInitiatedRatioCursor memo.vaccinesCompletedRatioCursor = d.vaccinesCompletedRatio ? d.vaccinesCompletedRatio : memo.vaccinesCompletedRatioCursor return { ...d, vaccinesInitiatedRatio: memo.vaccinesInitatedRatioCursor, vaccinesCompletedRatio: memo.vaccinesCompletedRatioCursor, } }) .map((d, i, arr) => ({ ...d, // Find the next death peak data point and record it's index in the dataset nextDeathMinimumIdx: ( arr.slice(i).findIndex(d => d.deathMinimum) !== -1 ? i + arr.slice(i).findIndex(d => d.deathMinimum) : null ), })) .map((d, i, arr) => ({ ...d, // Fetch the next death minimum date nextDeathMinimum: ( d.nextDeathMinimumIdx ? arr[d.nextDeathMinimumIdx].date : null ) })) .map((d, i, arr) => ({ ...d, // Calculate the time in days until the next death minimum daysTillNextDeathMinimum: ( d.nextDeathMinimum ? d3.timeDay.range(d.date, d.nextDeathMinimum).length : null ) })) .map((d, i, arr) => ({ ...d, // Mark all case minima that have a corresponding death minimum isValidCaseMinimum: validMinimaPair(d), })) .map((d, i, arr) => ({ ...d, // Mark all death minima that have a corresponding case minimum isValidDeathMinimum: !!arr.find(p => p.isValidCaseMinimum && p.nextDeathMinimumIdx === i) })) .map((d, i, arr) => { // This function annotates datapoints with what wave they're part of // When it encounters a new minimum, it increments the wave number if (d.isValidCaseMinimum) memo.caseWaveCursor += 1 if (d.isValidDeathMinimum) memo.deathWaveCursor += 1 return { ...d, // Break datapoints into discrete wave categories according to how many case or death minima precede that datapoint caseWave: memo.caseWaveCursor, deathWave: memo.deathWaveCursor, } }) }33:Taae,waveCfrChart = (data, width = 300, height = 200) => { const waves = Array.from(new Set(data.map(d => d.caseWave))) .map(w => ({ wave: w, waveLabel: `Wave ${w}`, caseDates: d3.extent(data.filter(d => d.caseWave === w), d => d.date), deathDates: d3.extent(data.filter(d => d.deathWave === w), d => d.date), caseCount: d3.sum(data.filter(d => d.caseWave === w), d => d.newCases), deathCount: d3.sum(data.filter(d => d.deathWave === w), d => d.newDeaths), })) .map(d => ({ ...d, cfr: d.deathCount / d.caseCount, })) const vaccineMilestones = stateVaccineMilestones.find(([state, _]) => state === data[0].state)[1] const chart = Plot.plot({ color: { type: 'categorical', scheme: COLOR_SCHEME }, y: { label: 'CFR ↑', tickFormat: '.1%' }, width, height, marks: [ // Vaccine Milestone Ticks Plot.tickX(vaccineMilestones, { x: 'date', stroke: '#000', strokeOpacity: 0.15, }), // Vaccine Milestone % Labels width > 400 ? Plot.text(vaccineMilestones.filter((_, i) => i % 2 === 0), { x: 'date', y: d3.max(waves, w => w.cfr), text: d => d3.format('.0%')(d.approxVaccineRatio), dy: -5, }) : null, // Vaccine Milestone Label width > 400 ? Plot.text(vaccineMilestones, Plot.selectFirst({ x: 'date', y: d3.max(waves, w => w.cfr), text: d => 'Fully vaccinated population:', textAnchor: 'end', fontWeight: 'bold', dy: -5, dx: -15 })) : null, // Wave rectangles Plot.rect(waves, { x1: d => d.caseDates[0], x2: d => d.caseDates[1], y1: d => 0, y2: 'cfr', fill: 'wave', }), // Wave % labels Plot.text(waves, { x: d => d.caseDates[0], y: d => d.cfr, text: (d, i) => d3.format('.1%')(d.cfr) + (i + 1 < waves.length ? '' : '*'), textAnchor: 'start', fill: '#000', opacity: (d, i) => i + 1 < waves.length ? 1 : 0.5, stroke: '#FFF', strokeWidth: 4, dx: 4, dy: -5, }), Plot.ruleY([0]) ] }) // Replace the last rectangle with a textured color let fillColor try { fillColor = d3.select(chart).selectAll('rect:last-of-type').attr('fill') } catch { fillColor = '#000' } const texture = textures.lines().size(10).thicker().stroke(fillColor).background('white') d3.select(chart).call(texture) d3.select(chart).selectAll('rect:last-of-type').attr('fill', texture.url()) // Adjust the strokes on the text to be under the text d3.select(chart).selectAll('text').attr('paint-order', 'stroke') return chart }34:T58c,US_STATES = ({ "AL": "Alabama", "AK": "Alaska", // "AS": "American Samoa", "AZ": "Arizona", "AR": "Arkansas", "CA": "California", "CO": "Colorado", "CT": "Connecticut", "DE": "Delaware", "DC": "District Of Columbia", // "FM": "Federated States Of Micronesia", "FL": "Florida", "GA": "Georgia", // "GU": "Guam", "HI": "Hawaii", "ID": "Idaho", "IL": "Illinois", "IN": "Indiana", "IA": "Iowa", "KS": "Kansas", "KY": "Kentucky", "LA": "Louisiana", "ME": "Maine", // "MH": "Marshall Islands", "MD": "Maryland", "MA": "Massachusetts", "MI": "Michigan", "MN": "Minnesota", "MS": "Mississippi", "MO": "Missouri", "MT": "Montana", "NE": "Nebraska", "NV": "Nevada", "NH": "New Hampshire", "NJ": "New Jersey", "NM": "New Mexico", "NY": "New York", "NC": "North Carolina", "ND": "North Dakota", "MP": "Northern Mariana Islands", "OH": "Ohio", "OK": "Oklahoma", "OR": "Oregon", // "PW": "Palau", "PA": "Pennsylvania", "PR": "Puerto Rico", "RI": "Rhode Island", "SC": "South Carolina", "SD": "South Dakota", "TN": "Tennessee", "TX": "Texas", "UT": "Utah", "VT": "Vermont", // "VI": "Virgin Islands", "VA": "Virginia", "WA": "Washington", "WV": "West Virginia", "WI": "Wisconsin", "WY": "Wyoming" })35:[] 2f:{"title":"Charting a Case Fatality Rate by Wave","cells":[{"id":0,"value":"# Charting a Case Fatality Rate by Wave","pinned":false,"hidden":false,"mode":"md"},{"id":550,"value":"_This is amateur analysis. I welcome all feedback, particularly from statisticians or epidemiologists. ([harris@chromamine.com](mailto:harris@chromamine.com))_\n\n***","pinned":false,"hidden":false,"mode":"md"},{"id":89,"value":"$31","pinned":false,"hidden":false,"mode":"md"},{"id":100,"value":"viewof state = Inputs.select(\n statesSummary.map(d => d.state),\n { format: s => US_STATES[s] }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":117,"value":"Now, a look at their cases and fatalities over time. Our data comes from [CovidActNow](https://www.covidactnow.org/):","pinned":false,"hidden":false,"mode":"md"},{"id":137,"value":"html`\n ${[['Cases', 'newCases'], ['Deaths', 'newDeaths']].map(([ label, key ], i, arr) => Plot.plot({\n y: { label, tickFormat: 's' },\n x: { ticks: i === arr.length - 1 ? 5 : 0 }, // only display x-axis for last chart\n marks: [\n Plot.areaY(annotatedStateData, {\n x: 'date',\n y: key,\n }),\n Plot.tickY([0]),\n ],\n height: 100,\n }))}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":202,"value":"Now let's get a cleaner curve using [Gaussian smoothing](https://mathisonian.github.io/kde/) (actually we're using a fast Gaussian smoothing approximation provided by the excellent [array-blur](https://github.com/Fil/array-blur)):","pinned":false,"hidden":false,"mode":"md"},{"id":208,"value":"html`\n ${[['Cases ↑', 'newCases', 'newCasesSmooth'], ['Deaths ↑', 'newDeaths', 'newDeathsSmooth']].map(\n ([label, key, smoothKey], i, arr) => Plot.plot({\n y: { label, tickFormat: 's' },\n x: { ticks: i === arr.length - 1 ? 5 : 0 }, // only display x-axis for last chart\n marks: [\n Plot.areaY(annotatedStateData, {\n x: 'date',\n y: key,\n fill: '#CCC'\n }),\n Plot.tickY([0]),\n Plot.line(annotatedStateData, {\n x: 'date',\n y: smoothKey,\n stroke: '#068',\n })\n ],\n height: 100,\n })\n )}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":223,"value":"A 7-day average is more typical for smoothing covid case and death data because it evens out weekly fluctuations in reporting. But we used Gaussian smoothing instead because it makes it easier to identify local minima, which we will use to break the data into discrete waves:","pinned":false,"hidden":false,"mode":"md"},{"id":228,"value":"html`\n ${[\n ['Cases ↑', 'newCases', 'newCasesSmooth', 'caseMinimum'],\n ['Deaths ↑', 'newDeaths', 'newDeathsSmooth', 'deathMinimum']\n ].map(\n ([ label, key, smoothKey, minimaKey ], i, arr) => Plot.plot({\n y: { label, tickFormat: 's' },\n x: { ticks: i === arr.length - 1 ? 5 : 0 }, // only display x-axis for last chart\n marks: [\n Plot.areaY(annotatedStateData, {\n x: 'date',\n y: key,\n fill: '#CCC'\n }),\n Plot.tickY([0]),\n Plot.line(annotatedStateData, {\n x: 'date',\n y: smoothKey,\n stroke: '#068',\n }),\n Plot.dot(annotatedStateData, {\n filter: d => d[minimaKey],\n x: 'date',\n y: smoothKey,\n fill: '#068',\n stroke: null,\n }),\n ],\n height: 100,\n })\n )}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":238,"value":"We find the minima in cases that also correspond to minima in deaths:","pinned":false,"hidden":false,"mode":"md"},{"id":356,"value":"Plot.plot({\n marginLeft: 75,\n y:{ domain: ['Cases', 'Deaths', 'Delay'] },\n x: { grid: true },\n marks: [\n Plot.link(annotatedStateData, {\n filter: d => d.isValidCaseMinimum,\n x1: 'date',\n y1: d => 'Cases',\n x2: d => d.nextDeathMinimum,\n y2: d => 'Deaths',\n stroke: '#AAAA'\n }),\n Plot.dot(annotatedStateData, {\n filter: d => d.caseMinimum,\n x: 'date',\n y: d => 'Cases',\n stroke: null,\n fill: d => '#068',\n }),\n Plot.dot(annotatedStateData, {\n filter: d => d.deathMinimum,\n x: 'date',\n y: d => 'Deaths',\n stroke: null,\n fill: d => '#068',\n }),\n Plot.text(annotatedStateData, {\n filter: d => d.isValidCaseMinimum,\n x: d => d.nextDeathMinimum,\n y: d => 'Delay',\n text: d => d.daysTillNextDeathMinimum + 'd',\n }),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":411,"value":"We discard any case minima that don't appear to have corresponding death minimum in a reasonable time range (${minDeathDelay} to ${maxDeathDelay} days). Note that it has been documented that covid deaths occur within three weeks of diagnosis, but the delays here might be larger or smaller depending on reporting delays for cases or deaths.\n\n*Note: If you think the breakdown here looks wrong—either we're lumping multiple waves together or capturing things that are too small to be waves—you can tweak the settings below under **Advanced Tweaks** to change the algorithm.*\n\nWe can use the local case and death minima to break the **${US_STATES[state]}** epidemic data into separate waves:","pinned":false,"hidden":false,"mode":"md"},{"id":799,"value":"html`\n ${[\n ['Cases ↑', 'newCases', 'newCasesSmooth', 'isValidCaseMinimum', 'caseWave'],\n ['Deaths ↑', 'newDeaths', 'newDeathsSmooth', 'isValidDeathMinimum', 'deathWave']\n ].map(\n ([ label, key, smoothKey, minimaKey, waveKey ], i, arr) => Plot.plot({\n color: { type: 'categorical', scheme: COLOR_SCHEME },\n y: { label, tickFormat: 's' },\n x: { ticks: i === arr.length - 1 ? 5 : 0 }, // only display x-axis for last chart\n marks: [\n Plot.areaY(annotatedStateData, {\n x: 'date',\n y: key,\n fill: waveKey,\n }),\n Plot.dot(annotatedStateData, {\n filter: d => d[minimaKey],\n x: 'date',\n y: smoothKey,\n fill: '#068',\n stroke: '#FFF',\n }),\n Plot.tickY([0]),\n ],\n height: 100,\n })\n )}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":810,"value":"And we calculate a case fatality rate for each wave by dividing the death count in that wave by the positive test count:","pinned":false,"hidden":false,"mode":"md"},{"id":839,"value":"waveCfrChart(annotatedStateData, 640, 320)","pinned":false,"hidden":false,"mode":"ojs"},{"id":913,"value":"_*Note: Since the most recent wave may still be in progress and deaths follow cases, it is likely an underestimate of the eventual CFR._\n\nThe case fatality rate could change between waves for a number of reasons:\n\n* Increased testing can increase the number of mild cases detected, driving the fatality rate down. Conversely, lack of testing drives the fatality rate up. You could have different case fatality rates in different contexts due to differences in testing, even if the (unknown) _infection fatality rate_ was the same.\n* Better medical care can drive the case fatality rate down. Conversely, an overwhelmed medical system will drive the case fatality rate up. This factor affects _both_ the case fatality rate and the infection fatality rate.\n* New variants could develop and spread that have higher or lower fatality rates.\n* Vaccines prevent many deaths, driving fatality rates down.","pinned":false,"hidden":false,"mode":"md"},{"id":465,"value":"## Advanced Tweaks\n\nHere's some values you can tweak. Play with these settings and scroll up to see how they change the calculations.","pinned":false,"hidden":false,"mode":"md"},{"id":525,"value":"### Start Date","pinned":false,"hidden":false,"mode":"md"},{"id":471,"value":"viewof startDate = date({\n min: d3.timeFormat('%Y-%m-%d')(d3.min(data, d => d.date)),\n max: d3.timeFormat('%Y-%m-%d')(d3.max(data, d => d.date)),\n value: \"2020-01-20\",\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":521,"value":"### Blur Radius\n\nLarger values will make our smoothed curves less noisy, smaller values will fit it more closely to the actual data.","pinned":false,"hidden":false,"mode":"md"},{"id":508,"value":"viewof blurRadius = Inputs.range([0, 100], { value: 10, step: 1 })","pinned":false,"hidden":false,"mode":"ojs"},{"id":560,"value":"### Range\n\nWe look for matching death minima within a certain number of days of case minima. If it's too early or too late, it probably doesn't actually correspond to the same wave. You can change what constitutes too soon or late.","pinned":false,"hidden":false,"mode":"md"},{"id":566,"value":"viewof minDeathDelay = Inputs.range([0, 100], { value: 3, label: 'Min', step: 1 })","pinned":false,"hidden":false,"mode":"ojs"},{"id":572,"value":"viewof maxDeathDelay = Inputs.range([0, 100], { value: 35, label: 'Max', step: 1 })","pinned":false,"hidden":false,"mode":"ojs"},{"id":1012,"value":"## All States\n\nVax number is _fully vaccinated_ people as a percentage of the population.","pinned":false,"hidden":false,"mode":"md"},{"id":981,"value":"

\n ${annotatedStates.map(([s, d]) => htl.html.fragment`\n

\n ${US_STATES[s]}\n | ${d.filter(d => d.vaccinesCompletedRatio).length && d3.format('.1%')(d.filter(d => d.vaccinesCompletedRatio).slice(-1)[0].vaccinesCompletedRatio)} vax\n

\n ${waveCfrChart(d, width > 800 ? width / 4 : width / 2, width > 800 ? width / 8 : width / 4)}\n

\n `)}\n

","pinned":false,"hidden":false,"mode":"html"},{"id":286,"value":"***\n\n## Calculations","pinned":false,"hidden":false,"mode":"md"},{"id":6,"value":"statesSummary = d3.csv(`https://api.covidactnow.org/v2/states.csv?apiKey=${API_KEY}`)","pinned":true,"hidden":false,"mode":"ojs"},{"id":10,"value":"data = d3.csv(`https://api.covidactnow.org/v2/states.timeseries.csv?apiKey=${API_KEY}`, d3.autoType)","pinned":true,"hidden":false,"mode":"ojs"},{"id":943,"value":"groupedData = d3.group(data, d => d.state)","pinned":true,"hidden":false,"mode":"ojs"},{"id":401,"value":"validMinimaPair = d => (\n // Make sure this point is a case minimum and there's a death minimum after it\n d.caseMinimum &&\n d.nextDeathMinimum &&\n // Exclude peaks that seem either too soon or too late to be related\n d.daysTillNextDeathMinimum > minDeathDelay &&\n d.daysTillNextDeathMinimum < maxDeathDelay\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":291,"value":"blur = arr => ArrayBlur.blur().radius(blurRadius)(arr)","pinned":true,"hidden":false,"mode":"ojs"},{"id":955,"value":"$32","pinned":true,"hidden":false,"mode":"ojs"},{"id":952,"value":"annotatedStates = Array.from(groupedData).map(([state, d]) => {\n return [state, annotateStateData(d)]\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":121,"value":"annotatedStateData = annotatedStates\n .find(([state_, d]) => state_ === state)[1]\n .filter(d => d.date > d3.timeParse('%Y-%m-%d')(startDate))","pinned":true,"hidden":false,"mode":"ojs"},{"id":1097,"value":"stateVaccineMilestones = {\n return annotatedStates.map(([state, data]) => {\n return [\n state,\n d3.range(0.1, 1, 0.1).map(p => {\n const closestPoint = data[d3.bisectLeft(data.map(d => d.vaccinesCompletedRatio), p)]\n return {\n state,\n approxVaccineRatio: p,\n actualVaccineRatio: closestPoint ? closestPoint.vaccinesCompletedRatio : null,\n date: closestPoint ? closestPoint.date : null,\n }\n })\n ]\n })\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":989,"value":"## Rendering Utilities","pinned":false,"hidden":false,"mode":"md"},{"id":993,"value":"$33","pinned":true,"hidden":false,"mode":"ojs"},{"id":18,"value":"## Appendix","pinned":false,"hidden":false,"mode":"md"},{"id":8,"value":"API_KEY = 'd7978daa8e1d48cf84c1f16f0e6428c0'","pinned":false,"hidden":false,"mode":"ojs"},{"id":277,"value":"ArrayBlur = require(\"array-blur@^1.2.1\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":473,"value":"import { date } from '@jashkenas/inputs'","pinned":false,"hidden":false,"mode":"ojs"},{"id":899,"value":"COLOR_SCHEME = 'tableau10'","pinned":false,"hidden":false,"mode":"ojs"},{"id":1050,"value":"textures = require('textures')","pinned":false,"hidden":false,"mode":"ojs"},{"id":1215,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}