32:T4f0,md`There may come a time in your life when you would like a computer to decide whether to plot some data series as a line or a set of bars. If that time comes, you could do worse than to use the [autocorrelation](https://en.wikipedia.org/wiki/Autocorrelation) of the series. Line charts ${miniline()} interpolate between points, which visually implies that each point depends on the last (like the series has inertia and wants to stay where it is), whereas each bar in a bar chart ${minibar()} is self-standing (like each point is independent). So the other day I asked “what’s the simplest way to measure the degree to which one point in a timeseries predicts the next” and my dear friend [Sam Shleifer](https://arxiv.org/abs/1903.09244) replied “\`ser.corr(ser.shift(1))\`”, a.k.a. [\`autocorr\`](https://pandas.pydata.org/pandas-docs/version/0.22/generated/pandas.Series.autocorr.html) in Python’s pandas library. (The rest of this notebook is JavaScript.) Autocorrelation is the [Pearson correlation coefficient](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient) of a series with a lagged version of itself, shifting every entry by some offset. Here, select an example and then drag the series left and right to change the offset:`33:T4d8,viewof example = { const w = width > 500 ? 80 : 50; const m = 5; const div = DOM.element("div"); const sel = d3.select(div); sel .selectAll("svg") .data(series) .enter() .append("svg") .attr("width", w) .attr("height", w) .attr("class", "pick-dataset") .each(function(d, i) { const sel = d3.select(this); const data = d.data.slice(0, 15); const x = d3 .scaleBand() .domain(data.map((d, i) => i)) .range([m, w - m]) .padding(0.2); const y = d3 .scaleLinear() .domain(d3.extent(data)) .range([w - m, m]); var tick = sel .selectAll("line.tick.shifted") .data(data) .enter() .append("line") .attr("class", "tick shifted") .attr("x1", (d, i) => x(i)) .attr("x2", (d, i) => x(i) + x.bandwidth()) .attr("y1", d => y(d) || 0) .attr("y2", d => y(d) || 0); }) .on("click", (_, d) => choose(d)) .append("svg:title").text(d => d.name); choose(series[0]); function choose(d) { sel.selectAll("svg").classed("active", dd => dd.name == d.name); div.value = d; div.dispatchEvent(new CustomEvent('input')); } return div; }34:T1a75,viewof demo = { var t = 0; const marginBottom = 50; const marginBetween = 35; const ratio = d3 .scaleLinear() .domain([964, 500]) .range([0.4, 1]) .clamp(true)(width); const height = width * ratio; const data = center(example.data.slice(0, Math.ceil(width / 9.75))); let unshifted = data.slice(0); const autocorrs = d3 .range(-data.length, data.length + 1) .map(t => [t, autocorr(data, t)]) .filter(d => !isNaN(d[1])); let shifted = center(lag(data, Math.round(t))); const posProduct = d => Math.sign(d[0]) === Math.sign(d[1]) ? Math.sign(d[0]) * Math.sqrt(Math.abs(d[0] * d[1])) : 0; const negProduct = d => Math.sign(d[0]) !== Math.sign(d[1]) ? -Math.sqrt(Math.abs(d[0] * d[1])) : 0; const x = d3 .scaleBand() .domain(data.map((d, i) => i)) .range([0, width]) .paddingInner(0.2); const y = d3 .scaleLinear() .domain(d3.extent(data)) .range([height - marginBottom - marginBetween, 0]); const line = d3 .line() .x((d, i) => x(i)) .y(d => y(d)); const posArea = d3 .area() .x((d, i) => (x(i) !== undefined ? x(i) : width)) .y0(d => y(0)) .y1(d => y(posProduct(d))) // .defined(d => d[0] !== undefined || d[1] !== undefined) .curve(d3.curveStepAfter); const negArea = d3 .area() .x((d, i) => (x(i) !== undefined ? x(i) : width)) .y0(d => y(0)) .y1(d => y(negProduct(d))) .defined(d => d[0] !== undefined && d[1] !== undefined) .curve(d3.curveStepAfter); var xcorr = d3 .scaleLinear() .domain([-data.length / 2, data.length / 2]) .range([0, width]); var ycorr = d3 .scaleLinear() .domain(d3.extent(autocorrs.map(d => d[1]).concat([0]))) .range([height - 13, height - marginBottom - 10]); var linecorr = d3 .line() .x(d => xcorr(d[0])) .y(d => ycorr(d[1])); const svg = d3 .select(this || init()) .attr("width", width) .attr("height", height) .attr("viewBox", `0,0,${width},${height}`); svg.property("value", Math.round(t)).dispatch("input"); function init(dom) { const svg = d3.select(DOM.svg(width, height)); svg.append("line").attr("class", "mean"); svg.append("line").attr("class", "unshifted marker"); svg .append("path") .attr("class", "shifted marker") .attr("marker-end", `url(${window.location.href}#arrowhead)`); svg .append("path") .attr("class", "posOverlap") .attr("fill", `url(${window.location.href}#diagonalHatch)`); svg .append("path") .attr("class", "negOverlap") .attr("fill", `url(${window.location.href}#negDiagonalHatch)`); svg.append("g").attr("class", "autocorrs"); svg .append("circle") .attr("class", "shifted") .attr("r", 4); svg .append("text") .attr("class", "dataset-label") .attr("y", "1.2em"); svg .append("text") .attr("class", "mean-label") .text("μ") .style("text-anchor", "end"); svg .append("text") .attr("class", "autocorrs-label") .text("autocorrelation") .style("font-variant", "small-caps"); svg .append("text") .attr("class", "drag-label") .text("← drag →") .style("font-size", "smaller") .style("font-style", "italic"); svg.append("rect").attr("class", "dragnet"); return svg.node(); } var tick = svg.selectAll("line.tick.normal").data(data); tick.exit().remove(); tick .enter() .append("line") .attr("class", "tick normal"); var tickShifted = svg.selectAll("line.tick.shifted").data(shifted); tickShifted.exit().remove(); tickShifted .enter() .append("line") .attr("class", "tick shifted"); svg .select("line.unshifted.marker") .attr("x1", xcorr(0)) .attr("x2", xcorr(0)) .attr("y1", ycorr.range()[0]) .attr("y2", ycorr.range()[1]); var autocorrCircles = svg .select(".autocorrs") .selectAll("circle") .data(autocorrs); autocorrCircles.exit().remove(); autocorrCircles .enter() .append("circle") .attr("r", 2) .merge(autocorrCircles) .attr("cx", d => xcorr(d[0])) .attr("cy", d => ycorr(d[1])) .attr("fill", d => (d[1] === 0 ? "#000" : d[1] > 0 ? "#00c" : "#c00")) .attr("stroke", "none"); svg.select(".autocorrs-label").attr("y", height - marginBottom - 15); svg .select(".drag-label") .attr("y", 100) .attr("x", 50) .attr("transform", "rotate(-10)"); svg .select(".dragnet") .attr("x", 0) .attr("y", 0) .attr("width", width) .attr("height", height) .call( d3.drag().on("drag", event => { svg.select(".drag-label").remove(); t += xcorr.invert(event.dx) - xcorr.invert(0); shifted = center(lag(data, Math.round(t))); unshifted = center( data.map((d, i) => (shifted[i] !== undefined ? d : undefined)) ); svg.property("value", Math.round(t)).dispatch("input"); render(); }) ); render(); function render() { const tr = Math.round(t); const current = autocorrs.filter(d => d[0] == tr)[0]; svg.select("text.dataset-label").text(example.name); svg .select("line.mean") .attr("x1", x.range()[0]) .attr("x2", x.range()[1]) .attr("y1", y(0)) .attr("y2", y(0)); svg .select("text.mean-label") .attr("x", x.range()[1]) .attr("y", y(0)) .attr("dy", "0.7em"); svg .select("path.shifted.marker") .style("visibility", tr === 0 ? "hidden" : "visible") .attr( "d", d3.line()([ [xcorr(0), ycorr(0)], [xcorr(tr) - Math.sign(tr) * 3, ycorr(0)] ]) ); svg .select("circle.shifted") .attr("cx", xcorr(tr)) .attr("cy", ycorr(current[1])) .style("fill", current[1] > 0 ? "#00c" : "#c00"); svg .selectAll("line.tick.normal") .data(unshifted) .style("visibility", d => (d !== undefined ? "visible" : "hidden")) .attr("x1", (d, i) => x(i)) .attr("x2", (d, i) => x(i) + x.bandwidth()) .attr("y1", d => y(d) || 0) .attr("y2", d => y(d) || 0); svg .selectAll("line.tick.shifted") .data(shifted) .style("visibility", d => (d !== undefined ? "visible" : "hidden")) .attr("x1", (d, i) => x(i)) .attr("x2", (d, i) => x(i) + x.bandwidth()) .attr("y1", d => y(d) || 0) .attr("y2", d => y(d) || 0); const areaData = d3.zip(unshifted, shifted); areaData.push([0, 0]); svg .select(".posOverlap") .datum(areaData) .attr("d", posArea); svg .select(".negOverlap") .datum(areaData) .attr("d", negArea); } return svg.node(); }35:T467,viewof threshold = { let val = 0.9 const w = Math.min(width, 640) - 20 const w2 = 50 const decide = series => autocorr(series, 1) > val ? line(series.slice(0,20), w2, w2) : bar(series.slice(0,20), w2, w2) const x = d3.scaleLinear() .domain([-0.05, 1]) .range([5, w - 5]) const div = html`

` div.querySelector("input").addEventListener('input', e => { val = e.target.value render() }) render() function render() { div.value = val div.dispatchEvent(new CustomEvent('input')) const previews = div.querySelector(".threshold-previews") const newPreviews = html`

${series.map(s => html`

${decide(s.data)}

`)}

` div.replaceChild(newPreviews, previews) } return div }36:T84d,md`If ${tex`X_i = Y_i`} for all ${tex`i`}, this simplifies to ${tex` \frac{\sum X_i X_i}{\sqrt{\sum X_i^2}\sqrt{\sum X_i^2}} = \frac{\sum X_i^2}{\sum X_i^2} = 1 `}. And if each ${tex`X_i`} is ${tex`-Y_i`}, it’ll be ${tex` \frac{\sum X_i (-X_i)}{\sqrt{\sum X_i^2}\sqrt{\sum (-X_i)^2}} = \frac{-\sum X_i^2}{\sum X_i^2} = -1 `}. When they’re neither equal nor opposites, but somewhere in-between, the top and bottom look sort of similar. But on the top (${tex`\sum X_i Y_i`}), we multiply each entry and _then_ sum them, whereas on the bottom (${tex`\sqrt{\sum X_i^2}\sqrt{\sum Y_i^2}`}), we square–sum–square-root the entries of each series, and _then_ multiply them. On the top, a negative ${tex`X`} at the same index ${tex`i`} as a positive ${tex`Y`} amounts to a negative contribution. On the bottom, the squaring means nothing can stay negative: an entry contributes nothing if it’s zero (i.e. at the mean), and something if it’s not. Insofar as there are ${tex`X_i`} and ${tex`Y_i`} with opposing signs counting against the top, the top will be less than the bottom, and the correlation will be less than one. If the opposing-sign pairs balance out the same-sign pairs, it’ll be zero. So unlike the covariance calculation near the start of this essay, which yields some arbitrary number like ${tex`${Math.round(cov(series[0].data, lag(series[0].data, 1)))}`} of this unit or ${tex`${String((cov(series[1].data, lag(series[1].data, 1))).toExponential(2)).split("e").join(" \\times 10^{")}}`} of that unit, we always get something between ${tex`1`} and ${tex`-1`} (dimensionless, since both top and bottom are in the original unit squared). That lets us say stuff like “use a line chart above an autocorrelation of ${tex`${threshold}`}”, and it doesn’t matter whether the units in the chart are babies born per day, or millions of dollars, or CO₂ parts per million. Pick two series, and their correlation will be computed as the lever’s center of gravity, with the negative (red) and positive (blue) parts of the numerator on top and the denominator (yellow) in the bottom middle:`37:T3b77,seesaw = { await visibility() const pWidth = Math.min(width, 640) const responsiveRatio = d3.scaleLinear() .domain([964, 500]) .range([0.4, 0.8]) .clamp(true) (width) const height = width * responsiveRatio const margin = 10 const n = 30 let data = d3.zip(correlX, correlY) .filter(d => d[0] !== undefined && d[1] !== undefined) .slice(0, n) const meanX = d3.mean(data.map(d => d[0])) const meanY = d3.mean(data.map(d => d[1])) data = data.map(d => [d[0] - meanX, d[1] - meanY]) const responsiveX = d3.scaleLinear() .domain([964, 500]) .range([0.8, 0.95]) .clamp(true) (width) const x = d3.scaleBand() .domain(data.map((d, i) => i)) .range([responsiveX * width, 1.5 * width]) .padding(0.2) const y0 = d3.scaleLinear() .domain(d3.extent(d3.merge(data).concat([0]))) .range([height - margin, margin]) // const y = d3.scaleLinear() // .range([0, y0(1) - y0(0)]) const y = d3.scaleLinear() .domain([0, Math.sqrt(Π(d3.transpose(data).map(d => Math.sqrt(Σ(d.map(d => d ** 2))) )))]) .range([0, -0.6 * height]) const color = d3.scaleLinear() .domain([-0.1, 0, 0.1]) .range(["#c00", "#000", "#00c"]) const calcCorr = data => Σ(data.map(Π)) / Π(d3.transpose(data).map(d => Math.sqrt(Σ(d.map(d => d ** 2))) )) const calcDenom = data => Π(d3.transpose(data).map(d => Math.sqrt(Σ(d.map(d => y(d) ** 2))) )) const calcDenomDimensions = data => d3.transpose(data).map(d => Math.sqrt(Σ(d.map(d => y(d) ** 2))) ) const ρx = d3.scaleLinear() .domain([-1, 1]) .range([0.35 * pWidth, 0.65 * pWidth]) const ρy0 = 0.4 * height const ρy1 = height - 30 const midpoint = (x.range()[0] + ρx(3)) / 2 const div = d3.select(DOM.element("div")) const texNum = tex`\sum X_i Y_i` const texDen = tex`\sqrt{\sum X_i^2} \sqrt{\sum Y_i^2}` texNum.classList.add("texNum") texNum.style.top = ρy0 - 30 + "px" // texNum.style.left = ρx(-3) + "px" texDen.classList.add("texDen") texDen.style.top = ρy0 + 7 + "px" // texDen.style.left = ρx(-3) + "px" div.node().appendChild(texNum) div.node().appendChild(texDen) const svg = div.append("svg") .attr("width", width) .attr("height", height) .style("position", "relative") .style("z-index", "-1") svg.append("line") .attr("class", "mean") .attr("x1", x.range()[0]) .attr("x2", x.range()[1]) .attr("y1", y0(0)) .attr("y2", y0(0)) svg.append("text") .attr("class", "mean-label") .text("μ") .attr("x", x.range()[0]) .attr("y", y0(0)) .attr("dy", "0.7em") const denominator = svg.append("g").attr("class", "denominator") denominator.append("path") .attr("class", "denominator") .attr("d", poly(square(0))) .attr("transform", `translate(${ρx(0)}, ${ρy0 + 2})`) svg.append("rect") .style("fill", `url(${window.location.href}#partialFadeToWhiteFlipped)`) .attr("height", height * 0.5) .attr("width", width * 2) .attr("x", - width / 2) .attr("y", height * 0.8) const meter = svg.append("g") .attr("class", "corellometer") meter.append("line") .attr("class", "lever") .attr("x1", ρx(-3)) .attr("x2", ρx(3)) .attr("y1", ρy0) .attr("y2", ρy0) const notch = meter.selectAll("line.notch").data([-1, 0, 1]).enter().append("line") .attr("class", "notch") .attr("x1", d => ρx(2 * d)) .attr("x2", d => ρx(2 * d)) .attr("y1", ρy0) .attr("y2", d => ρy0 + (d ? 4 : -4)) const fulcrum = meter.append("g") .attr("class", "fulcrum") .attr("transform", `translate(${ρx(0)}, 0)`) fulcrum.append("path") .attr("d", poly([ [0, ρy0 + 1], [6, ρy0 + 11], [2, ρy0 + 11], [0.5, ρy1 + 30], [-0.5, ρy1 + 30], [-2, ρy0 + 11], [-6, ρy0 + 11], ])) fulcrum.append("circle") .attr("cy", ρy1) .attr("r", 4) .style("fill", "rgb(0, 0, 0)") meter.append("path") .attr("class", "base") .attr("d", poly([ [ρx(-1.2), ρy1], [ρx(1.2), ρy1], [ρx(1.35), ρy1 + 30], [ρx(-1.35), ρy1 + 30], ])) meter.append("g") .attr("class", "axis") .attr("transform", `translate(-0.5, ${ρy1 - 0.5})`) .call(d3.axisBottom(ρx).ticks(5).tickSize(3) .tickFormat(d => (d === -0.5) ? "-½" : (d === 0.5) ? "½" : d)) meter.append("text") .attr("x", ρx(0)) .attr("y", ρy1 + 26) .text("correlation") .style("font-variant", "small-caps") .style("text-anchor", "middle") const g = svg.append("g") .attr("class", "series-hopper") const items = g.selectAll("g").data(data).enter().append("g") svg.append("rect") .style("fill", `url(${window.location.href}#fadeToWhite)`) .attr("height", height) .attr("width", width * 0.2) .attr("x", width * 0.9) svg.append("rect") .style("fill", `url(${window.location.href}#partialFadeToWhite)`) .attr("height", 500) .attr("width", width * 2) .attr("x", - width / 2) .attr("y", -500) items .attr("transform", (d, i) => `translate( ${x(i) + x.bandwidth() / 2}, ${y0(0)} )`) .each(function(d, i) { const sel = d3.select(this) sel.append("path") .attr("class", `interior ${d[0] * d[1] > 0 ? "positive" : "negative"}`) sel.append("line") .attr("class", "ascender normal") .attr("y1", y(d[0])) .attr("y2", y(d[0])) sel.append("line") .attr("class", "tick normal") .attr("x1", -x.bandwidth() / 2) .attr("x2", x.bandwidth() / 2) .attr("y1", y(d[0])) .attr("y2", y(d[0])) sel.append("line") .attr("class", "ascender shifted") .attr("y1", y(d[1])) .attr("y2", y(d[1])) sel.append("line") .attr("class", "tick shifted") .attr("x1", -x.bandwidth() / 2) .attr("x2", x.bandwidth() / 2) .attr("y1", y(d[1])) .attr("y2", y(d[1])) }) .transition() .duration(300) .delay((d,i) => i * 1000) .tween("attr.ascend", function(d, i) { const sel = d3.select(this) return function(t) { sel.select(".ascender.normal") .attr("y2", y(d[0]) * (1 - t)) sel.select(".ascender.shifted") .attr("y2", y(d[1]) * (1 - t)) sel.selectAll(".tick") .attr("x1", -x.bandwidth() / 2 * (1 - t)) .attr("x2", x.bandwidth() / 2 * (1 - t)) } }) .on("end", function(d, i) { d.denom = denominator.append("g") .attr("transform", `translate( ${x(0) + x.bandwidth() / 2}, ${y0(0)} )`) this.querySelectorAll(".ascender") .forEach(function(node) { const n = node.cloneNode(true) n.classList.remove("ascender") n.classList.add("ascender2") d.denom.node().appendChild(n) }) }) .transition() .duration(1000) .attr("transform", `translate(${midpoint}, ${ρy0 - 2})`) .tween("attr.rotate", function(d, i) { const sel = d3.select(this) const wlog = d.slice(0).sort(d3.ascending) const vectors = [ sel.select(d[0] <= d[1] ? ".normal" : ".shifted"), sel.select(d[0] <= d[1] ? ".shifted" : ".normal") ] const theta = [0, 0] if(wlog[0] >= 0 && wlog[1] >= 0) { theta[0] = 90 } else if(wlog[0] < 0 && wlog[1] < 0) { theta[0] = -90 theta[1] = -180 } else { theta[0] = 90 } const next = items.filter((dd, ii) => ii > i) const nextTransforms = [] next.each(function(_, ii) { nextTransforms.push(d3.interpolateString( d3.select(this).attr("transform"), `translate( ${x(ii) + x.bandwidth() / 2}, ${y0(0)} )` )) }) const denomDimensions = i ? calcDenomDimensions(data.slice(0, i)) : [0, 0] const denomTransform = d3.interpolateString( d.denom.attr("transform"), `translate( ${ρx(0) - denomDimensions[0] / 2}, ${ρy0 + 2 + denomDimensions[1]} )` ) const denomRotate0 = d3.interpolateNumber(0, d[0] > 0 ? 180 : 0) const denomRotate1 = d3.interpolateNumber(0, d[1] > 0 ? -90 : 90) const denomColor1 = d3.interpolate("black", "gold") return function(t) { vectors.forEach((v, i) => v.attr("transform", `rotate(${theta[i] * t})`)) sel.selectAll(".ascender") .style("stroke-width", 2 * (1 - t)) sel.select(".interior") .style("stroke-width", 0.5 * t) .attr("d", poly(parallelogram( ...wlog.map((d, i) => rotate(y(d), theta[i] * t)) ))) next.attr("transform", (_, ii) => nextTransforms[ii](t)) d.denom.attr("transform", denomTransform(t)) d.denom.select(".normal").attr("transform", `rotate(${denomRotate0(t)})`) .style("stroke", denomColor1(t)) d.denom.select(".shifted").attr("transform", `rotate(${denomRotate1(t)})`) .style("stroke", denomColor1(t)) } }) .on("end", function(d, i) { const denomDimensions = i ? calcDenomDimensions(data.slice(0, i)) : [0, 0] d.xlen = Math.sqrt(y(Math.abs(d[0])) ** 2 + denomDimensions[0] ** 2) d.ylen = Math.sqrt(y(Math.abs(d[1])) ** 2 + denomDimensions[1] ** 2) d.denom.select(".normal").remove() d.denom.select(".shifted").remove() const xg = d.denom.append("g").attr("class", "x pythag") xg.append("line").attr("class", "leg").attr("y2", -y(Math.abs(d[0]))) xg.append("line").attr("class", "old").attr("x2", denomDimensions[0]) xg.append("line").attr("class", "hyp") .attr("x2", denomDimensions[0]) .attr("y1", -y(Math.abs(d[0]))) .attr("stroke-dasharray", d.xlen) .attr("stroke-dashoffset", d.xlen) // .attr("opacity", 0) d.xgTheta = Math.atan2(Math.abs(y(d[0])), denomDimensions[0]) * 180 / Math.PI const yg = d.denom.append("g").attr("class", "y pythag") yg.append("line").attr("class", "leg").attr("x2", y(Math.abs(d[1]))) yg.append("line").attr("class", "old").attr("y2", -denomDimensions[1]) yg.append("line").attr("class", "hyp") .attr("y2", -denomDimensions[1]) .attr("x1", y(Math.abs(d[1]))) .attr("stroke-dasharray", d.ylen) .attr("stroke-dashoffset", d.ylen) // .attr("opacity", 0) d.ygTheta = Math.atan2(Math.abs(y(d[1])), denomDimensions[1]) * 180 / Math.PI }) .transition() .duration(1000) .tween("attr.ellify", function(d, i) { const sel = d3.select(this) const cum = data.slice(0, i) .filter(cur => Math.sign(Π(cur)) === Math.sign(Π(d))) .reduce((acc, cur) => acc + Π(cur.map(y)), 0) const A = d3.interpolateNumber(0, cum) const B = y(d[0]) * y(d[1]) const r = d3.interpolateNumber(Math.abs(Math.max(...d)/Math.min(...d)), 1) const sign = Math.sign(B) const curCorr = calcCorr(data.slice(0, i + 1)) const denomArea = calcDenom(data.slice(0, i + 1)) const denomDimensionsPrev = i ? calcDenomDimensions(data.slice(0, i)) : [0, 0] const denomDimensions = calcDenomDimensions(data.slice(0, i + 1)) const curNeg = Math.abs(Σ(data.slice(0, i + 1).filter(d => Π(d) < 0).map(d => Π(d.map(y))))) const curPos = Σ(data.slice(0, i + 1).filter(d => Π(d) >= 0).map(d => Π(d.map(y)))) const coef = (curNeg + curPos + denomArea) / (denomArea) const prev = items.filter((dd, ii) => ii <= i && Math.sign(Π(dd)) === sign) const prevTransforms = [] prev.each(function() { prevTransforms.push(d3.interpolateString( d3.select(this).attr("transform"), `translate( ${ρx(sign * coef) - sign * (Math.sqrt(Math.abs(cum + B)) / 2)}, ${ρy0 - 2} )` )) }) const prevOtherSign = items.filter((dd, ii) => ii <= i && Math.sign(Π(dd)) !== sign) const otherCum = data.slice(0, i) .filter(cur => Math.sign(Π(cur)) !== Math.sign(Π(d))) .reduce((acc, cur) => acc + Π(cur.map(y)), 0) const prevOtherSignTransforms = [] prevOtherSign.each(function() { prevOtherSignTransforms.push(d3.interpolateString( d3.select(this).attr("transform"), `translate( ${ρx(-sign * coef) + sign * (Math.sqrt(Math.abs(otherCum)) / 2)}, ${ρy0 - 2} )` )) }) const fulc = d3.interpolateString( fulcrum.attr("transform"), `translate( ${ρx(curCorr)}, 0 )` ) const fulcColor = d3.interpolateString( fulcrum.select("circle").style("fill"), color(curCorr) ) const denom = d3.interpolateString( denominator.select("path").attr("d"), poly([ [-denomDimensions[0] / 2, 0], [-denomDimensions[0] / 2, denomDimensions[1]], [denomDimensions[0] / 2, denomDimensions[1]], [denomDimensions[0] / 2, 0], ]) ) const notchI = [] notch.each(function(d) { notchI.push(d3.interpolateNumber( d3.select(this).attr("x1"), ρx(coef * d) )) }) const xTranslate = denomDimensions[0] - denomDimensionsPrev[0] const yTranslate = denomDimensions[1] - denomDimensionsPrev[1] const xgTranslate = d3.interpolateString( `translate(0,0) rotate(0 ${denomDimensionsPrev[0]} 0)`, `translate(${xTranslate/2}, ${yTranslate}) rotate(${d.xgTheta} ${denomDimensionsPrev[0]} 0)` ) const ygTranslate = d3.interpolateString( `translate(0,0) rotate(0 0 ${-denomDimensionsPrev[1]})`, `translate(${-xTranslate/2}, 0) rotate(${-d.ygTheta} 0 ${-denomDimensionsPrev[1]})` ) return function(t) { sel.select(".interior").attr("d", poly(getL(A(t), B, r(Math.sqrt(t))))) prev.attr("transform", (_, ii) => prevTransforms[ii](t)) prevOtherSign.attr("transform", (_, ii) => prevOtherSignTransforms[ii](t)) fulcrum.attr("transform", fulc(t)) .select("circle").style("fill", fulcColor(t)) denominator.select("path") .attr("d", denom(t)) notch .attr("x1", (_, ii) => notchI[ii](t)) .attr("x2", (_, ii) => notchI[ii](t)) const xg = d.denom.select(".x.pythag").attr("transform", xgTranslate(t)) xg.select(".hyp") // .attr("opacity", t) .attr("stroke-dashoffset", d.xlen * (1 - t) ** 4) xg.select(".leg").attr("opacity", 1 - t) xg.select(".old").attr("opacity", 1 - t) const yg = d.denom.select(".y.pythag").attr("transform", ygTranslate(t)) yg.select(".hyp") // .attr("opacity", t) .attr("stroke-dashoffset", d.ylen * (1 - t) ** 4) yg.select(".leg").attr("opacity", 1 - t) yg.select(".old").attr("opacity", 1 - t) } }) .on("end", function(d, i) { d.denom.remove() }) return div.node() }38:T5aa,md`

${bend(14)} This masses-on-a-fulcrum metaphor is a bit forced. Given existing total mass ${tex`m`}, adding a mass ${tex`d`} (here the denominator of the correlation) at ${tex`x=0`} divides the barycenter by ${tex`\frac{d+m}{m}`}, when we want to just divide by ${tex`d`}. So the existing masses ${tex`p, n`} (here the positive and negative parts of the numerator) magically slide to ${tex`x = \pm \frac{d+p+n}{d}`} to lever them up such that the barycenter equals the correlation.

The fact that the magnitude of the top is less than or equal to that of the bottom follows from the [Cauchy-Schwarz inequality](https://en.wikipedia.org/wiki/Cauchy%E2%80%93Schwarz_inequality). That fact in turn reminded my friend Colin that, if you consider the zero-mean series as vectors, their correlation is their [cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity) — often used in machine learning to measure the similarity of two things represented as vectors, be they documents or images or phonemes or faces. Which makes sense, since machine learning is largely based on the same statistical techniques. This pseudo-mechanical illustration does nothing to help you see that; nor that it minimizes the sum of the squared errors, which is why we use it; nor that the correlation coefficient is the ${tex`r`} in the linear regression’s coefficient of determination ${tex`r^2`}, from which it originally came. `39:T3118,md`Though named for Karl Pearson, the correlation coefficient was pioneered by his mentor, Sir Francis Galton. Galton was born in Birmingham, England in 1822 to Samuel Galton and Violetta Darwin; the latter was the daughter of Erasmus and the younger half-sister of Robert, father of Charles. When his cousin published _On the Origin of Species_ in 1859, Galton dedicated himself to extending similar thinking to within human populations. He took their family genius to be inheritable, and co-related to hereditary similarity, and sought to steer it. In 1865 he imagined a “Utopia” in which those who excel in an examination of mind and body are encouraged to marry and have children, and, if they do, rewarded with £5,000 and a royal ceremony. In 1883, he dubbed this pursuit “eugenic”. To that end of developing “the science of improving stock”, Galton studied eminent family trees, conducted large biological and psychological surveys, and ran experiments on rabbits and sweet peas. In 1885, having made 150,000 measurements of 10,000 people, from bodily proportions to reaction time to color perception, he tabulated the heights of 205 sets of parents and children. He found that, although very tall people indeed tended to have taller-than-average children, they tended not to be quite _as_ tall. He called this “regression towards mediocrity”; the line to which they regress is what we still call a regression (though he used the median where we now typically use the mean). As he looked over the page proofs of his book _Natural Inheritance_, he realized he had come to a far more generalizable measure of relationship, and in 1888 he formalized the then-colloquial term “correlation” as the slope of the normalized regression line. Among those who rallied to Galton’s new cause was the thirty-year-old mathematician Pearson, for whom correlation was the “key to the treasure chamber” that liberated science from mere causation and let mathematics finally be applied to social phenomena. A decade later, in Galton’s 1901 Huxley Lecture, he returned to the subject of eugenics, now framed probabilistically: “The faculties of future generations will necessarily be distributed according to laws of heredity, whose statistical effects are no longer vague, for they are measured and expressed in formulæ.” At a panel discussion in 1904, he said, “What nature does blindly, slowly and ruthlessly, man may do providently, quickly and kindly.” Pearson fawned over seeing one of the last great Victorian men of science “in the flesh”. George Bernard Shaw said, “There is now no reasonable excuse for refusing to face the fact that nothing but a eugenic religion can save our civilization from the fate that has overtaken all previous civilizations.” H. G. Wells was impressed but described the work as “premature”. Benjamin Kidd, himself a social Darwinist, warned, “It might renew, in the name of science, tyrannies that it took long ages of social evolution to emerge from.” On August 17, 1905, the German physician Alfred Ploetz wrote to Galton to send him a book he had written ten years prior, in a small town with only a small library, without knowledge of Galton or his eugenics: Grundlinien einer Rassenhygiene, or “Racial Hygiene Basics”. In 1904 he had started a journal, and in 1905 a society, and so began an international correspondence and collaboration. In 1906 his journal published German translations of Galton’s eugenics papers; in 1909 Ploetz offered Galton the title of honorary Vice-President of the society, which honour he gladly accepted. In 1910 they had tea. In America, the state of Indiana had passed the world’s first compulsory sterilization law in 1907. Charles Davenport, an American contributor to Pearson’s journal, founded the Eugenics Record Office in 1910 with Harry Laughlin, who drafted model sterilization legislation upon which eighteen more states passed their laws. For years Galton had been working on a idealistic novel like More’s _Utopia_ or Butler’s _Erewhon_; he called his _Kantsaywhere_. There, citizens were administered a eugenic examination; those who passed were granted a special marriage ceremony, and those who failed were sent to “Labour Colonies… under conditions that are not onerous, except that they must work hard and live in celibacy.” But he seemed intermittently embarrassed by the unpublished work of fiction, and did not publicly advocate for coercive policies. In 1907, at the age of 85, Galton wrote that the moment to act had not yet arrived. “Enough is already known to those who have studied the question to leave no doubt in their minds about the general results, but not enough to justify legislation or other action except in extreme cases. Continued studies will be required for some time to come, and the pace must not be hurried. When the desired fulness of information shall have been acquired then, and not till then, will be the fit moment to proclaim a ‘Jehad,’ or Holy War against customs and prejudices that impair the physical and moral qualities of our race.” In 1909, Sir Francis was knighted; in 1911, he died. Upon his death, his niece destroyed most of _Kantsaywhere_, and passed the surviving fragment to Pearson, who published it in his massive three-volume biography of Galton. Pearson continued Galton’s work as the Galton Chair of Eugenics at University College London, endowed for him in Galton’s will. In 1896, with a mathematical rigor that Galton had never mustered, he had formulated the correlation coefficient as a product-moment in terms of mean and standard deviation of a normal curve, and derived its probable error. He went on to conceive of p-values, the chi-squared test, principal component analysis, and the histogram. He was not oblivious to certain abuses of these tools; he was the first to speak of a “spurious” correlation, and wrote, “Are we not making a fetish of the method of least squares as others made a fetish of the normal distribution?” At the same time, he believed that civilization advanced “chiefly by way of war with inferior races.” G. K. Chesterton watched eugenic enthusiasm recede from 1914 to 1918 as “England went to war with the Superman in his native home.” Prussia was defeated, but what he saw as Prussianism was not, and in 1922 he published _Eugenics and Other Evils_. “There is no reason to suppose that Dr. Karl Pearson is any better judge of a bridegroom than the bridegroom is of a bride” — nor could he find such legitimate authority anywhere in the enterprise: >It happened one day that an atheist and a man were standing together on a doorstep; and the atheist said, “It is raining.” To which the man replied, “What is raining?” Now the same anarchic mystery that clings round the phrase, “_il pleut_,” clings round the phrase, “_il faut_.” In English it is generally represented by the passive mood in grammar, and the Eugenists and their like deal especially in it; they are as passive in their statements as they are active in their experiments. Lady Macbeth said, “Give me the daggers; I’ll let his bowels out.” The Eugenist would say, “In such cases the bowels should, etc.” But when the Eugenists say, “Conditions must be altered” or “Ancestry should be investigated,” or what not, it seems clear that they do not mean that the democracy must do it, whatever else they may mean. The question remains, therefore, whom they do instinctively trust when they say that this or that ought to be done. What is this flying and evanescent authority that vanishes wherever we seek to fix it? Who is the man who is the lost subject that governs the Eugenist’s verb? The man who today leaps to mind was at that time planning a putsch, which would land him in jail and soon snuff out all global enthusiasm for eugenics by virtue of implementing it. In 1923, disturbed by reports of miscegenation between German women and Black troops occupying the Rhineland, eminent plant geneticist Erwin Baur joined with Eugen Fischer and Fritz Lenz to write Menschliche Erblichkeitslehre und Rassenhygiene; the next year, imprisoned at Landsberg, Hitler read it. In 1927, the Supreme Court ruled 8-1 that compulsory sterilization did not violate due process. When the Nazi Party came to power in 1933, Ploetz, Lenz, Ernst Rüdin, and Heinrich Himmler were brought together for the Expert Committee on Questions of Population and Racial Policy. Their first act was a sterilization law modeled on Laughlin’s in America. Paris fell on June 14, 1940. Among those who had fled to Marseille was Simone Weil, a person unusually well-positioned between _l’esprit de géométrie_ and _l’esprit de finesse_; writing there, she defined force as “that ${tex`x`} that turns anybody who is subjected to it into a thing”. Similarly, normalization is that which allows a person to treat groups of people like sweet pea seeds. When I say “it doesn’t matter whether the units in the chart are babies born per day, or millions of dollars, or CO₂ parts per million”, a part of the soul ought to recoil in horror. I scroll up and down and my metaphorical devices, by which any subject may be submitted to the same machinery, remind me of Galton’s — except that his were better.

I’m scared of going all _ad hominem_ here with the squeamishness of the amateur (me) who, unable to judge a thing on its merits, defers to the taint of its provenance. The field of mathematics is more separable than most from the biographies of its pioneers, especially so long as it’s never applied. Others had already done similar math for different reasons: astronomers sought to combine their observations such that errors diminished instead of multiplying. The approach of Adrien-Marie Legendre in 1805, and Carl Friedrich Gauss in 1809, was to fit their solution so as to minimize the sum of the squares of the errors — which is just what regression does. But they were dealing in fallible measurements of known causal relationships, whereas Galton was dealing in fallible measurements of ambiguous relationships, and he sought measures of the strength of those relationships that the error theorists had not. (Galton knew of Gauss more for his normal distribution, and did not mainly build on him.) In 1846, Auguste Bravais drew a line in a diagram that is, precisely, a regression, but said nothing of it, because it wasn’t the point. So does the fact that they came so close mean the work is independent of the application? Or does the fact that they didn’t quite get there mean it’s not? I just wanted to make some charts and now we’re here, in a place that seems grotesque to even consider on the same page. I start to sympathize with the instinct to keep history sequestered in its own discipline, somewhere else, lest I go so far as to conclude _don’t do statistics on people_ — which may be incompatible with responsible governance of a large complex state. Our measures of inequality, of public health, of climate are also statistical. But every attribute of every person you count, for any reason, renders the human being more legible and pliable to machinic powers beyond our control or comprehension. We cannot see that with which our own work will one day be correlated, or the name of the usurper to our flying authorities. Here I have merely placed two histories side by side in chronology as they both rose; as we have seen, that is sufficient for correlation. But I have been selective. In place of Galton’s correspondence with the villain Ploetz, I could have cited his correspondence with the hero Florence Nightengale. Ploetz’s independent conception of eugenics had no especially statistical content, and the error theorists who developed least squares had no eugenic inclinations. They merely cross-bred, delivering the original example that what we find in that coefficient is a very small thing in a very large space. It points to no deep or necessary truth, nor implies any policy. It is just a measure of the angle between two series-vectors; its invention was hardly evil. It has done immeasurable good for doctors, chemists, and agronomists. You have used it. But if it is too easy to make correlation seem to have been correlated with evil, then that is just the problem with correlation.`3a:Tf10e,bibliography = html`

1805	Adrien-Marie Legendre, Nouvelles méthodes pour la détermination des orbites des comètes archive.org Pioneers the method of least squares to balance the errors in astronomical observations; rivals Gauss’s claim to priority.
1809	Carl Friedrich Gauss, Theoria motus corporum coelestium in sectionibus conicis solem ambientium archive.org Gauss’s “Theory of motion of the celestial bodies moving in conic sections around the Sun” includes his treatment of the method of least squares, which he claimed to have been using since before Legendre.
1846	Auguste Bravais, Analyse mathématique sur les probabilités des erreurs de situation d’un point books.google.com Looking at the distribution of observational errors, drew the same ellipse and regression line that Galton would later draw, but ascribed no special meaning to the relationship. Bravais has sometimes been credited with inventing correlation, but he does not seem to have recognized what he had stumbled upon; see Denis (2000) below.
1859	Charles Darwin, On the Origin of Species by Means of Natural Selection, or the Preservation of Favoured Races in the Struggle for Life gutenberg.org
1865	Francis Galton, “Hereditary Talent and Character” (MacMillan’s Magazine) galton.org A study of the family trees of eminent men to show that “talent is transmitted by inheritance in a very remarkable degree”. “Let us, then, give reins to our fancy, and imagine a Utopia — or a Laputa, if you will — in which a system of competitive examination for girls, as well as for youths, had been so developed as to embrace every important quality of mind and body, and where a considerable sum was yearly allotted to the endowment of such marriages as promised to yield children who would grow into eminent servants of the State.… Under these circumstances, if any or all of these marriages should be agreed upon, the Sovereign herself will give away the brides, at a high and solemn festival, six months hence, in Westmister Abbey. We, on our part, are prepared, in each case, to assign 5000£ as a wedding-present, and to defray the cost of maintaining and educating your children, out of the ample funds entrusted to our disposal by the State.” The idea was later further developed, this time with penalties for the unfit, in his unpublished story Kantsaywhere (see below).
1865	Gregor Mendel, “Versuche über Pflanzenhybriden” gutenberg.org; trans. Druery, Bateson and Blumberg “Experiments in Plant Hybridization” mendelweb.org Mendel’s work, establishing the action of dominant and recessive factors through pea plant experiments, languished in obscurity, and he died unknown in 1888. It was unknown to Darwin, who tended to side with Lamarck’s belief that characteristics acquired over a lifetime were inheritable, and to Galton, who doubted that they were. In early 1900, three botanists — the Dutch Hugo de Vries, German Carl Correns, and Austrian Erich von Tschermak — each published their rediscovery of Mendel. This soon settled the debate, demonstrating that inheritable genetic material was fixed at birth and unchanged over a lifetime. Lamarck had offered hope that striving would be passed down biologically; Mendel was taken by eugenicists to mean that it was useless to try to educate people from “poor stock” when the next generation would just have to start over. (Epigenetics has since complicated the biological inheritance issue.)
1869	Francis Galton, “Hereditary Genius” galton.org
1873	Francis Galton, “Hereditary Improvement” (Fraser’s Magazine) galton.org p.124: “I do not for a moment contemplate coercion as to whom any given person should marry; such an idea would be scouted now-a-days almost as much as that of polygamy, or of infanticide. But it is quite conformable to the customs of this century to employ social considerations to effect what is desirable, and their efficacy in this case would be as great as is needful. The great majority are sure to yield to it, and it is a trifling matter, when we look to general results, if a small percentage refuse obedience. I also lay great stress on the encouragement of the gifted caste to marry early, and to live under healthy conditions, and this I consider would be effected in the manner I shall briefly explain.” p.129: “The gifted families would be full of life and hope, and living under more intelligent and favourable sanitary conditions, would multiply rapidly, while the non-gifted would begin to decay out of the land, whenever they were brought face to face in competition with them, just in the same way as inferior races always disappear before superior ones. It is difficult to analyse the steps by which this invariable law has hitherto accomplished itself, and much more difficult is it to guess how it would be accomplished under the conditions here described, but I should expect it would be effected with little severity. I do not see why any insolence of caste should prevent the gifted class, when they had the power, from treating their compatriots with all kindness, so long as they maintained celibacy. But if these continued to procreate children, inferior in moral, intellectual and physical qualities, it is easy to believe the time may come when such persons would be considered as enemies to the State, and to have forfeited all claims to kindness.”
1874	Francis Galton, English Men of Science: their Nature and Nurture galton.org Published the results of his pioneering early psychological questionnaire of 200 members of the Royal Society; popularized the term “nature and nurture”.
1883	Francis Galton, Inquiries into Human Faculty and its Development galton.org Coins “eugenics”, p.24: “I do not propose to enter further into the anthropometric differences of race, for the subject is a very large one, and this book does not profess to go into detail. Its intention is to touch on various topics more or less connected with that of the cultivation of race, or, as we might call it, with ‘eugenic’ ¹ questions, and to present the results of several of my own separate investigations.… ¹ That is, with questions bearing on what is termed in Greek, eugenes, namely, good in stock, hereditarily endowed with noble qualities. This, and the allied words, eugeneia, etc., are equally applicable to men, brutes, and plants. We greatly want a brief word to express the science of improving stock, which is by no means confined to questions of judicious mating, but which, especially in tbe case of man, takes cognisance of all influences that tend in however remote a degree to give to the more suitable races or strains of blood a better chance of prevailing speedily over the less suitable than they otherwise would have had. The word eugenics would sufficiently express the idea; it is at least a neater word and a more generalised one than viriculture, which I once ventured to use.”
1886	Francis Galton, “Regression Towards Mediocrity in Hereditary Stature” ucla.edu In his surveys of both sweet pea seeds and human heights, Galton noticed that exceptional parents tend to beget somewhat less exceptional children; e.g., extremely tall parents tend to beget only moderately tall children — the heights have regressed toward the mean.
1888	Francis Galton, “Co-relations and their Measurement, chiefly from Anthropometric Data” galton.org Extending his work on regression to formalize the existing colloquial term “co-relation”, again based on the heritability of standard deviations in height and other anthropometric measurements.
1889	Francis Galton, Natural Inheritance galton.org Lays out regression in greater detail; develops the quincunx. It was after he had sent off the final page proofs the previous year that it struck Galton to extend regression to correlation. Pearson credits this book with effectively founding the Galton School (Pearson 1914 Vol. 3a p.57).
1890	Francis Galton, “Kinship and Correlation” galton.org Describes how he came to conceive of correlation as Natural Inheritance was going to press. “Few intellectual pleasures are more keen than those enjoyed by a person who, while he is occupied in some special inquiry, suddenly perceives that it admits of a wide generalization, and that his results hold good in previously-unsuspected directions.… Reflection soon made it clear to me that not only were the two new problems identical in principle with the old one of kinship which I had already solved, but that all three of them were no more than special cases of a much more general problem—namely, that of Correlation.”
1890	Francis Galton, “Why do we measure mankind?” galton.org
1896	Karl Pearson, “Mathematical contributions to the theory of evolution.—III. Regression, heredity, and panmixia” royalsocietypublishing.org Pearson’s major mathematical formalization of Galton’s statistical concepts, which were based solely on astute observation, with very little theory. Attributes the conception of correlation to Bravais 1846, a claim conclusively retracted in Pearson 1920.
1901	Karl Pearson, National life from the standpoint of science archive.org “You will see that my view — and I think it may be called the scientific view of a nation — is that of an organized whole, kept up to a high pitch of internal efficiency by insuring that its numbers are substantially recruited from the better stocks, and kept up to a high pitch of external efficiency by contest, chiefly by way of war with inferior races, and with equal races by the struggle for trade-routes and for the sources of raw material and of food supply.”
1901	Francis Galton, “The possible improvement of the human breed under the existing conditions of law and sentiment” galton.org Delivered as the Huxley Lecture; returned to focus on eugenics, now drawing more on regression and correlation. “The natural character and faculties of human beings differ at least as widely as those of the domesticated animals, such as dogs and horses, with whom we are familiar.… Whether it be in character, disposition, energy, intellect, or physical power, we each receive at our birth a definite endowment, allegorised by the parable related in St. Matthew, some receiving many talents, others few; but each person being responsible for the profitable use of that which has been entrusted to him.… A talent is a sum whose exact value few of us care to know, although we all appreciate the inner sense of the beautiful parable. I will, therefore, venture to adapt the phraseology of the allegory to my present purpose by substituting for ‘talent’ the words ‘normal-talent.’ The value of this normal talent in respect to each and any specified quality or faculty is such that one-quarter of the people receive for their respective shares more than one normal-talent over and above the average of all the shares. Our normal-talent is therefore identical with what is technically known as the ‘probable error.’ Therefore the whole of the following table starts into life, evolved from that of the ‘probability integral.’” “The first and pressing point is to thoroughly justify any crusade at all in favour of race improvement. More is wanted in the way of unbiased scientific inquiry along the many roads I have hurried over, to make every stepping-stone safe and secure, and to make it certain that the game is really worth the candle.… The faculties of future generations will necessarily be distributed according to laws of heredity, whose statistical effects are no longer vague, for they are measured and expressed in formulæ. We cannot doubt the existence of a great power ready to hand and capable of being directed with vast benefit as soon as we shall have learnt to understand and to apply it. To no nation is a high human breed more necessary than to our own, for we plant our stock all over the world and lay the foundation of the dispositions and capacities of future millions of the human race.”
1904	C. Spearman, “The Proof and Measurement of Association between Two Things” jstor.org Spearman credits Galton and Pearson in his early psychometric work on IQ and the “g factor”. See also Cosma Shalizi (2007), which in turn points to Clark Glymour (1998). Also Nassim Taleb (2019).
1904	Francis Galton, “Eugenics: its definition, scope and aims” galton.org Francis Galton: “What nature does blindly, slowly and ruthlessly, man may do providently, quickly and kindly.” Karl Pearson: “Are we to make the whole doctrine of descent, of inheritance, and of selection of the fitter, part of our everyday life, of our social customs, and of conduct? It is the question of the study now, but tomorrow it will be the question of the marketplace, of morality, and of politics. If I wanted to know how to put a saddle on a camel's back without chafing him, I should go to Francis Galton; if I wanted to know how to manage the women of a treacherous African tribe, I should go to Francis Galton; if I wanted an instrument for measuring a snail, or an arc of latitude, I should appeal to Francis Galton; if I wanted advice on any mechanical, of any geographical, or any sociological problem, I should consult Francis Galton. In all these matters, and many others, I feel confident he would throw light on my difficulties, and I am firmly convinced that, with his eternal youth, his elasticity of mind, and his keen insight, he can aid us in seeking an answer to one of the most vital of our national problems: How is the next generation of Englishmen to be mentally and physically equal to the past generation which has provided us with the great Victorian statesmen, writers, and men of science—most of whom are now no more—but which has not entirely ceased to be as long as we can see Francis Galton in the flesh?” George Bernard Shaw: “I agree with the paper, and go so far as to say that there is now no reasonable excuse for refusing to face the fact that nothing but a eugenic religion can save our civilization from the fate that has overtaken all previous civilizations.” H. G. Wells: “I must confess that much of Dr. Galton's classical work in this direction seems to me to be premature. I have been impressed by the idea—and even now I remain under the sway of the idea—that our analysis of human faculties is entirely inadequate for the purpose of tracing hereditary influence.… Take health, for example. Are there not types of health? The mating of two quite healthy persons may result in disease.… I am not even satisfied by the suggestion Dr. Galton seems to make that criminals should not breed. I am inclined to believe that a large proportion of our present-day criminals are the brightest and boldest members of families living under impossible conditions, and that in many desirable qualities the average criminal is above the average of the law-abiding poor and probably of the average respectable person.… It is in the sterilization of failures, and not in the selection of successes for breeding, that the possibility of an improvement of the human stock lies.” Benjamin Kidd: “Statistical and actuarial methods alone in the study of individual faculty often carry us to very incomplete conclusions, if not corrected by larger and more scientific conceptions of the social good. I remember our chairman, in his earlier social essays, once depicted an ideally perfect state of society. I have a distinct recollection of my own sense of relief that my birth had occurred in the earlier ages of comparative barbarism. For Mr. Pearson, I think, proposed to give the kind of people who now scribble on our railway carriages no more than a short shrift and the nearest lamp-post. I hope we shall not seriously carry this spirit into eugenics. It might renew, in the name of science, tyrannies that it took long ages of social evolution to emerge from. Judging from what one sometimes reads, many of our ardent reformers would often be willing to put us into lethal chambers, if our minds and bodies did not conform to certain standards. We are apt to forget in these matters that that sense of responsibility to life which distinguishes the higher societies is itself an asset painfully acquired by the race—a social asset of such importance that the more immediate gain aimed at would count by the side of it as no more than dust in the balance.” Alice Vickery: “Mr. Galton told us that a certain number of burgesses’ names had absolutely disappeared; but what about the names of their wives, and how would that consideration affect his conclusion? In the future, the question of population will, I hope, be considered very much from the feminine point of view; and if we wish to produce a well-developed race, we must treat our womankind a little better than we do at present. We must give them something more like the natural position which they should hold in society. Women’s specialized powers must be utilized for the intellectual advancement of the race.”
1907	Francis Galton, “Probability, The Foundation of Eugenics” galton.org Near the end of his life, Galton lays out a history of his contributions to the field. p.98: “Enough has been written elsewhere by myself and others to show that whenever public opinion is strongly roused it will lead to action, however contradictory it may be to previous custom and sentiment. Considering that public opinion is guided by the sense of what best serves the interests of society as a whole, it is reasonable to expect that it will be strongly exerted in favour of Eugenics when a sufficiency of evidence shall have been collected to make the truths on which it rests plain to all. That moment has not yet arrived. Enough is already known to those who have studied the question to leave no doubt in their minds about the general results, but not enough to justify legislation or other action except in extreme cases. Continued studies will be required for some time to come, and the pace must not be hurried. When the desired fulness of information shall have been acquired then, and not till then, will be the fit moment to proclaim a ‘Jehad,’ or Holy War against customs and prejudices that impair the physical and moral qualities of our race.”
1908	Francis Galton, Memories of My Life galton.org
1914	Karl Pearson, The Life, Letters and Labours of Francis Galton galton.org A massive three-volume biography by Pearson in piam memoriam of his mentor and hero. Naturally, it starts with more than fifty pages of ancestry, going back five generations. Re: Kantsaywhere— Vol. 3a, p.411: Pearson writes of Galton, “Thinking over the problem of books that have had lasting influence on mankind his thoughts turned to those ideal polities, Plato’s Republic, More’s Utopia, Harrington’s Oceana, and Butler’s Erewhon. Why should he not exercise a similar influence on generations to come by writing his own Utopia, a story of a land where the nation was eugenically organised? A modern Gulliver should start his travels again and seek a bride in Eugenia. Only a fragment of this Utopia, which was termed ‘Kantsaywhere,’ has reached me, it deals with ‘The Eugenic College of Kantsaywhere.’” He had been writing it for several years; upon his death, his niece bowdlerized it, “because the love-episodes were too absurdly unreal.” People in the land of Kantsaywhere take an examination in eugenics (identified with a fingerprint, which Galton had also pioneered); high marks qualify you for a special marriage ceremony. Vol. 3a, p.416, excerpt from Kantsaywhere manuscript: “I was much questioned about the papers that Mr Allfancy had sent in, as regards my personal knowledge of the authorities for the facts there set forth. They then smilingly gave me a first-class P. G. — Passed in Genetics — degree, and I had to imprint my fingers in their Register, for future identification if necessary. So I returned to my host with one small portion of a load of anxiety taken off my mind. I heard a little now, but must inform myself more particularly hereafter, as to the fate of those who failed to pass. A Bureau was charged with looking after the unclassed parents and their offspring, and much was done to make the lot of the unclassed as pleasant as might be, so long as they propagated no children. If they did do so kindness was changed into sharp severity. Labour Colonies are established where the very inferior are segregated under conditions that are not onerous, except that they must work hard and live in celibacy. It is difficult to describe the indignation and even the horror felt in Kantsaywhere, at acts that may spoil the goodness of their stock, of which they have become extremely proud and jealous. They look confidently forward to a coming time when Kantsaywhere shall have evolved a superior race of men.” Vol. 3b, p.616: Galton writes to Millie (December 28, 1910): “Kantsaywhere must be smothered or be superseded. It has been an amusement and has cleared my thoughts to write it. So now let it go to ‘Wont-say-where.’” Re: Ploetz— Vol. 3b, p.545–6: Ploetz writes to Galton (August 17, 1905): “From your standpoint you perhaps take also some interest in a little book, which I published ten years ago and which I allow myself to send you with the same mail. I started from an English use of the word ‘race’ and tried to investigate the conditions of preserving and developing a race—race-hygiene (‘Rassen-Hygiene’). Afterwards, in the first introducing article of our Archiv, I tried to sharpen the meaning of the word ‘race,’ so as to make it suitable for the theoretical and practical needs of a man, who will seize the real long (beyond the individuals) lasting unities of life, their conditions of preservation and development. I should be very much indebted to you, the senior of the practical application of the principles of evolution on man, if you would in an hour of leisure read my essays and write me your cool judgment. My book is written mostly in a small town, where I practised as physician, absent from a good library and therefore without much knowledge of current literature. That, together with the haste, with which I was compelled by my editor to deliver my paper to the press, may declare many omissions in respect of modern authors.” (According to Lenz’s 1924 paper, Ploetz did not know of Galton when he wrote his book.) Vol. 3b, p.562: Galton writes to Schuster (February 3, 1906): “I have just received an offprint of a German translation of all my Eugenics papers, inserted in that excellent periodical Archiv für Rassen- und Gesellschafts-Biologie of which I know Professor Pearson has a high opinion.” Vol. 3b, p.599: Galton writes to his niece Milly Lethbridge (July 24, 1909): “A Eugenics Review, under the title of ‘Rassen-Hygiene,’ has been started in Munich, by a very capable man, Dr Ploetz, who is the editor of a really solid anthropological periodical. The society that issues the review has five honorary members among whom are Haeckel and Weismann, and I am asked to be its honorary Vice-President, which honour I have gladly accepted. But I must work up my German!” Vol. 3a, p.388: Galton writes to Pearson (August 7, 1909): “For my part, I creak on, not unhappily but little usefully. However, I have small jobs on hand which interest me. One of these I shall want before long to consult you about. It is due to the proposal of Ploetz's society to give some sort of diploma to those who rank eugenically in the uppermost quarter of the population. I have long considered how some such scheme could be practically worked out, and am putting my ideas on paper. Ploetz (I strongly suspect on your initiative) has asked me to accept the Hon. Vice-Presidentship of the Society. They have only five Hon. Members, among whom are Haeckel and Weismann. It is a great honour.” Vol. 3a, p.389, Pearson replies to Galton (August 8, 1909): “I think Ploetz is a sound man, and keen on Eugenics. I should not, however, allow his ‘International’ Society to absorb yours as a branch, which he may suggest. I fancy he is working in the first place to accumulate material with regard to families.” Vol. 3b, p.606, Galton writes to Milly (July 2, 1910): “The novel [Kantsaywhere] has had a long set-back, having found the plot not to be as useful a one as was wanted. I have at last re-cast it in a better form, but written nothing yet. Some of what was written will still serve. I will keep you au courant. I expect Miss Elderton every moment for the week-end and have asked a few friends for a Eugenics tea to-morrow. Ploetz, the German author of the paper you kindly looked at for me, and who is in London now about a Race Congress, will come. It is long since I last ventured out of doors, but a convenient alteration has been made in my balcony fittings, which will make going in and out of it still easier.” Vol. 3a, p.429, Galton to Pearson (July 4, 1910): “Yesterday I got together to tea, Miss Elderton, Crackanthorpe and Ploetz.” Vol. 3b, p.606, Galton writes to Milly (July 11, 1910): “Ploetz proved to be a pleasant acquaintance and full of ‘go.’ A fair-haired South German. My novel gets on but is quite re-written. I now get up an hour before breakfast and lie down for a bit after. The plan seems to suit well.” Re: Quetelet and other precedents— Vol. 2, p.12: “There are scarcely two dozen books in Galton’s library as we now have it, which we can assert he must have purchased to forward his work. There are masses of measurements and observations of Galton’s own, but unlike Darwin he did not start by analysing published material. He collected afresh either directly or through others and formed his conclusions de novo: I do not think he ever studied Laplace or Poisson; I am confident that he had never considered the original papers of Gauss; even while Galton’s work seems to flow naturally from that of Quetelet, I am very doubtful how far he owed much to a close reading of the great Belgian statistician. He formed no collection of his books, and the few references to Quetelet in Galton’s writings are such as might easily arise from indirect sources. Galton took up his problems one after another and worked at them largely disregarding their past history, when indeed they had one. This is only possible in a man of great insight and brimming over with suggestive ideas and novel processes. But the method has some drawbacks, when adopted by lesser men, and even, as in his above-cited account of his observations on the Corona, we may find it open to criticism in Galton himself. But unless my readers grasp this characteristic of Galton’s nature they will fail to understand how, with all his travel and the social and executive calls on his time, he was yet able to accomplish so much. It is dangerous advice to give to every scientific worker, but it is the only useful advice to give to a young man of genius: Find a little trodden path, and explore it rapidly alone, without regard to the work of others; many precious hours will be wasted if yon follow up the spoor of each one who has passed athwart your path before. Galton took some time to find his individual track, but having found it, he went ahead without much regard to forerunners or even to those working on parallel lines. It was this individuality of method which impressed itself on his environment and rendered him so independent of the usual appurtenances of the scholar. He thought and he worked with the simplest of tools, and these mostly of his own making.”
1918	R. A. Fisher, “The Correlation between Relatives on the Supposition of Mendelian Inheritance” adelaide.edu.au Instrumental synthesis of the statistical and Mendelian traditions
1920	Karl Pearson, “Notes on the History of Correlation” jstor.org Extensive, meticulous, lucid, dense technical account of the mathematical development from Laplace (1783) to Gauss (1823, 1826) to Bravais (1846) to Galton (1877, 1885, 1886, 1888, 1889) and beyond to Weldon (1890, 1892, 1893) to Edgeworth (1892) to Pearson himself. Perhaps my favorite treatment of the subject, and the last I read. “Many years ago I called the Laplace-Gaussian curve the normal curve, which name, while it avoids an international question of priority, has the disadvantage of leading people to believe that all other distributions of frequency are in one sense or another ‘abnormal’. That belief is, of course, not justifiable. It has led many writers to try and force all frequency by aid of one or another process of distortion into a ‘normal’ curve.” Later: “Are we not making a fetish of the method of least squares as others made a fetish of the normal distribution?” In conclusion: “It is a long step from Francis Galton’s ‘reversion’ in sweet pea seeds to the full theory of multiple correlation, which we now know to be identical with the spherical trigonometry of high-dimensioned space, the total correlation coefficients being the cosines of the edges of the polyhedra and the partial correlation coefficients the cosines of the polyhedral angles. But to find the correlation of the health of a child with the number of people per room while you render neutral its age, the health of its parents, the wages of its father, and the habits of its mother, is no less vital a problem than Galton's correlation of character in parent and offspring. It requires indeed more mathematics, but the mathematics are not there for the joy of the analyst but because they are essential to the solution. It is the transition from the mill as pestle and mortar to the mill with steam driven grain crushing steel rollers. But the inventor of milling was the person who bruised grain between two stones, and Galton was the man who discovered the highway across this new country with what he aptly terms ‘its easy descents to different goals.’” Also, he tried to read Edgeworth, couldn’t understand him, and eventually determined he was just wrong — which makes me feel slightly better about when I flunked an Edgeworth reading I chose in college (although my bigger problem was not understanding any of that calculus of variations class at all): “On re-examining his memoir 25 years later I think he harnessed imperfect mathematical analysis to a jolting car and drove it into an Irish bog on his road, and that it was doubtful analysis not errors of printing which led to his obscure conclusions.”
1922	G. K. Chesterton, Eugenics and Other Evils gutenberg.org “Though most of the conclusions, especially towards the end, are conceived with reference to recent events, the actual bulk of preliminary notes about the science of Eugenics were written before the war… And then the hour came when I felt, not without relief, that I might well fling all my notes into the fire… Either Prussianism would win and the protest would be hopeless, or Prussianism would lose and the protest would be needless… So I thought all I had written irrelevant, and put it out of my mind. I am greatly grieved to say that it is not irrelevant.”
1924	Fritz Lenz, “Eugenics in Germany” (Journal of Heredity) oup.com Contradicting the Galton Institute’s own account that the German Society for Racial Hygiene was founded by followers of Galton, Lenz (who was there) reports that, when founder Alfred Ploetz published his 1895 work coining “Rassenhygiene”, he knew the work of Darwin, Wallace, and Haeckel, but not Galton. Lenz translates “Rassenhygiene” not as “racial hygiene” but as simply “eugenics”: “The word means exactly the same”. Ploetz followed August Weismann, who, unlike Darwin, rejected Lamarckian inheritance of acquired characteristics, before the rediscovery of Mendel. Wilhelm Schallmayer, the next most important founder of German eugenics, accepted Lamarckism and also did not know of Galton. Lenz notes that during the first World War, “the eugenics movement in Germany stood entirely still”, but fears of decline reinvigorated it afterward. Lenz called Spengler’s 1918 The Downfall of the Occident “unsound and unscientific” and “so lacking in eugenic understanding”, but: “Nevertheless it may be hoped that even Spenglerism can be turned to the profit of eugenics: the question of the downfall of western civilization has been made timely, and the ground has perhaps thus been prepared for a scientific investigation of the causes and prevention of such a downfall.” p.229: “There is a school of eugenists in Germany that becomes highly indignant every time the differences between the great races of the world are mentioned in connection with eugenics. These eugenists, mostly either Jews or Jewish partisans, therefore oppose the use of the word ‘race hygiene’; many of them, indeed, are extremely hostile to the mere mention of the word ‘race’ in any sense. Thus it has come about that the Jewish or Jew-influenced eugenists of Germany speak of themselves as eugenists or eugenetists; the German eugenists on the other hand call themselves ‘race hygienists’—a fact that Anglo-Saxon eugenists may be interested to know. A large part of the German literature on heredity and eugenics is written by Jews, and thus it happens that Lamarckistic notions of an ‘inheritance of acquired characters’ are often spread in the press, because most Jews have an emotional bias in this direction, being thus able to believe that racial differences are unimportant, since they can be obliterated by the action of one's surroundings. Hand in hand with this attitude one usually finds a disposition to deny the influence of selection.” (Lenz joined the Nazi party in 1937 and served on its Expert Committee on Questions of Population and Racial Policy with, among others, Heinrich Himmler.) p.231: “On the whole, one must say that, on the legislative side, everything yet remains to be done. The Germans are more disposed toward scientific investigation than toward practical statesmanship. It is, then, perhaps no loss that the Anglo-Saxons have an advantage in the field of eugenic legislation. Too hasty legislation in this matter may also provoke disastrous reactions. The main thing is that the eugenic education of the nation goes ahead; and it is going. Once public sentiment is won over to eugenics, the legislation will come of itself.”
1927	Buck v. Bell, 274 U.S. 200 justia.com The Supreme Court ruled 8-1 that compulsory sterilization did not violate the Due Process clause of the 14th Amendment. Bell was a doctor at a psychiatric hospital who had ordered the sterilization of Carrie Buck, an 18-year-old accused of feeble-mindedness and promiscuity who had given birth to an illegitimate child. Oliver Wendell Holmes, Jr. delivered the majority opinion: “We have seen more than once that the public welfare may call upon the best citizens for their lives. It would be strange if it could not call upon those who already sap the strength of the State for these lesser sacrifices, often not felt to be such by those concerned, in order to prevent our being swamped with incompetence. It is better for all the world if, instead of waiting to execute degenerate offspring for crime or to let them starve for their imbecility, society can prevent those who are manifestly unfit from continuing their kind. The principle that sustains compulsory vaccination is broad enough to cover cutting the Fallopian tubes. Three generations of imbeciles are enough.” Pierce Butler was the lone dissenter; Holmes blamed his Catholicism. It was later determined that Buck’s pregnancy had been due to rape by her adoptive cousin.
1927	H. L. Mencken, “On Eugenics” (The Chicago Tribune) mencken.org “Nietzsche, in fact, preached a gospel that, to most human beings, is unbearable, and it will probably remain unbearable for centuries to come. Its adoption by Dr. Coolidge, by and with the advice and consent of the senate, would plunge this republic into dreadful woe. And Nietzsche himself was a chronic invalid who died insane—the sort of wreck who, had he lived into our time, would have been a customer of chiropractors. Worse, he suffered from a malady of a scandalous nature, and of evil effects upon the sufferer’s offspring. Was it good or bad luck for the world, eugenically speaking, that he was a bachelor?”
1939	Simone Weil, “L’Iliade ou le poème de la force” teuwissen.ch; trans. Mary McCarthy, “The Iliad, or The Poem of Force” url.edu.gt Weil wrote “La force, c’est ce qui fait de quiconque lui est soumis une chose”; the English quoted above, with its algebraic ${tex`x`}, is McCarthy’s.
1946	Max Weinreich, Hitler’s Professors: The Part of Scholarship in Germany’s Crimes Against the Jewish People p.33: “Rüdin’s name appears together with two others on a rather early commentary on German eugenic legislation intended primarily for physicians… The leading ideas of the authors were expressed in the introduction to the volume: ‘Our whole cultural life for decades has been more or less under the influence of biological thinking, as it was begun particularly around the middle of the last century, by the teachings of Darwin, Mendel, and Galton and afterwards has been advanced by the studies of Ploetz, Schallmayer, Correns, de Vries, Tschermak, Baur, Rüdin, Fischer, Lenz, and others. Though it took decades before the courage was found, on the basis of the initial findings of the natural sciences, to carry on a systematic study of heredity, the progress of the teaching and its application to man could not be delayed any more. It was recognized that the natural laws discovered for plants and animals ought also to be valid for man, and this could fully and completely be confirmed during the last three decades both through family research and through the study of bastards and twins.’”
1981	Bentley Glass, “A Hidden Chapter of German Eugenics between the Two World Wars” jstor.org “Why did the esteemed and scrupulous plant geneticist Erwin Baur collaborate with Eugen Fischer and Fritz Lenz to write in 1923 the influential book, Menschliche Erblichkeitslehre und Rassenhygiene (Eng. trans. Human Heredity), which became the cornerstone of Nazi eugenics?” Glass writes that, in 1921, Baur was extremely concerned about miscegenation between Germans and Black foreign troops occupying the Rhineland.
1981	Heinrich Böll, “At the Bridge”, collected in The Stories of Heinrich Böll; trans. Leila Vennewitz tandfonline.com
1986	Stephen M. Stigler, The History of Statistics: The Measurement of Uncertainty before 1900
1989	Zygmunt Bauman, Modernity and the Holocaust p.7: “The unspoken terror permeating our collective memory of the Holocaust (and more than contingently related to the overwhelming desire not to look the memory in its face) is the gnawing suspicion that the Holocaust could be more than an aberration, more than a deviation from an otherwise straight path of progress, more than a cancerous growth on the otherwise healthy body of the civilized society; that, in short, the Holocaust was not an antithesis of modern civilization and everything (or so we like to think) it stands for. We suspect (even if we refuse to admit it) that the Holocaust could merely have uncovered another face of the same modern society whose other, more familiar, face we so admire. And that the two faces are perfectly comfortably attached to the same body. What we perhaps fear most, is that each of the two faces can no more exist without the other than can the two sides of a coin.” p.16: “The most shattering of lessons deriving from the analysis of the ‘twisted road to Auschwitz’ is that – in the last resort – the choice of physical extermination as the right means to the task of Entfernung was a product of routine bureaucratic procedures: means–ends calculus, budget balancing, universal rule application. To make the point sharper still – the choice was an effect of the earnest effort to find rational solutions to successive ‘problems’, as they arose in the changing circumstances. It was also affected by the widely described bureaucratic tendency to goal-displacement – an affliction as normal in all bureaucracies as their routines. The very presence of functionaries charged with their specific tasks led to further initiatives and a continuous expansion of original purposes. Once again, expertise demonstrated its self-propelling capacity, its proclivity to expand and enrich the target which supplied its raison d’etre.” p.68: “First, with the Enlightenment came the enthronement of the new deity, that of Nature, together with the legitimation of science as its only orthodox cult, and of scientists as its prophets and priests. Everything, in principle, had been opened to objective inquiry; everything could, in principle, be known – reliably and truly. Truth, goodness and beauty, that which is and that which ought to be, had all become legitimate objects of systematic, precise observation. In turn, they could legitimize themselves only through objective knowledge which would result from such observation. In George L. Mosse’s summary of his most convincingly documented history of racism, ‘it is impossible to separate the inquiries of the Enlightenment philosophies into nature from their examination of morality and human character… [From] the outset… natural science and the moral and aesthetic ideals of the ancient joined hands.’ In the form in which it was moulded by the Enlightenment, scientific activity was marked by an ‘attempt to determine man’s exact place in nature through observation, measurements, and comparisons between groups of men and animals’ and ‘belief in the unity of body and mind’. The latter ‘was supposed to express itself in a tangible, physical way, which could be measured and observed’. Phrenology (the art of reading the character from the measurements of the skull) and physiognomy (the art of reading the character from facial features) captured most fully the confidence, strategy and ambition of the new scientific age. Human temperament, character, intelligence, aesthetic talents, even political inclinations, were seen as determined by Nature; in what way exactly, one could find out through diligent observation and comparison of the visible, material ‘substratum’ of even the most elusive or concealed spiritual attributes. Material sources of sensual impressions were so many clues to Nature’s secrets; signs to be read, records written down in a code which science must crack. What was left to racism was merely to postulate a systematic, and genetically reproduced distribution of such material attributes of human organism as bore responsibility for characterological, moral, aesthetic or political traits. Even this job, however, had already been done for them by respectable and justly respected pioneers of science, seldom if ever listed among the luminaries of racism. Observing sine ira et studio the reality as they found it, they could hardly miss the tangible, material, indubitably ‘objective’ superiority that the West enjoyed over the rest of the inhabited world.”
1989	Stephen M. Stigler, “Francis Galton’s Account of the Invention of Correlation” projecteuclid.org Pearson believed that Galton was referring to the “eureka” moment for correlation when he wrote (1908): “It was in the grounds of Naworth Castle, where an invitation had been given to ramble freely. A temporary shower drove me to seek refuge in a reddish recess in the rock by the side of the pathway. There the idea flashed across me, and I forgot everything else for a moment in my great delight.” But Stigler believes that Galton was there talking about having the idea of a statistical scale (like z-score); Pearson seems to have overlooked Galton’s 1890 “Kinship and Correlation”, where he clearly lays out that the inspiration came while looking over the proofs for Natural Inheritance.
1991	Michael Burleigh and Wolfgang Wippermann, The Racial State: Germany 1933–1945 p.28: “Darwin was a reclusive Victorian gentleman scholar of a liberal, progressive cast of mind. He was opposed to slavery, and strongly supported ideas of human equality by avoiding references to ‘lower’ or ‘higher’ races. He was concerned about poverty and established Friendly Societies in Kent. The Origins did not contain racial theories, and was almost exclusively concerned with plants and animals. Nonetheless, Darwin, rather than Gobineau, was the involuntary progenitor of racist ideology, for he was responsible for the theory of natural selection as the mechanism of evolution. Selection was to beome central to all subsequent racist discourse.” p.29: “Francis Galton (1922–1911) took the principle of selection further, in the interests of improving the biological health of the human race. Healthy parents, by whom he meant members of the middle classes and the learned professions, should be encouraged to marry early and have as many children as possible. These should be issued with certificates of hereditary health. By contrast, those persons who failed this ‘Passed in Genetics’ test were to be encouraged to emigrate to the land of ‘Cantsayanywhere’. Man, in other words, was to take control of his own evolutionary process. Galton was the founder of hereditary health care, for which in 1883 he coined the term ‘Eugenics’, a programme for improving the human race by genetic means. The prescriptive measures were not confined to the question of ‘judicious mating’, but encompassed education, public health, and welfare. Darwin and Galton’s ideas were gradually diffused throughout Europe and North America, where through the mediation of Herbert Spencer the notion of the ‘survival of the fittest’ was used to legitimise laissez-faire capitalism.” (I think Burleigh and Wippermann are a little bit off: it was ‘Kantsaywhere’, not ‘Cantsayanywhere’, and it was the land they were exiled from, not emigrated to. It is also worth noting that it was a disavowed and half-destroyed manuscript for a novel.) p.32: “Put simply, these discoveries ruled out the prospect of improving the mental or physical abilities of successive generations through education or sport, while emphasizing the pre-determinedness of, for example, criminality or alcoholism.” p.36: “By no means all of the racial hygienicists and eugenicists were either politically conservative or anti-Semites. Responsibility for fusing racial-hygienic and Social Darwinist ideas with anti-Semitism may be attributed to the (elective German) Englishman Houston Stewart Chamberlain (1855–1927).” p. 37: “It is not certain which racialist works Hitler actually read. There is no ‘man who gave Hitler his ideas’ in a simplistic teleological sense. However, it is certain that Hitler knew the most important racial-anthropological, racial-hygienic, and racial anti-Semitic theories, and in Mein Kampf turned them into a comprehensive, self-contained, if totally insane, racial-political programme.
1998	James C. Scott, Seeing Like a State Pat read the bit that says “legible” above and said “Seeing Like A State is pretty much required to be in the bibliography if you leave this sentence in.” He’s right of course, though I haven’t actually read it, only stared at the spine on my bookshelf for years.
2000	Daniel J. Denis, “The Origins of Correlation and Regression: Francis Galton or Auguste Bravais and the Error Theorists?” york.ac.uk Recounts the pre-Galton history of those who very nearly discovered correlation and regression, but ultimately didn’t. Denis takes the math and history equally seriously, and views them as entangled: “Perhaps there is no better example in the history of statistics than the one before us to show that statistical techniques do not simply arise out of mathematical manipulation, but are often preceded by a social-political cause or other equivalent goal. Hence, in this context, the statistical technique is but a ‘weapon’ in the pocket of artillery of the proponent. Since Bravais had no particular reason for finding the technique of correlation, he did not. Conversely, since Galton's program of eugenics and theory of inheritance relied heavily on him finding a ‘mechanism’ of inheritance, he had all the reason and motivation for discovering the technique of regression and correlation, and so he did.”
2005	Warren Gilchrist, “Galton misrepresented” (Significance) wiley.com Highlights that Galton’s regression was based on the median, not the mean, and on the interquartile range divided by two (which he called “probable deviation”), not the standard deviation. This approach has some useful properties; e.g., the median of a function equals the function of a median. Statisticians did not seriously consider median regressions again until a hundred years after Galton.
2006	Ian Hacking, “Making Up People” lrb.co.uk, mirrored at generation-online.org “I have long been interested in classifications of people, in how they affect the people classified, and how the effects on the people in turn change the classifications. We think of many kinds of people as objects of scientific inquiry. Sometimes to control them, as prostitutes, sometimes to help them, as potential suicides. Sometimes to organise and help, but at the same time keep ourselves safe, as the poor or the homeless. Sometimes to change them for their own good and the good of the public, as the obese. Sometimes just to admire, to understand, to encourage and perhaps even to emulate, as (sometimes) geniuses. We think of these kinds of people as definite classes defined by definite properties. As we get to know more about these properties, we will be able to control, help, change, or emulate them better. But it’s not quite like that. They are moving targets because our investigations interact with them, and change them. And since they are changed, they are not quite the same kind of people as before. The target has moved. I call this the ‘looping effect’. Sometimes, our sciences create kinds of people that in a certain sense did not exist before. I call this ‘making up people’.”
2012	Nick Land, “The Dark Enlightenment” Land cites Galton as a hero of neoreaction (1, 2). I think he thinks that to scorn what descriptive statistics tell us about our nature and nurture is to court the retribution of the goddess Nemesis for our hubris, and that concern is quaint in the face of a genetically engineered and bionically unfathomable future.
2014	Michael Brendan Dougherty, “What critics of The Bell Curve get wrong about racial equality” theweek.com “It is a categorical error to search out some material fact in the universe that is consonant with our egalitarian notions. That is just too shaky a foundation. The conviction that all men are possessed of equal dignity, and therefore deserve equality before the law, must be a political — or even theological — assertion against any fact of history or science that would tempt us to think otherwise. Political equality isn’t something we discover in nature; it's something we must create.”
2018	Judea Pearl, The Book of Why Traces how Galton and Pearson sought to explain the world but ended up with models that showed only relations stripped of causality. p.68: “In 1877, Galton was in pursuit of a causal explanation and thought that regression to the mean was a causal process, like a law of physics. He was mistaken, but he was far from alone. Many people continue to make the same mistake to this day.” Galton initially thought regression was due to the mechanics of heredity, and was later surprised to realize that the parents of very tall children also regressed toward the mean. p.73: “The first sacrifice on the altar of correlation was Galton’s elaborate machinery to explain the stability of the population’s genetic endowment. The quincunx simulated the creation of variations in height and their transmission from one generation to the next. But Galton had to invent the inclined chutes in the quincunx specifically to rein in the ever-growing diversity in the population. Having failed to find a satisfactory biological mechanism to account for this restoring force, Galton simply abandoned the effort after eight years and turned his attention to the siren song of correlation. Historian Stephen Stigler, who has written extensively about Galton, noticed this sudden shift in Galton’s aims and aspirations: ‘What was silently missing was Darwin, the chutes, and all the “survival of the fittest.”… In supreme irony, what had started out as an attempt to mathematize the framework of the Origin of Species ended with the essence of that great work being discarded as unnecessary!’” p.77: “It remained to Galton’s disciple, Karl Pearson, to complete the task of expunging causation from statistics. Yet even he was not entirely successful. Reading Galton’s Natural Inheritance was one of the defining moments of Pearson’s life: ‘I felt like a buccaneer of Drake’s days—one of the order of men “not quite pirates, but with decidedly piratical tendencies,” as the dictionary has it!’ he wrote in 1934. ‘I interpreted… Galton to mean that there was a category broader than causation, namely correlation, of which causation was only the limit, and that this new conception of correlation brought psychology, anthropology, medicine and sociology in large part into the field of mathematical treatment. It was Galton who first freed me from the prejudice that sound mathematics could only be applied to natural phenomena under the category of causation.’” p.82: “Looking at the same example through the lens of causality, we can only say, What a missed opportunity!”

${md`See also the Wikipedia entries for [Galton](https://en.wikipedia.org/wiki/Sir_Francis_Galton), [Pearson](https://en.wikipedia.org/wiki/Karl_Pearson), [autocorrelation](https://en.wikipedia.org/wiki/Autocorrelation), [correlation](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient), and the [history of eugenics](https://en.wikipedia.org/wiki/History_of_eugenics); the comprehensive archives at [galton.org](http://galton.org/bibliography/index.html); and the [Galton Institute biography](http://www.galtoninstitute.org.uk/sir-francis-galton/early-life/). I’m sorry I don’t yet have an inline citation format I’m happy with.`}

`3b:T1471,html`

Acknowledgements

${md`I have reached the point where the original application of autocorrelation here feels obvious, which is why it’s so important to remember that I had to ask Sam where to even begin. And I work pretty close to this stuff! The [Wikipedia article for autocorrelation](https://en.wikipedia.org/wiki/Autocorrelation), to pick just one, was [45.5% written](https://xtools.wmflabs.org/articleinfo/en.wikipedia.org/Autocorrelation#authorship) by users Dicklyon, Manoguru, Rgclegg, Melcombe, Bender235, Fvultier, Omegatron, 7804j, WavePart, and Loraof, and 54.5% by 168 others. I’m especially indebted to [this illustration](https://commons.wikimedia.org/wiki/File:Comparison_convolution_correlation.svg) by user [Cmglee](https://en.wikipedia.org/wiki/User:Cmglee), which inspired the topmost illustration here. As with so many Wikipedians, [their contributions](https://commons.wikimedia.org/wiki/User:Cmglee/svg), several hundred sophisticated illustrations with unfathomable reach, place Cmglee and their ilk among the great unsung educators of our time. This [Alan Zucconi article](https://www.alanzucconi.com/2016/06/06/autocorrelation-function/) helped by connecting an explanation and equations with code samples. Thanks also to many StackExchange posts, including [GeoMatt22’s answer to Jaken Herman](https://stats.stackexchange.com/questions/235673/is-there-any-relationship-among-cosine-similarity-pearson-correlation-and-z-sc/235676#235676), [Jeromy Anglim’s answer to Neil McGuigan](https://stats.stackexchange.com/questions/2125/whats-the-difference-between-correlation-and-simple-linear-regression/2128#2128), [gung’s answer to user9097](https://stats.stackexchange.com/questions/22718/what-is-the-difference-between-linear-regression-on-y-with-x-and-x-with-y/22721#22721), [kennytm’s answer to Dan](https://stackoverflow.com/questions/3425439/why-does-corrcoef-return-a-matrix/3425548#3425548), [Sebastien’s answer to Christian Varga](https://stackoverflow.com/questions/11303740/svg-linear-gradient-doesnt-work-in-safari/26220870#26220870), and [various answers to mister.nobody.nz](https://stackoverflow.com/questions/13187778/convert-pandas-dataframe-to-numpy-array/); on Reddit, [various answers to thunderking500](https://www.reddit.com/r/AskStatistics/comments/8z5a9j/is_it_true_that_modern_statistics_was_created_to/); and, on Quora, [Martin O’Shea](https://www.quora.com/Why-is-it-called-product-moment-correlation-coefficient). As I was wrapping up this post I came across [Quentin André’s illustration of linear regression and PCA](https://twitter.com/andre_quentin/status/1087358882551394304), which I am in awe of. Nassim Taleb’s been [talking a lot lately](https://twitter.com/search?q=from%3Anntaleb%20correlation&src=typed_query) about [various problems with correlation](https://medium.com/incerto/iq-is-largely-a-pseudoscientific-swindle-f131c101ba39), which none of this even touches on, but for which I am also grateful. Joshua Horowitz helped me understand what’s going on in the Cmglee illustration, Josh Shapiro kindly sat with me and corrected my algebra, Nicky Case encouraged me early on, Colin Carroll explained cosine similarity, and Andrew Mauboussin kept prodding me to finish it. Patrick O’Neill very helpfully pressed on some very weak and unsupported pieces — some of which buckled, some of which were successfully reinforced, and some of which persist precariously. The Heinrich Böll story is via Katherine Ye via Heike Geissler’s _Seasonal Associate_, and “Making Up People” via Angela Zhou via Katherine; thanks to them both for thoughtful reflections. Adam Pearce very valiantly attempted to rescue me from the obtuse and contrived barycenter metaphor. Rob Ochshorn offered more support than I was even in a state of mind to be able to accept. This post was finished at the Recurse Center, for which I’m very grateful. And for Samantha, Nora, Leo, Olivia, Linda, Allison, Richard, Joe, Blacki, Ingrid, Nicole, Michael and everyone who has listened. At one point, trying to think through the inheritance of guilt, I asked my friend “does god punish sons for the sins of the father or not? so confusing!”, with two verses: >Deuteronomy 24:16: “Fathers shall not be put to death because of their children, nor shall children be put to death because of their fathers. Each one shall be put to death for his own sin.” >Exodus 34:7: “Keeping steadfast love for thousands, forgiving iniquity and transgression and sin, but who will by no means clear the guilty, visiting the iniquity of the fathers on the children and the children's children, to the third and the fourth generation.” He replied: “First is about men, second about God. God does, men shouldn’t. And we should remember how God does: not as the sons being guilty, but as suffering the effects of the evil done by the father.” --- May 24, 2019: Thanks to Stephen Stigler for correcting my spelling of Cauchy-Schwarz. June 23, 2022: More on “Whether anyone is punished for another's sin?” from [Aquinas](https://www.ccel.org/ccel/aquinas/summa.FS_Q87_A8.html). `}

`3c:T526,// To add an example dataset to the notebook, add an object to this array with `name` and `data`. The `src` property is optional and is only used for the attribution link; it’s not actually fetched. The `data` property should be an array of numbers; they have been split into separate variables here to keep this compact. series = [ { name: 'S&P 500', data: spx, src: 'https://finance.yahoo.com/quote/%5EGSPC/history?p=%5EGSPC' }, { name: 'S&P 500, daily change', data: spxChg, src: 'https://finance.yahoo.com/quote/%5EGSPC/history?p=%5EGSPC' }, { name: 'Amazon revenue', data: amznRev, src: 'https://ycharts.com/companies/AMZN/revenues_annual' }, { name: 'Amazon profit', data: amznProfit, src: 'https://ycharts.com/companies/AMZN/net_income_annual' }, { name: 'AMZN trading volume', data: amznVol, src: 'https://finance.yahoo.com/quote/AMZN/history?p=AMZN' }, { name: 'U.S. daily births', data: births, src: 'https://github.com/fivethirtyeight/data/tree/master/births' }, { name: 'Mauna Loa atmospheric CO₂ parts per million', data: carbon, src: 'ftp://aftp.cmdl.noaa.gov/products/trends/co2/co2_mm_mlo.txt' }, { name: 'Your data', data: userdata }, ].filter(d => d.data.length)3d:T9cb,spx = [2389,2384,2388,2391,2388,2390,2399,2399,2397,2400,2394,2391,2402,2401,2357,2366,2382,2394,2398,2404,2415,2416,2413,2412,2430,2439,2436,2429,2433,2434,2432,2429,2440,2438,2432,2433,2453,2437,2436,2435,2438,2439,2419,2441,2420,2423,2429,2433,2410,2425,2427,2426,2443,2448,2459,2459,2461,2474,2473,2473,2470,2477,2478,2475,2472,2470,2476,2478,2472,2477,2481,2475,2474,2438,2441,2466,2465,2468,2430,2426,2428,2453,2444,2439,2443,2444,2446,2458,2472,2477,2458,2466,2465,2461,2488,2496,2498,2496,2500,2504,2507,2508,2501,2502,2497,2497,2507,2510,2519,2529,2535,2538,2552,2549,2545,2551,2555,2551,2553,2558,2559,2561,2562,2575,2565,2569,2557,2560,2581,2573,2575,2579,2580,2588,2591,2591,2594,2585,2582,2585,2579,2565,2586,2579,2582,2599,2597,2602,2601,2627,2626,2648,2642,2639,2630,2629,2637,2652,2660,2664,2663,2652,2676,2690,2681,2679,2685,2683,2681,2683,2688,2674,2696,2713,2724,2743,2748,2751,2748,2768,2786,2776,2803,2798,2810,2833,2839,2838,2839,2873,2854,2822,2824,2822,2762,2649,2695,2682,2581,2620,2656,2663,2699,2731,2732,2716,2701,2704,2747,2780,2744,2714,2678,2691,2721,2728,2727,2739,2787,2783,2765,2749,2747,2752,2713,2717,2712,2644,2588,2659,2613,2605,2641,2582,2614,2645,2663,2604,2613,2657,2642,2664,2656,2678,2706,2709,2693,2670,2670,2635,2639,2667,2670,2648,2655,2636,2630,2663,2673,2672,2698,2723,2728,2730,2711,2722,2720,2713,2733,2724,2733,2728,2721,2690,2724,2705,2735,2747,2749,2772,2770,2779,2782,2787,2776,2782,2780,2774,2763,2767,2750,2755,2717,2723,2700,2716,2718,2727,2713,2737,2760,2784,2794,2774,2798,2801,2798,2810,2816,2804,2802,2807,2820,2846,2837,2819,2803,2816,2813,2827,2840,2850,2858,2858,2854,2833,2822,2840,2818,2841,2850,2857,2863,2862,2857,2875,2897,2898,2914,2901,2902,2897,2889,2878,2872,2877,2888,2889,2904,2905,2889,2904,2908,2931,2930,2919,2916,2906,2914,2914,2925,2923,2926,2902,2886,2884,2880,2786,2728,2767,2751,2810,2809,2769,2768,2756,2741,2656,2706,2659,2641,2683,2712,2740,2723,2738,2755,2814,2807,2781,2726,2722,2702,2730,2736,2691,2642,2650,2633,2673,2682,2744,2738,2760,2790,2700,2696,2633,2638,2637,2651,2651,2600,2546,2546,2507,2467,2417,2351,2468,2489,2486,2507,2510,2448,2532,2550,2574,2585,2597,2596,2583,2610,2616,2636,2671,2633,2639,2642,2665,2644,2640,2681,2704,2707,2725,2738,2732,2706,2708,2710,2745,2753,2746,2776,2780,2785,2775,2793,2796,2794,2792,2784,2804,2793,2790,2771,2749,2743,2783,2792,2811,2808,2822,2833,2833,2824,2855,2801,2798,2818,2805,2815,2834,2867,2867,2873,2879,2893,2896,2878,2888,2888,2907,2906,2907,2900,2905,2908,2934]3e:T1276,spxChg = [0.000553,-0.001913,0.001732,0.001189,-0.001271,0.000582,0.004089,0.000037,-0.001025,0.001131,-0.002163,-0.001478,0.004777,-0.000687,-0.018178,0.003687,0.006767,0.005160,0.001838,0.002489,0.004442,0.000311,-0.001205,-0.000460,0.007571,0.003708,-0.001218,-0.002779,0.001568,0.000267,-0.000830,-0.000979,0.004512,-0.000996,-0.002240,0.000284,0.008347,-0.006697,-0.000583,-0.000456,0.001561,0.000316,-0.008073,0.008808,-0.008600,0.001533,0.002311,0.001453,-0.009369,0.006403,0.000928,-0.000783,0.007306,0.001875,0.004674,-0.000053,0.000598,0.005373,-0.000154,-0.000368,-0.001064,0.002923,0.000283,-0.000973,-0.001341,-0.000728,0.002449,0.000493,-0.002184,0.001889,0.001647,-0.002414,-0.000364,-0.014474,0.001276,0.010044,-0.000499,0.001420,-0.015437,-0.001835,0.001163,0.009941,-0.003454,-0.002074,0.001673,0.000487,0.000843,0.004615,0.005721,0.001983,-0.007551,0.003129,-0.000178,-0.001489,0.010839,0.003364,0.000757,-0.001101,0.001847,0.001456,0.001110,0.000634,-0.003046,0.000648,-0.002222,0.000072,0.004085,0.001205,0.003705,0.003874,0.002159,0.001247,0.005647,-0.001074,-0.001804,0.002322,0.001804,-0.001687,0.000878,0.001751,0.000673,0.000742,0.000328,0.005117,-0.003972,0.001618,-0.004663,0.001271,0.008073,-0.003192,0.000944,0.001592,0.000190,0.003097,0.001271,-0.000189,0.001444,-0.003762,-0.000898,0.000984,-0.002310,-0.005526,0.008196,-0.002626,0.001276,0.006541,-0.000750,0.002056,-0.000384,0.009849,-0.000369,0.008191,-0.002025,-0.001052,-0.003739,-0.000114,0.002932,0.005506,0.003202,0.001549,-0.000473,-0.004071,0.008974,0.005363,-0.003230,-0.000828,0.001986,-0.000458,-0.001058,0.000791,0.001834,-0.005183,0.008303,0.006399,0.004029,0.007034,0.001662,0.001303,-0.001112,0.007034,0.006750,-0.003524,0.009415,-0.001616,0.004385,0.008067,0.002174,-0.000560,0.000603,0.011841,-0.006732,-0.010899,0.000489,-0.000648,-0.021209,-0.040979,0.017441,-0.005002,-0.037536,0.014936,0.013915,0.002613,0.013402,0.012069,0.000373,-0.005841,-0.005497,0.000974,0.016028,0.011757,-0.012707,-0.011096,-0.013324,0.005072,0.011032,0.002639,-0.000484,0.004463,0.017379,-0.001274,-0.006364,-0.005725,-0.000782,0.001703,-0.014204,0.001482,-0.001844,-0.025163,-0.020967,0.027157,-0.017276,-0.002917,0.013770,-0.022337,0.012615,0.011566,0.006863,-0.021920,0.003337,0.016727,-0.005525,0.008251,-0.002887,0.008109,0.010662,0.000831,-0.005726,-0.008537,0.000056,-0.013381,0.001837,0.010434,0.001114,-0.008187,0.002549,-0.007206,-0.002254,0.012811,0.003458,-0.000266,0.009682,0.009371,0.001708,0.000883,-0.006842,0.004061,-0.000856,-0.002632,0.007387,-0.003136,0.003248,-0.002023,-0.002357,-0.011564,0.012696,-0.006880,0.010849,0.004480,0.000703,0.008567,-0.000714,0.003126,0.001069,0.001743,-0.004026,0.002472,-0.001017,-0.002126,-0.004023,0.001712,-0.006346,0.001862,-0.013725,0.002205,-0.008604,0.006179,0.000758,0.003068,-0.004947,0.008621,0.008481,0.008823,0.003473,-0.007094,0.008749,0.001079,-0.001028,0.003974,0.002161,-0.003953,-0.000948,0.001838,0.004781,0.009102,-0.003032,-0.006562,-0.005754,0.004885,-0.001040,0.004926,0.004644,0.003538,0.002824,-0.000262,-0.001442,-0.007114,-0.004006,0.006389,-0.007602,0.007919,0.003323,0.002428,0.002069,-0.000398,-0.001691,0.006199,0.007670,0.000269,0.005701,-0.004430,0.000134,-0.001654,-0.002803,-0.003652,-0.002213,0.001898,0.003740,0.000357,0.005282,0.000275,-0.005570,0.005369,0.001253,0.007841,-0.000369,-0.003516,-0.001305,-0.003289,0.002763,-0.000007,0.003641,-0.000397,0.000712,-0.008169,-0.005528,-0.000395,-0.001418,-0.032864,-0.020573,0.014206,-0.005905,0.021496,-0.000253,-0.014392,-0.000361,-0.004300,-0.005512,-0.030864,0.018625,-0.017327,-0.006560,0.015667,0.010851,0.010558,-0.006317,0.005600,0.006259,0.021209,-0.002509,-0.009199,-0.019701,-0.001482,-0.007567,0.010594,0.002223,-0.016643,-0.018151,0.003043,-0.006555,0.015532,0.003262,0.022974,-0.002183,0.008171,0.010941,-0.032365,-0.001522,-0.023320,0.001762,-0.000356,0.005420,-0.000200,-0.019087,-0.020773,0.000086,-0.015396,-0.015772,-0.020588,-0.027112,0.049594,0.008563,-0.001242,0.008492,0.001268,-0.024757,0.034336,0.007010,0.009695,0.004098,0.004518,-0.000146,-0.005258,0.010722,0.002222,0.007591,0.013183,-0.014157,0.002203,0.001376,0.008489,-0.007847,-0.001456,0.015549,0.008597,0.000899,0.006776,0.004708,-0.002224,-0.009357,0.000676,0.000709,0.012890,0.003024,-0.002652,0.010879,0.001499,0.001777,-0.003526,0.006411,0.001232,-0.000790,-0.000544,-0.002826,0.006895,-0.003881,-0.001132,-0.006524,-0.008126,-0.002132,0.014666,0.002953,0.006950,-0.000868,0.004985,0.003706,-0.000131,-0.002944,0.010852,-0.018975,-0.000839,0.007183,-0.004644,0.003589,0.006734,0.011569,0.000017,0.002148,0.002085,0.004636,0.001047,-0.006067,0.003478,0.000038,0.006609,-0.000629,0.000509,-0.002274,0.001579,0.001012,0.008841]3f:T7f8,amznVol = [4465400,7494700,6587300,8801100,13053200,5464100,4572100,4360300,4251900,3443600,3801900,3067900,3637300,2817500,2263900,2509500,5077500,2570600,2147600,2642600,2925200,2115600,3299800,3430000,2698400,3846500,2907400,3166300,3313400,3187700,4782200,5473200,3765700,2955100,2335500,2259200,3327500,3174400,4777600,3107700,4290100,4332600,4941100,4075100,7511200,4386600,4872200,4529700,4543500,3185700,2177300,2983100,2650300,3012000,3002900,3209800,4532700,4383200,5466200,5682900,4861900,4676900,3884400,3888500,4278700,3738200,9924400,9681000,6562300,5738700,4153100,4354700,3460500,3391800,3377500,3963000,4849900,3639900,5512600,3986100,7700700,3957100,4104300,2862000,3105600,3080500,3563000,2800900,3569000,3100700,6531800,7277300,4204400,5721100,8220600,7488700,4892600,4544800,5033600,4414000,3621500,3642000,7050200,4268700,4056800,3154900,6855900,4213700,4538400,4313500,4329400,4085100,3460500,5400700,5253100,7257000,6822300,7393200,4772900,10988900,13935900,9444600,6437200,5859900,5295200,5874000,5907200,4500000,6723900,6928400,10285700,14963800,13866100,12460100,9390200,8135500,6955500,5624700,4257400,8192200,6534900,5902200,6806200,5933300,6486900,8427300,6066100,7790000,10878800,5716800,2707600,6257700,5783200,8458700,6613200,5761800,6862300,8694500,8789400,7576100,7494800,6244700,6598000,5271300,6367200,8829800,6523000,8792200,9991800,13640300,7220000,10411800,9722000,8829000,6954500,7983100,6953200,9182600,7993200,8881400,6348800,6507700,4686200,6005900,5998500,6366900,4208900,6020500,6416800,5225200,4089900,4945900,4837700,4632800,5783800,10910300,11506200,4929100,4453100,3939900,4626600,5657500,3317300,4858600,3560300,4120500,4343900,3681700,3337600,3483400,3096200,3184500,2665800,3148800,3025900,4974900,6167400,3681500,3996000,4948100,4667000,3876400,3614500,3552000,2946600,7550900,5429100,6364200,6265600,5767800,6363000,5103800,4865900,4324800,3043000,3320800,4238800,3448100,3980600,3623900,3640500,3752800,3714400,2964000,2654800,3114400,3724400,3044600,2893500,2749900,3373800,4634800] 40:Tb27,births = [12426,13051,12903,12873,12805,8801,7638,12293,13131,13129,12873,12814,8759,7620,7896,12910,13680,13337,12876,8800,7737,12653,13440,12074,13460,12669,8795,7865,12842,13542,13242,13333,13040,8863,7817,12715,13173,13042,13044,12774,8704,7618,12115,13195,12614,12880,12493,8593,7471,11877,12765,12393,12804,12192,8223,7172,11736,12582,12227,12220,12169,7995,7121,11687,12462,12223,12320,12210,8077,7139,12199,12519,11752,10975,12653,8407,7368,12164,12785,12113,12566,12026,8092,7258,11418,13669,12130,12206,12339,8305,7348,12154,12413,12503,12499,12459,8239,7362,12658,13211,11782,7175,9077,7778,7449,12512,13352,12947,12437,11861,8068,7226,11824,12597,12413,12485,11484,8163,7305,12316,13252,13132,13320,13248,8836,7535,12140,9194,6620,11375,13588,9203,7896,13096,12525,8018,11171,12317,8199,7174,11400,12310,11829,11827,11989,8292,7298,11714,12598,11785,11822,11932,8143,6924,11024,12177,11936,12162,11840,8190,7424,11561,12031,11381,11651,11843,8206,7265,11825,12519,11914,11807,12246,8129,7060,11886,12311,12154,11384,12494,8391,7259,11144,12331,11932,12142,11988,8049,7092,11621,12291,11932,11856,11671,8186,7267,11793,12065,12129,12069,12035,8140,6789,11620,12084,11819,11623,11841,8080,6973,11936,12231,11953,12266,11987,8109,7147,11542,12342,11809,11823,11730,8153,7215,11511,11710,12003,11984,12319,7926,7127,11508,12187,11875,12026,12018,7980,7181,11912,12274,11858,12070,11132,7641,6877,11664,12528,12014,12178,11811,7949,7184,11681,12121,11591,12412,12011,8015,7171,11788,12487,12127,12166,11949,8100,7145,11671,12057,12234,12387,12065,8051,7200,11925,12854,12545,12634,12631,8456,7393,7657,12490,13411,12753,12257,8462,7474,11885,12246,12320,12256,11965,8164,7359,11886,12513,12090,12442,11651,8199,7375,12068,12555,12524,12217,12532,8299,7414,11939,12704,12499,12621,12236,8441,7563,12243,13575,13183,13043,9278,7778,7568,13038,13500,12577,13146,12699,8650,7626,12394,13207,12881,12817,12439,8720,7567,12497,13239,12692,12929,12662,8740,7822,12532,13252,12752,12673,12782,8814,7679,12511,13415,12900,13152,12887,8958,7832,12589,13413,12817,13031,13048,8885,7783,12582,13198,13167,13153,12891,8912,7630,12346,13278,13113,13418,13099,8899,7884,8043,13071,13863,13556,13457,8806,7770,12743,13629,13230,12104,13415,8903,8109,13018,13661,13341,13451,13129,9202,8002,12713,13149,13033,12908,12478,8797,7639,12318,12959,13082,12841,12539,8425,7484,11893,12860,12363,12130,12155,8262,7283,11241,12658,12405,12506,12087,8305,7279,11744,12565,12282,12162,12119,8256,7576,12007,12831,12102,12192,10837,8367,7736,12309,12748,12461,12489,12422,8268,7479,11898,12794,12244,12103,12234,8305,7219,11784,12822,12100,12541,12638,8647,7497,12791,13349,11747,7351,9049,8035,7228,12584,13083,12427,12384,11964,8280,7196,11773,12419,12203,12042,12001,8596,7291,12013,12748,12684,12816,12714,8465,7382,12799,12604,9308,6749,10386,8656,7724,12811,13634,11990]41:T142c,// ftp://aftp.cmdl.noaa.gov/products/trends/co2/co2_mm_mlo.txt // via https://www.bloomberg.com/graphics/carbon-clock/ carbon = [315.71,317.45,317.5,317.1,315.86,314.93,313.2,312.66,313.33,314.67,315.62,316.38,316.71,317.72,318.29,318.15,316.54,314.8,313.84,313.26,314.8,315.58,316.43,316.97,317.58,319.02,320.03,319.59,318.18,315.91,314.16,313.83,315,316.19,316.93,317.7,318.54,319.48,320.58,319.77,318.57,316.79,314.8,315.38,316.1,317.01,317.94,318.56,319.68,320.63,321.01,320.55,319.58,317.4,316.26,315.42,316.69,317.69,318.74,319.08,319.86,321.39,322.25,321.47,319.74,317.77,316.21,315.99,317.12,318.31,319.57,320.07,320.73,321.77,322.25,321.89,320.44,318.7,316.7,316.79,317.79,318.71,319.44,320.44,320.89,322.13,322.16,321.87,321.39,318.81,317.81,317.3,318.87,319.42,320.62,321.59,322.39,323.87,324.01,323.75,322.39,320.37,318.64,318.1,319.79,321.08,322.07,322.5,323.04,324.42,325,324.09,322.55,320.92,319.31,319.31,320.72,321.96,322.57,323.15,323.89,325.02,325.57,325.36,324.14,322.03,320.41,320.25,321.31,322.84,324,324.42,325.64,326.66,327.34,326.76,325.88,323.67,322.38,321.78,322.85,324.11,325.03,325.99,326.87,328.13,328.07,327.66,326.35,324.69,323.1,323.16,323.98,325.13,326.17,326.68,327.18,327.78,328.92,328.57,327.34,325.46,323.36,323.57,324.8,326.01,326.77,327.63,327.75,329.72,330.07,329.09,328.05,326.32,324.93,325.06,326.5,327.55,328.54,329.56,330.3,331.5,332.48,332.07,330.87,329.31,327.51,327.18,328.16,328.64,329.35,330.71,331.48,332.65,333.2,332.16,331.07,329.12,327.32,327.28,328.3,329.58,330.73,331.46,331.9,333.17,333.94,333.45,331.98,329.95,328.5,328.34,329.37,330.59,331.59,332.75,333.52,334.64,334.77,334,333.06,330.68,328.95,328.75,330.15,331.62,332.66,333.13,334.95,336.13,336.93,336.17,334.88,332.56,331.29,331.27,332.41,333.6,334.95,335.25,336.66,337.69,338.03,338.01,336.41,334.41,332.37,332.41,333.75,334.9,336.14,336.69,338.27,338.96,339.21,339.26,337.54,335.75,333.98,334.19,335.31,336.81,337.9,338.34,340.01,340.93,341.48,341.33,339.4,337.7,336.19,336.15,337.27,338.32,339.29,340.55,341.62,342.53,343.03,342.54,340.78,338.44,336.95,337.08,338.58,339.88,340.96,341.73,342.81,343.97,344.63,343.79,342.32,340.09,338.28,338.29,339.6,340.9,341.68,342.9,343.33,345.25,346.03,345.63,344.19,342.27,340.35,340.38,341.59,343.05,344.1,344.79,345.52,346.84,347.63,346.98,345.53,343.55,341.4,341.67,343.1,344.7,345.21,346.16,347.74,348.34,349.06,348.38,346.72,345.02,343.27,343.13,344.49,345.88,346.56,347.28,348.01,349.77,350.38,349.93,348.16,346.08,345.22,344.51,345.93,347.21,348.52,348.73,349.73,351.31,352.09,351.53,350.11,348.08,346.52,346.59,347.96,349.16,350.39,351.64,352.4,353.69,354.21,353.72,352.69,350.4,348.92,349.13,350.2,351.41,352.91,353.27,353.96,355.64,355.86,355.37,353.99,351.81,350.05,350.25,351.49,352.85,353.8,355.04,355.73,356.32,357.32,356.34,354.84,353.01,351.31,351.62,353.07,354.33,354.84,355.73,357.23,358.66,359.13,358.13,356.19,353.85,352.25,352.35,353.81,355.12,356.25,357.11,357.86,359.09,359.59,359.33,357.01,354.94,352.96,353.32,354.32,355.57,357,357.31,358.47,359.27,360.19,359.52,357.33,355.64,354.03,354.12,355.41,356.91,358.24,358.92,359.99,361.23,361.65,360.81,359.38,357.46,355.73,356.08,357.53,358.98,359.92,360.86,361.83,363.3,363.69,363.19,361.64,359.12,358.17,357.99,359.45,360.68,362.07,363.24,364.17,364.57,365.13,364.92,363.55,361.38,359.54,359.58,360.89,362.24,363.09,364.03,364.51,366.35,366.64,365.59,364.31,362.25,360.29,360.82,362.49,364.38,365.27,365.98,367.24,368.66,369.42,368.99,367.82,365.95,364.02,364.4,365.52,367.13,368.18,369.07,369.68,370.99,370.96,370.3,369.45,366.9,364.81,365.37,366.72,368.1,369.29,369.54,370.6,371.81,371.58,371.7,369.86,368.13,367,367.03,368.37,369.67,370.59,371.51,372.43,373.37,373.85,373.21,371.51,369.61,368.18,368.45,369.76,371.24,372.53,373.2,374.12,375.02,375.76,375.52,374.01,371.85,370.75,370.55,372.25,373.79,374.88,375.64,376.45,377.73,378.6,378.28,376.7,374.38,373.17,373.15,374.66,375.99,377,377.87,378.88,380.35,380.62,379.69,377.47,376.01,374.25,374.46,376.16,377.51,378.46,379.73,380.77,382.29,382.45,382.21,380.74,378.74,376.7,377,378.35,380.11,381.38,382.19,382.67,384.61,385.03,384.05,382.46,380.41,378.85,379.13,380.15,381.82,382.89,383.9,384.58,386.5,386.56,386.1,384.5,381.99,380.96,381.12,382.45,383.95,385.52,385.82,386.03,387.21,388.54,387.76,386.36,384.09,383.18,382.99,384.19,385.56,386.94,387.48,388.82,389.55,390.14,389.48,388.03,386.11,384.74,384.43,386.02,387.42,388.71,390.2,391.17,392.46,393,392.15,390.2,388.35,386.85,387.24,388.67,389.79,391.33,391.86,392.6,393.25,394.19,393.74,392.51,390.13,389.08,389,390.28,391.86,393.12,393.86,394.4,396.18,396.74,395.71,394.36,392.39,391.11,391.05,392.98,394.34,395.55,396.8,397.43,398.41,399.78,398.61,397.32,395.2,393.45,393.7,395.16,396.84,397.85,398.01,399.77,401.38,401.78,401.25,399.1,397.03,395.38,396.03,397.28,398.91,399.98,400.28,401.54,403.28,403.96,402.8,401.31,398.93,397.63,398.29,400.16,401.85,402.56,404.12,404.87,407.45,407.72,406.83,404.41,402.27,401.05,401.59,403.55,404.45,406.17,406.46,407.22,409.04,409.69,408.88,407.12,405.13,403.37,403.63,405.12,406.81,407.96,408.32,409.41,410.24,411.24,410.79,408.71,406.99,405.51,406,408.02,409.07,410.83,411.75,411.97]42:T465,miniline = (n = 3, w = 15, h = 20) => { let data = d3.range(n) .map(i => Math.sin(i ** 2 + Date.now() / 6000)) const x = d3.scaleLinear().domain([0, data.length - 1]).range([0, w]) const y = d3.scaleLinear().domain([-1, 1]).range([h, 0]) const line = d3.line().x((d,i) => x(i)).y(d => y(d)) const node = svg`` const path = d3.select(node).select("path") const circle = d3.select(node).selectAll("circle").data(data) const timer = d3.timer(function(t) { data = d3.range(n) .map(i => Math.sin(i ** 2 + Date.now() / 6000)) path .attr("d", line(data)) .attr("stroke-dashoffset", 30 - (Date.now() / 240) % 60) circle.data(data).attr("cy", d => y(d)) if (t > 10000) timer.stop() }) return node }43:T1389,style = html``44:T78a,// https://stackoverflow.com/questions/13069446/simple-fill-pattern-in-svg-diagonal-hatching defs = svg``45:T6af,seriesPicker = (_t = 0) => { const w = width > 500 ? 55 : 35 const m = 3 let i = 1 let t = _t const node = html`

${series.map((d, i) => { const data = d.data.slice(0, 15) const x = d3.scaleBand() .domain(data.map((d, i) => i)) .range([m, w - m]) .padding(0.2) const y = d3.scaleLinear() .domain(d3.extent(data)) .range([w - m, m]) return html`` })}

with a lag of

` const choices = node.querySelectorAll(".pick-dataset") const name = node.querySelector(".pick-dataset-name") const input = node.querySelector("input") choices.forEach(n => n.addEventListener("click", () => setI(+n.dataset.i))) input.addEventListener("input", e => setT(+e.target.value)) update() function setI(_i) { i = _i update() } function setT(_t) { t = _t update() } function update() { choices.forEach((n, ii) => { if (i === ii) { n.classList.add("active") } else { n.classList.remove("active") } }) name.innerText = series[i].name node.value = lag(series[i].data, t); node.dispatchEvent(new CustomEvent('input')); } return node }46:[] 30:{"title":"Inferring chart type from autocorrelation and other evils","cells":[{"id":1704,"value":"md`# Inferring chart type from autocorrelation and other evils`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3137,"value":"md`_First I will consider a concrete problem for which I sought to use correlation; then, being not much of a statistician, I will refresh my memory of how it works; finally, I will consider the history of its conception._`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3112,"value":"i = html`

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4538,"value":"html`

“And there’s a dreadful law here — it was made by mistake, but there it is — that if any one asks for machinery they have to have it and keep on using it.”

E. Nesbit
The Magic City

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":1158,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":420,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":775,"value":"md`Where the deviations from the mean ${tex`\\mu`} are in the same direction, there’s a positive (blue-shaded) contribution to the autocorrelation; where they point in opposite directions, it’s negative (red-shaded); the autocorrelation is the normalized average. The dots, showing autocorrelation vs. lag, are sometimes called a [correlogram](https://en.wikipedia.org/wiki/Correlogram). It’s symmetrical around ${tex`0`}, since shifting one copy to the left is the same as shifting the other copy to the right. For periodic series, it’s periodic: births are highly correlated with themselves on a ${tex`7`}-day lag, and then again on a ${tex`14`}-day lag, and so on.\n\nTo compute this, we can start with the covariance ${tex`cov`} of the two series ${tex`X`} and ${tex`Y`}. For each entry in the series (where neither is undefined), multiply ${tex`X`}’s distance from its mean by ${tex`Y`}’s distance from its mean, then average:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":831,"value":"tex`cov \\left( X, Y \\right) = \\frac{1}{N} \\sum_{i=1}^{N} \\left( X_i - \\mu\\left(X\\right) \\right)\\left( Y_i - \\mu\\left(Y\\right) \\right)`","pinned":false,"hidden":false,"mode":"ojs"},{"id":61,"value":"_cov = (X, Y) => (1 / (X.length - 1)) * Σ(\n X.map((_, i) => \n (X[i] - μ(X)) * (Y[i] - μ(Y))\n )\n)","pinned":true,"hidden":false,"mode":"ojs"},{"id":945,"value":"md`We normalize the covariance by dividing it by the product of the standard deviations ${tex`\\sigma`} of the two series. This is the correlation coefficient ${tex`corr`}, which will always be between ${tex`-1`} and ${tex`1`}, regardless of the units of the two series:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":828,"value":"tex`corr \\left(X, Y \\right) = \\frac{cov\\left(X, Y\\right)}{\\sigma\\left(X\\right) \\sigma\\left(Y\\right)}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":63,"value":"_corr = (X, Y) => cov(X, Y) / (σ(X) * σ(Y))","pinned":true,"hidden":false,"mode":"ojs"},{"id":1593,"value":"md`Finally, ${tex`lag`} (or \\`shift\\`) moves the ${tex`i-t`}th entry of a series ${tex`X`} to the ${tex`i`}th spot. So, for ${tex`t = ${demo}`}, ${demo ? `each moves ${Math.abs(demo)} spot${Math.abs(demo) === 1 ? \"\" : \"s\"} ${demo > 0 ? \"forward\" : \"backward\"}` : \"nothing changes\"}.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":806,"value":"tex`lag\\left(X_i,t\\right) = X_{i-t}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":50,"value":"lag = (X, t) => X.map((_, i) => X[i - t])","pinned":true,"hidden":false,"mode":"ojs"},{"id":947,"value":"md`The autocorrelation ${tex`autocorr`} of ${tex`X`} for offset ${tex`t`} is the correlation of ${tex`X`} and the lagged version of itself, ${tex`lag(X,t)`} — thus, roughly, the degree to which each value in the series predicts the one ${tex`t`} places later:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":814,"value":"tex`autocorr\\left(X,t\\right) = corr\\left(X, lag\\left(X,t\\right) \\right)`","pinned":false,"hidden":false,"mode":"ojs"},{"id":75,"value":"autocorr = (X, t) => corr(X, lag(X, t))","pinned":true,"hidden":false,"mode":"ojs"},{"id":955,"value":"md`Recall that the point here is that the degree to which each value in a series predicts the next is a good heuristic for picking what kind of chart to use for the series (because lines connect points and bars don’t). So that’s an autocorrelation with an offset of ${tex`t=1`} (since offsetting by 1 compares each point to the next). If \\`autocorr(series, 1)\\` is above some \\`threshold\\`, draw a \\`line\\` chart of the \\`series\\`; otherwise, a \\`bar\\` chart:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":315,"value":"decide = series => autocorr(series, 1) > threshold ? line(series) : bar(series)","pinned":true,"hidden":false,"mode":"ojs"},{"id":959,"value":"md`Here, that logic is applied to each of our examples, with a threshold of ${threshold}:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":317,"value":"html`

\n ${series.map(series => html`\n

${series.name} ${series.src ? html`src` : ''}

autocorr. ${autocorr(series.data, 1).toFixed(3)}

\n ${decide(series.data)}`\n )}\n

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4398,"value":"md`Try adding your own data:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3159,"value":"viewof yourdata = text({placeholder: \"1, 2, 3… (space-, tab-, or comma-separated)\", submit: \"Update\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4396,"value":"md`Or change the threshold:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1260,"value":"$35","pinned":false,"hidden":false,"mode":"ojs"},{"id":1935,"value":"md`One can imagine many more heuristics, especially if you have metadata. Points drawn from a stationary distribution should probably be bars, but I don’t think the inverse is true. If the scale doesn’t have a meaningful zero, bars make a lot less sense. If the items in the series are ratios, then zero is as unattainable as infinity, the y-scale should probably be logarithmic, and you should mark the multiplicative identity (one) instead of the additive (zero). I should probably know more about [levels of measurement](https://en.wikipedia.org/wiki/Level_of_measurement) and read more Tukey. This sort of thing is no substitute for good taste and judgment. But it may be good enough to pretend for a little while that you don’t need those.\n\n---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3326,"value":"ii = html`

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3736,"value":"html`

“They have patched up my legs and given me a job I can do sitting down: I count the people crossing the new bridge. They beam when I hand over the results of my day’s labors, the higher the number the broader their smiles. But their statistics are wrong. I was only one short. My little sweetheart had walked past, and as long as I live I won’t allow that adorable child to be whisked off into a percentage.”

Heinrich Böll
At the Bridge

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3324,"value":"md`Motivated by the chart-selection problem, we’ve taken a look at one kind of correlation, and understood just enough to get to some kind of solution. I previously defined correlation as ${tex`\\frac{cov\\left(X, Y\\right)}{\\sigma\\left(X\\right) \\sigma\\left(Y\\right)}`}, which works. But I don’t understand covariance and standard deviation well enough to see what’s going on inside them. If we expand the covariance and standard deviations to their definitions, we get:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1627,"value":"(width > 500 ? tex.block: tex)`${width > 500 ? `corr(X, Y) =` : ''}\\frac\n {\\sum_{i=1}^{N} \\left( X_i - \\mu\\left(X\\right) \\right)\\left( Y_i - \\mu\\left(Y\\right) \\right)}\n {\\sqrt{\\sum_{i=1}^{N} \\left( X_i - \\mu\\left(X\\right) \\right)^2} \n \\sqrt{\\sum_{i=1}^{N} \\left( Y_i - \\mu\\left(Y\\right) \\right)^2}}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2039,"value":"md`If we assume the means are zero it may be easier to see:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2041,"value":"tex.block`\\frac\n {\\sum X_i Y_i}\n {\\sqrt{\\sum X_i^2} \n \\sqrt{\\sum Y_i^2}}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2051,"value":"$36","pinned":false,"hidden":false,"mode":"ojs"},{"id":3223,"value":"viewof correlX = seriesPicker(0)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3225,"value":"viewof correlY = seriesPicker(1)","pinned":false,"hidden":false,"mode":"ojs"},{"id":2138,"value":"$37","pinned":false,"hidden":false,"mode":"ojs"},{"id":3996,"value":"md`On the top (in the wings), elements are multiplied (forming the sides of a rectangle) and added (stacked, here in an L-shape). On the bottom (in the middle), the rectangle is made by two sets of stacking right triangles; each new element in the series is added perpendicular to the existing side, and then the new hypotenuse becomes the new rectangle side. This represents a generalization of the Pythagorean theorem to take any number of numbers, i.e. the Euclidean distance or ${tex`l^2`}-norm.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2933,"value":"$38","pinned":false,"hidden":false,"mode":"ojs"},{"id":2569,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3133,"value":"iii = html`

III

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4537,"value":"html`

“Then Lila, drying her tears with the back of her hand, asked ‘Who are the Nazi Fascists, Pascà? Who are the monarchists? What’s the black market?’ It’s hard to say what Pasquale’s answers did to Lila. But she, in her usual way, was moved and altered by them, so that for the entire summer she tormented me with a single concept that I found quite unbearable: there are no gestures, words, or sighs that do not contain the sum of all the crimes that human beings have committed and commit.”\n

Elena Ferrante
My Brilliant Friend

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":766,"value":"$39","pinned":false,"hidden":false,"mode":"ojs"},{"id":1885,"value":"sources = md`---\n\n#### Sources and further reading`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4016,"value":"viewof excerpts = radio({\n options: [\n { label: 'Full notes', value: 'full' },\n { label: 'Mini notes', value: 'mini' },\n { label: 'Hide notes', value: 'hide' },\n ],\n value: 'mini'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1887,"value":"$3a","pinned":false,"hidden":false,"mode":"ojs"},{"id":3473,"value":"$3b","pinned":false,"hidden":false,"mode":"ojs"},{"id":3310,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4334,"value":"md`#### Code\n\nTo keep the function definitions in the main text as short and close to the math as possible, the logic to trim entries where either series is undefined has been relegated to here.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4323,"value":"cov = (X, Y) => _cov(...trim([X, Y]))","pinned":true,"hidden":false,"mode":"ojs"},{"id":4311,"value":"corr = (X, Y) => _corr(...trim([X, Y]))","pinned":true,"hidden":false,"mode":"ojs"},{"id":4194,"value":"trim = arr => {\n const trimmed = d3.transpose(\n d3.transpose(arr)\n .filter(i => i.every(d => d !== undefined))\n )\n return trimmed.length ? trimmed : arr.map(() => [])\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":196,"value":"md`#### Data\n\nTo add an example dataset to the notebook, add an object to this array with \\`name\\` and \\`data\\`.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":313,"value":"$3c","pinned":false,"hidden":false,"mode":"ojs"},{"id":174,"value":"$3d","pinned":false,"hidden":false,"mode":"ojs"},{"id":176,"value":"$3e","pinned":false,"hidden":false,"mode":"ojs"},{"id":178,"value":"amznRev = [0.0157,0.1478,0.6098,1.64,2.762,3.122,3.933,5.264,6.921,8.49,10.71,14.84,19.17,24.51,34.2,48.08,61.09,74.45,88.99,107.01,135.99,177.87,232.89]","pinned":false,"hidden":false,"mode":"ojs"},{"id":180,"value":"amznProfit = [-6.2,-31,-124.5,-720,-1411,-567.28,-149.13,35.28,588,359,190,476,645,902,1152,631,-39,274,-241,596,2371,3033,10070]","pinned":false,"hidden":false,"mode":"ojs"},{"id":329,"value":"$3f","pinned":false,"hidden":false,"mode":"ojs"},{"id":404,"value":"$40","pinned":false,"hidden":false,"mode":"ojs"},{"id":365,"value":"$41","pinned":false,"hidden":false,"mode":"ojs"},{"id":1549,"value":"userdata = {\n const attempts = [\",\", \"\\t\", \" \"].map(d => yourdata.split(d))\n const lengths = attempts.map(d => d.length)\n const i = lengths.indexOf(Math.max(...lengths))\n return attempts[i]\n .map(parseFloat)\n .filter(d => !isNaN(d))\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4584,"value":"md`#### Charts`","pinned":false,"hidden":false,"mode":"ojs"},{"id":224,"value":"bar = (data, w = width, h) => {\n if (h === undefined) h = width / 4\n const x = d3.scaleBand()\n .domain(data.map((d, i) => i))\n .range([0, w])\n .padding(0.1)\n const y = d3.scaleLinear()\n .domain(d3.extent(data.concat([0])))\n .range([h, 0])\n \n return (\n svg``\n )\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":217,"value":"line = (data, w = width, h) => {\n if (h === undefined) h = width / 4\n const x = d3.scaleLinear().domain([0, data.length - 1]).range([0, w])\n const y = d3.scaleLinear().domain(d3.extent(data)).range([h, 0])\n const line = d3.line().x((d,i) => x(i)).y(d => y(d))\n return svg``\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4954,"value":"minibar = (n = 3, w = 15, h = 20) => { \n let data = d3.range(n)\n .map(i => Math.sin(i ** 2 + Date.now() / 5000))\n \n const x = d3.scaleBand()\n .domain(data.map((d, i) => i))\n .range([0, w])\n .padding(0.2)\n const y = d3.scaleLinear()\n .domain([-1, 1])\n .range([h, 0])\n \n const node = svg``\n \n const bar = d3.select(node).selectAll(\"rect\")\n \n const timer = d3.timer(function(t) {\n data = d3.range(n)\n .map(i => Math.sin(i ** 2 + Date.now() / 5000))\n bar.data(data)\n .attr(\"y\", d => y(Math.max(d, 0)))\n .attr(\"height\", d => Math.abs(y(0) - y(d)))\n if (t > 10000) timer.stop()\n })\n \n return node\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4952,"value":"$42","pinned":false,"hidden":false,"mode":"ojs"},{"id":194,"value":"md`#### Graphics`","pinned":false,"hidden":false,"mode":"ojs"},{"id":372,"value":"$43","pinned":false,"hidden":false,"mode":"ojs"},{"id":3043,"value":"$44","pinned":false,"hidden":false,"mode":"ojs"},{"id":2969,"value":"bend = s => svg``","pinned":false,"hidden":false,"mode":"ojs"},{"id":2502,"value":"poly = arr => \"M\" + arr.join(\"L\") + \"Z\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":2423,"value":"square = area => [\n [0, 0],\n [Math.sign(area) * Math.sqrt(Math.abs(area)), 0],\n [Math.sign(area) * Math.sqrt(Math.abs(area)), -Math.sqrt(Math.abs(area))],\n [0, -Math.sqrt(Math.abs(area))],\n]","pinned":false,"hidden":false,"mode":"ojs"},{"id":2417,"value":"parallelogram = (a, b) => [\n [0, 0], a, [a[0] + b[0], a[1] + b[1]], b\n]","pinned":false,"hidden":false,"mode":"ojs"},{"id":2451,"value":"getL = (_A, _B, r) => {\n \n const A = Math.abs(_A)\n const B = Math.abs(_B)\n const s = Math.sign(_B)\n \n // Shown for negative outer area B; \n // points start at origin (inner concave vertex) \n // negative B: ↻ clockwise\n // positive B: ↺ counterclockwise\n \n /* ← b₂ → \n ⁴┌─────────┐⁵ \n │ B │ A = a₁ * a₂\n ↑ │ ¹┌─────┘₆ B = b₁ * b₂ - a₁ * a₂\n b₁│ │↑←a₂→ a₁ / a₂ = r\n ↓ │ │a₁ b₁ / b₂ = r\n │ │↓ A\n └───┘ ∙(0,0)\n ³ ²\n */\n \n const a2 = Math.sqrt(A / r)\n const a1 = A / a2 || 0\n const b2 = Math.sqrt((A + B) / r)\n const b1 = (A + B) / b2 || 0\n \n return [\n [s*a2, -a1],\n [s*a2, 0],\n [s*b2, 0],\n [s*b2, -b1],\n [0, -b1],\n [0, -a1]\n ]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":2514,"value":"$45","pinned":false,"hidden":false,"mode":"ojs"},{"id":3906,"value":"md`#### Math`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2492,"value":"Π = arr => arr.reduce((acc, cur) => acc * cur, 1)","pinned":false,"hidden":false,"mode":"ojs"},{"id":68,"value":"σ = d3.deviation","pinned":false,"hidden":false,"mode":"ojs"},{"id":77,"value":"Σ = d3.sum","pinned":false,"hidden":false,"mode":"ojs"},{"id":65,"value":"μ = d3.mean","pinned":false,"hidden":false,"mode":"ojs"},{"id":2415,"value":"rotate = (length, degrees) => [Math.sin, Math.cos]\n .map(f => length * f(-degrees * (Math.PI / 180)))","pinned":false,"hidden":false,"mode":"ojs"},{"id":4470,"value":"center = arr => {\n const mean = μ(arr)\n return arr.map(d => d === undefined ? undefined : d - mean)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3976,"value":"md`#### Imports`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3157,"value":"import {radio, text} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":26,"value":"d3 = require(\"d3@6\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":3853,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4535,"value":"html`

“Wouldn’t you be happier if I had been able to show you that all the perpetrators were crazy?”

Raul Hilberg

`\n\n/* To-do\n- Spark animations? https://observablehq.com/d/a2d2bfd89759ed2d\n- Fig. 1: label 1, 0, x-axis\n- Fig. 2: show fewer points on mobile\n- Fig. 2: re-use colored dots motif?\n- Fig. 2: debounce threshold slider input\n- Fig. 3: y-scale as geometric mean of extent of series and sqrt of areas?\n- Fig. 3: mask/clip stuff far above/below?\n- Fig. 3: factor out interpolation boilerplate\n- Fig. 3: refactor a lot of redundant math\n*/","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W46","stdlib":"1","titling":"auto"}