2c:T481,{ const combinedWidth = Math.min(640, width); const nullPlotWidth = 26; const flexGap = 2; // First get the shared y scale (to ensure its domain includes the max of both) const y = Plot.plot({ marks: [ Plot.rectY(olympians, Plot.binX({y: "count"}, {x: "weight", thresholds})), Plot.barY(olympians.filter(d => d.weight === null), Plot.groupX({y: "count"}, {x: "weight"})) ] }).scale("y"); // To have disjoint x-axis (one continuous, one ordinal), we split the two marks into their own plots const plot1 = Plot.plot({ width: combinedWidth - (nullPlotWidth + flexGap), y, marks: [ Plot.rectY(olympians, Plot.binX({y: "count"}, {x: "weight", thresholds})), Plot.ruleY([0]) ] }) const plot2 = Plot.plot({ y: {...y, axis: false}, x: {tickFormat: d => d === null ? "NULL" : "", label: null}, marginLeft: 8, marginRight: 8, width: nullPlotWidth, marks: [ Plot.barY(olympians.filter(d => d.weight === null), Plot.groupX({y: "count"}, {x: "weight"})) ] }) return htl.html`

${plot1} ${plot2}

` }2d:T779,{ const flexGap = 10; const combinedWidth = Math.min(640, width); // First get the shared y scale (to ensure its domain includes the max of both) const y = Plot.plot({ marks: [ Plot.rectY(olympians, Plot.binX({y: "count"}, {x: "weight", thresholds})), Plot.barY(olympians.filter(d => d.weight === null), Plot.groupX({y: "count"}, {x: "weight"})) ] }).scale("y"); // To have disjoint x-axis (one continuous, one ordinal), we split the two marks into their own plots const plot1Options = { width: 640, y, style: {flexShrink: 1}, marginRight: 0, marginLeft: 40, style: {overflow: "visible"}, marks: [ Plot.rectY(olympians, Plot.binX({y: "count"}, {x: "weight", thresholds})), Plot.ruleY([0]) ] }; const _plot1 = Plot.plot(plot1Options); // Infer bin width from x scale. I don’t bother trying to do the math to fit both plots into a fixed width const x = _plot1.scale("x"); const binWidth = x.apply(thresholds[1]) - x.apply(thresholds[0]); // Math to scale down to fit const unscaledWidth = plot1Options.width + flexGap + binWidth; const widthScaleFactor = combinedWidth / unscaledWidth; const plot1 = Plot.plot({ ...plot1Options, width: plot1Options.width * widthScaleFactor, marginLeft: plot1Options.marginLeft * widthScaleFactor, marginRight: plot1Options.marginRight * widthScaleFactor }); const plot2 = Plot.plot({ y: {...y, axis: false}, x: {tickFormat: d => d === null ? "NULL" : "", label: null, padding: 0}, marginLeft: 0, marginRight: 0, width: binWidth * widthScaleFactor, style: {overflow: "visible"}, marks: [ Plot.barY(olympians.filter(d => d.weight === null), Plot.groupX({y: "count"}, {x: "weight"})) ] }); return htl.html`

${plot1} ${plot2}

` }2e:T7c5,{ const flexGap = 10; const combinedWidth = Math.min(640, width); const thresholds = d3.range(...d3.extent(olympians, d => d.height), 0.02) // First get the shared y scale (to ensure its domain includes the max of both) const y = Plot.plot({ marks: [ Plot.rectY(olympians, Plot.binX({y: "count"}, {x: "height", thresholds})), Plot.barY(olympians.filter(d => d.height === null), Plot.groupX({y: "count"}, {x: "height"})) ] }).scale("y"); // To have disjoint x-axis (one continuous, one ordinal), we split the two marks into their own plots const plot1Options = { width: 640, y, style: {flexShrink: 1}, marginRight: 0, marginLeft: 40, style: {overflow: "visible"}, marks: [ Plot.rectY(olympians, Plot.binX({y: "count"}, {x: "height", thresholds})), Plot.ruleY([0]) ] }; const _plot1 = Plot.plot(plot1Options); // Infer bin width from x scale. I don’t bother trying to do the math to fit both plots into a fixed width const x = _plot1.scale("x"); const binWidth = x.apply(thresholds[1]) - x.apply(thresholds[0]); // Math to scale down to fit const unscaledWidth = plot1Options.width + flexGap + binWidth; const widthScaleFactor = combinedWidth / unscaledWidth; const plot1 = Plot.plot({ ...plot1Options, width: plot1Options.width * widthScaleFactor, marginLeft: plot1Options.marginLeft * widthScaleFactor, marginRight: plot1Options.marginRight * widthScaleFactor }); const plot2 = Plot.plot({ y: {...y, axis: false}, x: {tickFormat: d => d === null ? "NULL" : "", label: null, padding: 0}, marginLeft: 0, marginRight: 0, width: binWidth * widthScaleFactor, style: {overflow: "visible"}, marks: [ Plot.barY(olympians.filter(d => d.height === null), Plot.groupX({y: "count"}, {x: "height"})) ] }); return htl.html`

${plot1} ${plot2}

` }2f:T7d1,Are more or fewer heights null? When you’re paying attention, you can tell (it’s fewer, ${olympians.filter(d => d.height === null).length}) — but your automatic intuition would not be reliable, because it depends on comparing rectangles of different proportions to different sums of areas. Then again, comparing multiple histograms with different _x_-axes and binning is always going to be difficult, but at least you can compare _shapes_. A histogram excels at showing the shape of a distribution. The shape is more stable across possible bin sizes than the bin counts, and more important. Looking at a histogram, we more readily say “it peaks around 70 kg and is skewed to the right” than “there are 1,404 athletes between 70 and 75 kg”. In the limit (of infinitely many samples of a continuous variable binned infinitesimally), a histogram approximates the probability density function. And there, the _y_ value represents the abstract density, which is not intuitively interpretable; it’s just whatever it needs to be to maintain the property that the area between two _x_ coordinates represents the probability of the variable falling in that range. Above we have maintained that property: the area of a bar, as a fraction of the total area, represents the probability of either (1) falling within that bin or (2) being exactly null. But the shape is not stable. We perceive either a tall right tail or a short one. In the limit, as bin size shrank, the height of the null bar would diverge to infinity. That would be illegible, but it would be honest about density: if you have any null data at all, it has _infinite density_, because you’d have some number of rows divided by zero variation in _x_. So considering density reminds us that, from another point of view, the null bar should always be zero-width! After all, the _x_-axis represents bin size. And the null bar has none; it represents exactly one value, not a range; so it should have zero extent on the _x_-axis.30:T50d,{ const combinedWidth = Math.min(640, width); const nullPlotWidth = 26; const flexGap = 2; // First get the shared y scale (to ensure its domain includes the max of both) const y = Plot.plot({ marks: [ Plot.rectY(olympians, Plot.binX({y: "count"}, {x: "weight", thresholds})), Plot.barY(olympians.filter(d => d.weight === null), Plot.groupX({y: "count"}, {x: "weight"})) ] }).scale("y"); // To have disjoint x-axis (one continuous, one ordinal), we split the two marks into their own plots const plot1 = Plot.plot({ width: combinedWidth - (nullPlotWidth + flexGap), y, marks: [ Plot.rectY(olympians, Plot.binX({y: "count"}, {x: "weight", thresholds})), Plot.ruleY([0]) ] }) const plot2 = Plot.plot({ y: {...y, axis: false}, x: {tickFormat: d => d === null ? "NULL" : "", label: null}, marginLeft: 8, marginRight: 8, width: nullPlotWidth, marks: [ Plot.ruleY([0]), Plot.tickY(olympians.filter(d => d.weight === null), Plot.groupX({y: "count"}, {x: "weight"})), Plot.ruleX(olympians.filter(d => d.weight === null), Plot.groupX({y1: () => 0, y2: "count"}, {x: "weight"})) ] }) return htl.html`

${plot1} ${plot2}

` }31:T59b,This is like a variation of the first chart with a more honest _x_ encoding. But now it looks weirder, and we’ve lost the areal commensurability again. Trying to make this work feels like map projections trying to flatten a sphere: you have to forfeit something. When you try to combine ordinal and continuous _x_ values, some part of the visual metaphor of a histogram will break. We tried to put everything on the same _y_-axis, because “count” (of rows in a bin) and “count” (of null rows) seem like they’re in the same units. But I think they’re actually not. The histogram’s _y_-axis is not independent of its _x_-axis. Its _y_-axis is more like “count _per bin_”, whereas the null bar is a straight “count”. The meaningful thing about the bin bars is the area, but the meaningful thing about the null bar is the height. The types don’t match and never will. The dimensional analysis fails. As a clue that we’re missing something, consider a histogram with varying bin sizes. (See [Wikipedia](https://en.wikipedia.org/wiki/Histogram#Variable_bin_widths), [StackOverflow](https://stackoverflow.com/questions/37766893/how-to-make-variable-width-histogram-in-r-with-labels-aligned-to-bin-edges), [Jonathan Helfman’s notebook](https://observablehq.com/@jonhelfman/plot-histogram-with-unequal-bin-widths).) This is possible! This works! See, here’s the same weight data, except each bin is a quartile:32:T561,But note that, to make this work, the _y_-axis no longer measures the straight-up count of items in the bin. (If it did, then _e.g._ extending the right tail bin twice as far would include no more points, but would double the area, which would be misleading.) Instead, it shows density: the count _per_ width of the bin. In fact, you could argue that histograms _always_ show density! It’s just that if you have equal-width bins, then the denominator cancels out, and you can simplify the _y_-axis to label exact counts. But by introducing the null bar, we’ve broken the equal-width criterion, and the metaphor has begun to fall apart. And we can’t just respect the varying density, because the null bin would be zero-width, and thus infinite density. We could still draw a null bar with “areal commensurability” for the variable-bin-size histogram, but instead of just matching the bin bar width, it’d have to calculate the total area under the non-null bars, multiply it by the ratio of null rows to non-null rows, and then draw a shape with that area. What sort of shape? It could be a rectangle, but neither width nor height could have any meaning, since strictly speaking the width (bin size) should be zero and the height (density) should be infinite. So, if the only meaningful thing is the area, it could be _any_ shape. It could be like a sun in the sky!33:T4ad,{ const weights = olympians.map(d => d.weight).filter(d => d !== null).sort(d3.ascending); const nBins = 4; const itemsPerBin = weights.length / nBins; const bins = []; for (let i = 0; i < weights.length; i += itemsPerBin) { const chunk = weights.slice(i, i + itemsPerBin); bins.push(Object.assign(chunk, {x0: chunk[0], x1: chunk.at(-1)})); } const x = variable.scale("x"); const y = variable.scale("y"); let nonNullArea = 0; for (const bin of bins) { const dx = x.apply(bin.x1) - x.apply(bin.x0); const dy = y.apply(0) - y.apply(bin.length / (bin.x1 - bin.x0)); nonNullArea += dx * dy; } const nullArea = nonNullArea * (olympians.filter(d => d.weight === null).length / weights.length); return Plot.plot({ x: {label: "weight"}, y: {label: "density"}, marks: [ Plot.rectY(bins, {x1: "x0", x2: "x1", y: d => d.length / (d.x1 - d.x0), insetRight: 1}), Plot.dot([null], {frameAnchor: "top-right", r: Math.sqrt(nullArea / Math.PI), fill: "black", dx: -50, dy: 50}), Plot.text([null], {frameAnchor: "top-right", text: () => "NULL", dx: -50, dy: 46, fill: "white", textAnchor: "middle"}), Plot.ruleY([0]) ] }) }34:T866,That’s ugly but you get the idea. Something like that. Now we can see the null fraction on a constant scale, comparable across all histograms (if we were to draw many of these). Notice there are no axes on the null chart. Is that confusing? Are you able to intuit the spatial domain over which the axes have bearing? Do you see the null bar as a kind of separate “progress bar”, or out-of-100% bar, like those thermometer displays that fundraisers do? You’re supposed to! But, in the name of minimalism, I’ve omitted separate axis labels; I’m not spelling it out! Consider how similar this looks to my first chart. I say this one is less misleading, but whether I’m right depends on the perceptual idiosyncrasies of everyone who looks at this and how they mentally group the different bits of black against the white screen. That is: this notebook did all this work wrestling with “do these geometrical proportions agree?”, and then decided to punt on it, and just juxtapose two charts. But two juxtaposed charts look an awful lot like two marks on one chart! So now we’re in the realm of gestalt psychology. Previously, there was nothing stopping me from drawing nonsensical rectangles. Now there’s nothing stopping me from juxtaposing non-sequiturs. I guess all language is like that. All mimsy were the borogoves! Then again, I did have to break out of the normal Observable Plot patterns to draw these charts. So there _was_ something stopping me, and I just overcame it. Can you imagine a “grammar of graphics” with such fine-grained type assertions and automatic labeling that it makes whole categories of lies impossible? You’d have to be able to say things like “this number represents a scale with no meaningful zero, and this one does have a meaningful zero, and that one is a ratio,” and so on. It could describe aspects of the handling of the data, like how it was binned and such — but then you wouldn’t know where the data’s from. So you could have whole cryptographically signed Merkle trees of data provenance. Who has a null height, anyway? Who was measuring? Did the Olympians have socks on?35:[] 2a:{"title":"Showing discrete nulls with a histogram","cells":[{"id":0,"value":"# Showing discrete nulls with a histogram\nConsider a typical histogram of the weight of athletes in the _olympians_ [sample dataset](https://observablehq.com/@observablehq/sample-datasets). The _x_-axis would range from about ${d3.min(olympians, d => d.weight)} to ${d3.max(olympians, d => d.weight)}; the _y_-axis would range from 0 to the highest count in any one bin. But you’d have no way of seeing how many rows in the data have a _null_ weight!\n\nSo you can imagine putting a special “null” bar alongside the histogram:","pinned":false,"hidden":false,"mode":"md"},{"id":7,"value":"$2c","pinned":false,"hidden":false,"mode":"ojs"},{"id":74,"value":"At first glance, this looks reasonable. You can read off the _y_-axis that there are about 650 nulls (${olympians.filter(d => d.weight === null).length}, to be exact) — almost as many as there are at the peak of the non-null distribution. But histograms’ bin size is arbitrary; see how it’d feel to change it:","pinned":false,"hidden":false,"mode":"md"},{"id":26,"value":"viewof binSize = Inputs.range([1, 20], {label: \"Bin size\", step: 1, value: 2})","pinned":false,"hidden":false,"mode":"ojs"},{"id":106,"value":"No matter the bin size, the null bar will always go up to ${olympians.filter(d => d.weight === null).length} on the _y_-axis. But the subjective impression of the prevalence of nulls varies wildly. With narrow bins, it feels like a lot of nulls; with wide bins, it feels like fewer. I think there are two reasons: (1) we compare the null bar to the peak of the distribution, which is a function of the arbitrary bin size instead of a property of the data; (2) we compare the _areas_ of the bars, which we haven’t made commensurable.\n\nSo we can make the width of the null bar match the width of the bin bars:","pinned":false,"hidden":false,"mode":"md"},{"id":148,"value":"$2d","pinned":false,"hidden":false,"mode":"ojs"},{"id":238,"value":"Inputs.bind(Inputs.range([1, 20], {label: \"Bin size\", step: 1, value: 2}), viewof binSize)","pinned":false,"hidden":false,"mode":"ojs"},{"id":383,"value":"That feels better. It now passes an important gut check: if we had one bin of 50 non-null rows, and 50 nulls, then the two bars would be of equal width and height, and thus equal area, and I would subjectively feel them to be of equal weight, and I would correctly intuit that half the data was null.\n\nBut it still feels weird! I think the main thing I want to know is _what fraction of the data is null_, which entails mentally summing up the non-null bars and comparing the null bar. It feels like I’m being subjected to a Piaget conservation task. The null fraction doesn’t depend on bin size, and the result of the mental calculation doesn’t change, but the constituent parts do.\n\nThis would be especially problematic if you had to compare multiple histograms with different binning. Here’s height instead of weight:","pinned":false,"hidden":false,"mode":"md"},{"id":361,"value":"$2e","pinned":false,"hidden":false,"mode":"ojs"},{"id":187,"value":"$2f","pinned":false,"hidden":false,"mode":"md"},{"id":257,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":274,"value":"Inputs.bind(Inputs.range([1, 20], {label: \"Bin size\", step: 1, value: 2}), viewof binSize)","pinned":false,"hidden":false,"mode":"ojs"},{"id":255,"value":"$31","pinned":false,"hidden":false,"mode":"md"},{"id":326,"value":"variable = {\n const weights = olympians.map(d => d.weight).filter(d => d !== null).sort(d3.ascending);\n const nBins = 4;\n const itemsPerBin = weights.length / nBins;\n const bins = [];\n for (let i = 0; i < weights.length; i += itemsPerBin) {\n const chunk = weights.slice(i, i + itemsPerBin);\n bins.push(Object.assign(chunk, {x0: chunk[0], x1: chunk.at(-1)}));\n }\n return Plot.plot({\n x: {label: \"weight\"},\n y: {label: \"density\"},\n marks: [\n Plot.rectY(bins, {x1: \"x0\", x2: \"x1\", y: d => d.length / (d.x1 - d.x0), insetRight: 1}),\n Plot.ruleY([0])\n ]\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":418,"value":"$32","pinned":false,"hidden":false,"mode":"md"},{"id":445,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":477,"value":"But, again, interpreting that is not easy. And it feels deceptive to draw this on the same Cartesian plane when it’s flouting the meaning the of axes and only respecting area.\n\nMy conclusion is that, even though numeric values and null values make up slices of the same whole pie, the histogram metaphor can only show the distribution of a binned continuous numeric variable. Even though a histogram _looks like_ a bar chart with discrete ordinal values, the histogram cannot accommodate discrete ordinal values. We’d be better off not trying to mix these representations.\n\nIf you still want to show null values, you could show them like you could show any incommensurable-but-related thing: by a second chart, with its own coordinate plane, its own encoding scope, its own axes. Juxtaposition is very forgiving; it makes no demands about common units! We can just put things next to each other.","pinned":false,"hidden":false,"mode":"md"},{"id":480,"value":"{\n const combinedWidth = Math.min(640, width);\n const nullPlotWidth = 32;\n const flexGap = 2;\n\n const plot1 = Plot.plot({\n width: combinedWidth - (nullPlotWidth + flexGap),\n marks: [\n Plot.rectY(olympians, Plot.binX({y: \"count\"}, {x: \"weight\", thresholds})),\n Plot.ruleY([0])\n ]\n });\n \n const plot2 = Plot.plot({\n y: {axis: false, reverse: true},\n marginLeft: 8,\n marginRight: 8,\n insetTop: 20,\n insetBottom: 30,\n width: nullPlotWidth,\n marks: [\n Plot.barY(olympians, Plot.groupZ({y: \"count\"}, {fill: d => d.weight === null})),\n Plot.text([null], {frameAnchor: \"bottom\", text: () => \"NULL\", dy: -13})\n ]\n });\n \n return htl.html`

\n ${plot1}\n ${plot2}\n

`\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":569,"value":"$34","pinned":false,"hidden":false,"mode":"md"},{"id":185,"value":"---","pinned":false,"hidden":false,"mode":"md"},{"id":20,"value":"thresholds = d3.range(...d3.extent(olympians, d => d.weight), binSize)","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}