31:T7a9,csv = { const EOL = Symbol("EOL"); const EOF = Symbol("EOF"); const QUOTE = 34; const NEWLINE = 10; const RETURN = 13; function dsv(DELIMITER) { return function(text) { const columns = []; let N = (text = text + "").length; let I = 0; // current character index let n = 0; // current line number let k = 0; // current column number let t; // current token let eof = N <= 0; // current token followed by EOF? let eol = false; // current token followed by EOL? // Strip the trailing newline. if (text.charCodeAt(N - 1) === NEWLINE) --N; if (text.charCodeAt(N - 1) === RETURN) --N; function token() { if (eof) return EOF; if (eol) return eol = false, EOL; // Unescape quotes. var i, j = I, c; if (text.charCodeAt(j) === QUOTE) { while (I++ < N && text.charCodeAt(I) !== QUOTE || text.charCodeAt(++I) === QUOTE); if ((i = I) >= N) eof = true; else if ((c = text.charCodeAt(I++)) === NEWLINE) eol = true; else if (c === RETURN) { eol = true; if (text.charCodeAt(I) === NEWLINE) ++I; } return text.slice(j + 1, i - 1).replace(/""/g, "\""); } // Find next delimiter or newline. while (I < N) { if ((c = text.charCodeAt(i = I++)) === NEWLINE) eol = true; else if (c === RETURN) { eol = true; if (text.charCodeAt(I) === NEWLINE) ++I; } else if (c !== DELIMITER) continue; return text.slice(j, i) || undefined; } // Return last token before EOF. return eof = true, text.slice(j, N) || undefined; } while ((t = token()) !== EOF) { while (t !== EOL && t !== EOF) { (columns[k] = (columns[k] || new Array(n)))[n] = t; t = token(); ++k; } k = 0; ++n; } return columns; }; } return dsv(",".charCodeAt(0)); }32:[] 2f:{"title":"Manipulating Flat Arrays","cells":[{"id":0,"value":"md`# Manipulating Flat Arrays\n\nAnalyzing data in JavaScript often involves arrays of objects.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1298,"value":"[\n {date: new Date(2012, 0, 1), precipitation: 0},\n {date: new Date(2012, 0, 2), precipitation: 10.9},\n {date: new Date(2012, 0, 3), precipitation: 0.8},\n {date: new Date(2012, 0, 4), precipitation: 20.3}\n]","pinned":true,"hidden":false,"mode":"ojs"},{"id":1296,"value":"md`Arrays of objects are convenient: they’re similar to rows of a database table and work well with [Array methods](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array) and libraries such as [lodash](https://lodash.com/) and [d3-array](https://github.com/d3/d3-array).\n\nUnfortunately, objects also tend to be slow. They incur significant overhead compared to column-oriented (“flat”) [typed arrays](https://developer.mozilla.org/docs/Web/JavaScript/Typed_arrays). Modern JavaScript engines are surprisingly clever at optimizing object usage, but with larger data, you’ll probably want to switch to flat arrays.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1358,"value":"({\n date: Float64Array.of(1325404800000, 1325491200000, 1325577600000, 1325664000000),\n precipitation: Float64Array.of(0, 10.9, 0.8, 20.3)\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":1356,"value":"md`Flat arrays can be challenging. This notebook explains some common techniques. In the near future, columnar data structure libraries such as [Apache Arrow](https://arrow.apache.org/docs/js/) will likely grow in popularity. The techniques described here will help you to manipulate data efficiently and to use these libraries more effectively.\n\nColumnar data use separate arrays to represent each column, with the critical property that the arrays are *parallel*: each array has the same number of elements in the same order. For example, in a weather dataset, we might have one column (one array) representing the dates, and another column (another array) representing the precipitation amount on each date. Here is the array of dates:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":358,"value":"date","pinned":true,"hidden":false,"mode":"ojs"},{"id":363,"value":"md`And here’s the parallel array of precipitation in millimeters:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":354,"value":"precipitation","pinned":true,"hidden":false,"mode":"ojs"},{"id":374,"value":"md`Because the *date* and *precipitation* arrays are parallel, for each *i* in 0 … *length* - 1, the precipitation on *date*[*i*] is *precipitation*[*i*]. For example, on ${data.date[1].toLocaleDateString()}, it rained ${data.precipitation[1]} millimeters.\n\n---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1402,"value":"md`## Conversion\n\nIt’s most efficient if your data is loaded directly into a columnar representation (preferrably from a binary file format, such as an [Arrow file](https://github.com/apache/arrow)). But you can also easily convert between column-oriented arrays and row-oriented objects.\n\nFor example, to convert columns to rows using [Array.from](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/from):`","pinned":false,"hidden":false,"mode":"ojs"},{"id":379,"value":"objects = Array.from({length}, (_, i) => ({\n date: date[i],\n precipitation: precipitation[i]\n}))","pinned":true,"hidden":false,"mode":"ojs"},{"id":394,"value":"md`Likewise, to convert from rows to columns:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1416,"value":"({\n date: objects.map(d => d.date),\n precipitation: Float64Array.from(objects, d => d.precipitation)\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":1268,"value":"md`## Take\n\nThe first rule of working with parallel arrays is to maintain parallelism. For example, if we filter the *date* array to retain only the Mondays, we must filter the *precipitation* array in exactly the same way; if we retain *date*[*i*] in the filtered array, we must also retain *precipitation*[*i*] in its parallel filtered array. If we don’t, parallelism is broken and our filtered data is meaningless!\n\nA useful primititive operation on arrays, then, is to construct a new array by arranging the desired elements into the desired order. Given an array of *values* and an array of *index*es, the *take* function returns a new array (of the same type), containing *values*[*i*] for each *i* in *index*.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1199,"value":"function take(values, index) {\n return values.constructor.from(index, i => values[i]);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":1441,"value":"md`For example, to extract the first three dates:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1205,"value":"take(date, [0, 1, 2])","pinned":true,"hidden":false,"mode":"ojs"},{"id":1443,"value":"md`And the corresponding precipitation amounts:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1284,"value":"take(precipitation, [0, 1, 2])","pinned":true,"hidden":false,"mode":"ojs"},{"id":1445,"value":"md`Note that you can repeat an index if desired:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1450,"value":"take(date, [2, 1, 2, 1])","pinned":true,"hidden":false,"mode":"ojs"},{"id":1514,"value":"md`We can write a small variation of *take* that allows it to work well with *array*.map, allowing us to take parallel arrays simultaneously with a given *index*. The *taker* function returns a *take* function with a baked-in *index*.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1501,"value":"function taker(index) {\n return values => values.constructor.from(index, i => values[i]);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":1505,"value":"[date, precipitation].map(taker([0, 1, 2]))","pinned":true,"hidden":false,"mode":"ojs"},{"id":1516,"value":"md`Similarly, we can write an *get* function that returns a function that when passed an array returns the element at index *i*.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1508,"value":"function get(i) {\n return values => values[i];\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":1511,"value":"[date, precipitation].map(get(0))","pinned":true,"hidden":false,"mode":"ojs"},{"id":1464,"value":"md`When taking arrays, we often want to start with the current order and then modify it in some way, say by filtering or sorting it. This *range* function can be used to generate an array [0, 1, 2, …, *array*.length - 1]. If you were to pass this array to *take*, you’d get back an identical copy of the input *array*.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1460,"value":"function range(start, stop, step) {\n const n = arguments.length;\n start = +start, stop = +stop, step = n < 2 ? (stop = start, start = 0, 1) : n < 3 ? 1 : +step;\n return Uint32Array.from({length: Math.ceil((stop - start) / step)}, (_, i) => start + i * step);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":1465,"value":"range(length)","pinned":true,"hidden":false,"mode":"ojs"},{"id":412,"value":"md`## Filter\n\nNow say we want to extract all of the days where it rained more than fifty millimeters (2 inches). 🌧\n\nWith array-of-objects, we could use [*array*.filter](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/filter):`","pinned":false,"hidden":false,"mode":"ojs"},{"id":434,"value":"objects.filter(d => d.precipitation > 50)","pinned":true,"hidden":false,"mode":"ojs"},{"id":440,"value":"md`We can do the equivalent by filtering the precipitation array, but this will only tell us the matching precipitation values—not the dates on which the rain occurred.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":444,"value":"precipitation.filter(d => d > 50)","pinned":true,"hidden":false,"mode":"ojs"},{"id":448,"value":"md`Instead, let’s take! First apply our filter to construct a sparse index of all *i* in 0 … *length* - 1 for which *precipitation*[*i*] is greater than 50 mm; then use this index to take the *date* array.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1276,"value":"take(date, range(length).filter(i => precipitation[i] > 50))","pinned":true,"hidden":false,"mode":"ojs"},{"id":1550,"value":"md`Since filtering arrays will be a common task, we can make a helper *filter* function that computes the sparse index by applying a *test* function to an array of *values*. We can also use a for loop that is faster than *array*.filter.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1528,"value":"function filter(values, test) {\n const I = [], n = values.length;\n for (let i = 0; i < n; ++i) {\n if (test(values[i], i, values)) {\n I.push(i);\n }\n }\n return I;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2360,"value":"md`We’ll also want a variation of *filter* that takes an existing *index*. This allows us to filter another sparse index, say to intersect two filters.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2014,"value":"function filteri(values, index, test) {\n const I = [], n = index.length;\n for (let i = 0; i < n; ++i) {\n const j = index[i];\n if (test(values[j], j, values)) {\n I.push(j);\n }\n }\n return I;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2362,"value":"md`Now let’s filter the *date* array to find the rainy days:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1531,"value":"take(date, filter(precipitation, d => d > 50))","pinned":true,"hidden":false,"mode":"ojs"},{"id":2364,"value":"md`We can filter both *date* and *precipitation* to see the precipitation amounts:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1553,"value":"[date, precipitation].map(taker(filter(precipitation, d => d > 50)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":426,"value":"md`Similarly, we can filter by date, say to see the rainfall on Christmas day:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1559,"value":"[date, precipitation].map(taker(filter(date, d => d.getMonth() === 11 && d.getDate() === 25)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":473,"value":"md`Here’s the equivalent code for array-of-objects:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":471,"value":"objects.filter(d => d.date.getMonth() === 11 && d.date.getDate() === 25)","pinned":true,"hidden":false,"mode":"ojs"},{"id":477,"value":"md`## Map\n\nUnlike filtering, mapping an array doesn’t change the number of values: for each input value, there’s a corresponding output value. For example, we can convert millimeters to inches using [*array*.map](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/map):`","pinned":false,"hidden":false,"mode":"ojs"},{"id":480,"value":"precipitation.map(d => d * 0.0393701)","pinned":true,"hidden":false,"mode":"ojs"},{"id":484,"value":"md`Note that because the original array was a Float64Array, the mapped array is, too! If you want to change the array type, you can use [Array.from](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/from) instead. For example:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":488,"value":"Uint16Array.from(precipitation, d => d * 0.0393701)","pinned":true,"hidden":false,"mode":"ojs"},{"id":492,"value":"md`Mapping is less efficient with the array-of-objects representation: you must create a new object for each value *and* you must copy over the unmapped fields. You could modify the object in-place, but defining a new field can also be slow, and mutability makes your code harder to understand.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":495,"value":"objects.map(d => ({\n date: d.date,\n precipition: d.precipitation * 0.0393701\n}))","pinned":true,"hidden":false,"mode":"ojs"},{"id":1976,"value":"md`## Slice\n\n[Slicing](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/slice) is a bit like filtering, except you’re extracting a continguous portion of the input array. For example, this might be used to extract the last ten elements.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1978,"value":"function slice(...options) {\n return array => array.slice(...options);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":1981,"value":"[date, precipitation].map(slice(-10))","pinned":true,"hidden":false,"mode":"ojs"},{"id":1998,"value":"md`## Reverse\nYou might want to reverse an array. Unlike [*array*.reverse](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/reverse), this returns a copy of the input array rather than mutating it in-place.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2002,"value":"function reverse(array) {\n return array.slice().reverse();\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2006,"value":"[date, precipitation].map(slice(-10)).map(reverse)","pinned":true,"hidden":false,"mode":"ojs"},{"id":1803,"value":"md`## Sort\n\nSorting reorders the parallel arrays. First the sorted index is constructed, and then a sorted array is taken from each parallel array. Again, unlike [*array*.sort](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/sort), this returns a copy of the input array rather than mutating it in-place.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1805,"value":"function sort(values, order = ascending) {\n return range(values.length).sort((i, j) => order(values[i], values[j]));\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2021,"value":"function sorti(values, index, order = ascending) {\n return index.slice().sort((i, j) => order(values[i], values[j]));\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2443,"value":"function ascending(a, b) {\n return a < b ? -1 : a > b ? 1 : a >= b ? 0 : NaN;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2446,"value":"function descending(a, b) {\n return b < a ? -1 : b > a ? 1 : b >= a ? 0 : NaN;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2452,"value":"md`For example, we can sort by descending precipitation to find the rainiest days.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1808,"value":"[date, precipitation].map(taker(sort(precipitation, descending).slice(0, 10)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":503,"value":"md`## Sum\n\nNow say we want to compute the total rainfall. One way to do that is [*array*.reduce](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array/reduce): *reducing* is the process of accumulating many values into a single value, such as by summing them. In addition to summing, we can ignore any missing data in our dataset by treating \\`NaN\\` as zero. With array-of-objects, we’d say:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":510,"value":"objects.reduce((p, d) => (d = +d.precipitation) ? p + d : p, 0)","pinned":true,"hidden":false,"mode":"ojs"},{"id":537,"value":"md`With flat arrays, the code is slightly shorter (and faster):`","pinned":false,"hidden":false,"mode":"ojs"},{"id":505,"value":"precipitation.reduce((p, d) => (d = +d) ? p + d : p, 0)","pinned":true,"hidden":false,"mode":"ojs"},{"id":543,"value":"md`Notice that with flat arrays, the function passed to *array*.reduce doesn’t reference precipitation: it’s entirely generic! This means that we can write a generic function to compute the sum for any array of numbers. Even better, flat arrays allow us to sum more efficiently than *array*.reduce by using a simple for loop:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":517,"value":"function sum(values) {\n let sum = 0;\n for (let i = 0, n = values.length; i < n; ++i) {\n const v = +values[i];\n if (v) sum += v;\n }\n return sum;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2458,"value":"md`As before, we also want a variant of *sum* that takes a sparse index:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2028,"value":"function sumi(values, index) {\n let sum = 0;\n for (let i = 0, n = index.length; i < n; ++i) {\n const v = +values[index[i]];\n if (v) sum += v;\n }\n return sum;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2460,"value":"md`${(sum(precipitation) * 0.0393701).toFixed(0)} inches of rain over four years. Sounds about right.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":596,"value":"sum(precipitation)","pinned":true,"hidden":false,"mode":"ojs"},{"id":603,"value":"md`Of course, we could use our generic *sum* with the array-of-objects representation too, by converting to a flat array first. But that’s cheating: we’d be using both flat arrays and array-of-objects!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":600,"value":"sum(objects.map(d => d.precipitation))","pinned":true,"hidden":false,"mode":"ojs"},{"id":760,"value":"md`## Count\n\nIn addition to summing, we can also count the number of valid values in our dataset. It’s tempting to use *array*.length, but note that this will include missing values (\\`NaN\\`) which we want to ignore.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":773,"value":"precipitation.length","pinned":true,"hidden":false,"mode":"ojs"},{"id":793,"value":"md`We could again use *array*.reduce, but since the for loop is faster, let’s skip ahead.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":764,"value":"function count(values) {\n let count = 0;\n for (let i = 0, n = values.length; i < n; ++i) {\n const v = +values[i];\n if (v === v) ++count;\n }\n return count;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2038,"value":"function counti(values, index) {\n let count = 0;\n for (let i = 0, n = index.length; i < n; ++i) {\n const v = +values[index[i]];\n if (v === v) ++count;\n }\n return count;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2486,"value":"md`As it turns out, this dataset isn’t missing any values. Good job, [NOAA](https://www.ncdc.noaa.gov/cdo-web/)!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":769,"value":"count(precipitation)","pinned":true,"hidden":false,"mode":"ojs"},{"id":1897,"value":"[date, precipitation].map(count)","pinned":true,"hidden":false,"mode":"ojs"},{"id":780,"value":"md`## Mean\n\nWith sum and count, we can compute the [arithmetic mean](https://en.wikipedia.org/wiki/Arithmetic_mean), telling us the daily average rainfall in the dataset (a number which is not particularly useful given that many days have no rain at all):`","pinned":false,"hidden":false,"mode":"ojs"},{"id":782,"value":"function mean(values) {\n return sum(values) / count(values);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2042,"value":"function meani(values, index) {\n return sumi(values, index) / counti(values, index);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":787,"value":"mean(precipitation)","pinned":true,"hidden":false,"mode":"ojs"},{"id":815,"value":"md`Perhaps a more interesting statistic is the proportion of days that saw any rain:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":801,"value":"mean(precipitation.map(d => d > 0))","pinned":true,"hidden":false,"mode":"ojs"},{"id":827,"value":"md`A ${(mean(data.precipitation.map(d => d > 0)) * 100).toFixed(0)}% probability of rain on any given day! That’s quite high. If you haven’t guessed yet, this data is from Seattle. Per [Wikipedia](https://en.wikipedia.org/wiki/Seattle): “Despite having a reputation for frequent rain, Seattle receives less precipitation than many other U.S. cities like Chicago or New York City. However, unlike many other U.S. cities, Seattle has many more ‘rain days’, when a very light drizzle falls from the sky for many days.”\n\nWe’ll look at annual rainfall when we discuss grouping below.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1900,"value":"[temp_min, temp_max].map(mean)","pinned":true,"hidden":false,"mode":"ojs"},{"id":614,"value":"md`## Min & Max\n\nNow let’s say we want to know the date in our dataset with the maximum rainfall. We can use *array*.reduce again for this, as with array-of-objects:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":650,"value":"objects.reduce((p, d) => d.precipitation > p.precipitation ? d : p)","pinned":true,"hidden":false,"mode":"ojs"},{"id":677,"value":"md`We can similarly use *array*.reduce with flat arrays:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":661,"value":"precipitation.reduce((p, d) => d > p ? d : p)","pinned":true,"hidden":false,"mode":"ojs"},{"id":679,"value":"md`Yet again this approach will tell us the maximum rainfall but *not* the date on which this rainfall occurred. So instead, let’s compute the index of the maximum value (the precipitation) and use that to extract the corresponding date.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1490,"value":"function max(values) {\n let maxIndex, maxValue = -Infinity;\n for (let i = 0, n = values.length; i < n; ++i) {\n const v = +values[i];\n if (v > maxValue) maxIndex = i, maxValue = v;\n }\n return maxIndex;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2046,"value":"function maxi(values, index) {\n let maxIndex, maxValue = -Infinity;\n for (let i = 0, n = index.length; i < n; ++i) {\n const v = +values[index[i]];\n if (v > maxValue) maxIndex = index[i], maxValue = v;\n }\n return maxIndex;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":1616,"value":"[date, precipitation].map(get(max(precipitation)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":692,"value":"md`Naturally we can similarly compute the minimum. The minimum rainfall is zero on 838 days in this dataset, so to make this example more interesting, we’ll use another parallel array of low temperatures in Celsius.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":700,"value":"precipitation.reduce((p, d) => p + (d === 0), 0)","pinned":true,"hidden":false,"mode":"ojs"},{"id":635,"value":"function min(values) {\n let minIndex, minValue = Infinity;\n for (let i = 0, n = values.length; i < n; ++i) {\n const v = +values[i];\n if (v < minValue) minIndex = i, minValue = v;\n }\n return minIndex;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2062,"value":"function mini(values, index) {\n let minIndex, minValue = Infinity;\n for (let i = 0, n = index.length; i < n; ++i) {\n const v = +values[index[i]];\n if (v < minValue) minIndex = index[i], minValue = v;\n }\n return minIndex;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":623,"value":"[date, temp_min].map(get(min(temp_min)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":711,"value":"md`Brr, that’s ${(-7.1 * 9 / 5 + 32).toFixed(0)}°F. ❄️ And the maximum?`","pinned":false,"hidden":false,"mode":"ojs"},{"id":708,"value":"[date, temp_max].map(get(max(temp_max)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":722,"value":"md`A toasty ${(35.6 * 9 / 5 + 32).toFixed(0)}°F in August! 🌞`","pinned":false,"hidden":false,"mode":"ojs"},{"id":844,"value":"md`## Quantile\n\n[Quantiles](https://en.wikipedia.org/wiki/Quantile) are a generalization of minimum and maximum. Given a number *p* in [0, 1], a proportion *p* of values are greater than or equal to the *p*-quantile. Here is an implementation of the [R-7 method](https://en.wikipedia.org/wiki/Quantile#Quantiles_of_a_population) adapted from [d3-array](https://github.com/d3/d3-array/blob/master/README.md#quantile):`","pinned":false,"hidden":false,"mode":"ojs"},{"id":846,"value":"function quantile(values, p) {\n values = Float64Array.from(values).filter(number).sort(ascending);\n const n = values.length;\n if (!n) return NaN;\n if (!((p = +p) > 0) || n < 2) return values[0];\n if (p >= 1) return values[n - 1];\n const i = (n - 1) * p;\n const i0 = Math.floor(i);\n const v0 = values[i0];\n const v1 = values[i0 + 1];\n return v0 + (v1 - v0) * (i - i0);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2066,"value":"function quantilei(values, index, p) {\n return quantile(take(values, index), p);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":979,"value":"md`To compute the third quartile of precipitation, *p* = 0.75:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":851,"value":"quantile(precipitation, 0.75)","pinned":true,"hidden":false,"mode":"ojs"},{"id":899,"value":"md`So, 1 in 4 days in this dataset saw at least 2.8 mm (${(2.8 * 0.0393701).toFixed(1)} in) of rain. Similarly the median is *p* = 0.5:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":856,"value":"function median(values) {\n return quantile(values, 0.5);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2070,"value":"function mediani(values, index) {\n return quantilei(values, index, 0.5);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":860,"value":"median(precipitation)","pinned":true,"hidden":false,"mode":"ojs"},{"id":984,"value":"md`This median of zero is expected; only 43% of days saw any rain.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":865,"value":"md`## Top & Bottom\n\nSometimes we want to look at *multiple* extreme values—not just the minimum or maximum. [Quickselect](https://en.wikipedia.org/wiki/Quickselect) is a good [selection algorithm](https://en.wikipedia.org/wiki/Selection_algorithm) for this purpose, but for the purposes of this notebook, let’s take the simpler approach of sorting the entire dataset and extracting the first few values.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":867,"value":"function top(values, k = 1, order = ascending) {\n return range(values.length)\n .sort((i, j) => order(values[j], values[i]))\n .slice(0, (k = +k) > 0 ? k : 0);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2074,"value":"function topi(values, index, k = 1, order = ascending) {\n return index.slice()\n .sort((i, j) => order(values[j], values[i]))\n .slice(0, (k = +k) > 0 ? k : 0);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2521,"value":"md`The five rainiest days:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":877,"value":"[date, precipitation].map(taker(top(precipitation, 5)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":970,"value":"md`Similarly for the bottom values:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":872,"value":"function bottom(values, k = 1, order = ascending) {\n return range(values.length)\n .sort((i, j) => order(values[i], values[j]))\n .slice(0, (k = +k) > 0 ? k : 0);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2078,"value":"function bottomi(values, index, k = 1, order = ascending) {\n return index.slice()\n .sort((i, j) => order(values[i], values[j]))\n .slice(0, (k = +k) > 0 ? k : 0);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2524,"value":"md`The five coldest days:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":882,"value":"[date, temp_min].map(taker(bottom(temp_min, 5)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":1707,"value":"md`## Group\n\nGrouping is another common operation for data analysis. For example, we might wish to group the data by year to compute the annual rainfall. To group data, we’ll use the concept of a *key*: if two values share a *key*, then we put those values in the same group. This works well with [Map](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map).\n\nThe *group* function also takes an optional *value* function; if specified, the *value* function is used to map each group’s index to the desired value.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1139,"value":"function group(keys, value) {\n const map = new Map;\n for (let i = 0, n = keys.length; i < n; ++i) {\n let k = keys[i], v = map.get(k);\n if (v === undefined) map.set(k, [i]);\n else v.push(i);\n }\n if (value !== undefined) {\n for (const [key, index] of map) {\n map.set(key, value(index));\n }\n }\n return map;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2535,"value":"md`First, with no *value* function, showing the indexes for each year. This data is sorted chronologically, so no surprises here.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2199,"value":"group(date.map(d => d.getFullYear()))","pinned":true,"hidden":false,"mode":"ojs"},{"id":2538,"value":"md`Now to compute the rainfall per year:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1142,"value":"group(date.map(d => d.getFullYear()), i => sumi(precipitation, i))","pinned":true,"hidden":false,"mode":"ojs"},{"id":2540,"value":"md`Now the average rainfall per month of year, showing the May–September dry period:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2208,"value":"group(date.map(d => d.getMonth()), i => meani(precipitation, i))","pinned":true,"hidden":false,"mode":"ojs"},{"id":80,"value":"md`Similarly, the median rainfall per month of year:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2099,"value":"group(date.map(d => d.getMonth()), i => mediani(precipitation, i))","pinned":true,"hidden":false,"mode":"ojs"},{"id":66,"value":"md`We can likewise count the valid values, but it’s not particularly interesting since we already know that this dataset isn’t missing any data.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1159,"value":"group(date.map(d => d.getFullYear()), i => counti(precipitation, i))","pinned":true,"hidden":false,"mode":"ojs"},{"id":70,"value":"md`The coldest day for each year:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1173,"value":"group(date.map(d => d.getFullYear()), i => [date, temp_min].map(get(mini(temp_min, i))))","pinned":true,"hidden":false,"mode":"ojs"},{"id":2565,"value":"md`The rainiest day for each year:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":45,"value":"group(date.map(d => d.getFullYear()), i => [date, precipitation].map(get(maxi(precipitation, i))))","pinned":true,"hidden":false,"mode":"ojs"},{"id":248,"value":"md`The five rainest days for each year:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1060,"value":"group(date.map(d => d.getFullYear()), i => take(date, topi(precipitation, i, 5)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":2588,"value":"md`The five coldest days for each year:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":306,"value":"group(date.map(d => d.getFullYear()), i => take(date, bottomi(temp_min, i, 5)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":2590,"value":"md`---\n\nWhew! That was a whirlwind tour of techniques for working with parallel arrays. I’ll probably package these functions up into a reusable library and publish it on npm. In the meantime, you can [import the code](/@mbostock/introduction-to-imports) here directly into your notebooks.\n\nThanks for reading!`","pinned":false,"hidden":false,"mode":"ojs"},{"id":99,"value":"md`---\n\n## Appendix`","pinned":false,"hidden":false,"mode":"ojs"},{"id":325,"value":"identity = x => x","pinned":true,"hidden":false,"mode":"ojs"},{"id":838,"value":"number = v => !isNaN(v)","pinned":true,"hidden":false,"mode":"ojs"},{"id":1962,"value":"length = date.length","pinned":true,"hidden":false,"mode":"ojs"},{"id":1576,"value":"date = data.date","pinned":true,"hidden":false,"mode":"ojs"},{"id":1578,"value":"precipitation = data.precipitation","pinned":true,"hidden":false,"mode":"ojs"},{"id":1580,"value":"temp_max = data.temp_max","pinned":true,"hidden":false,"mode":"ojs"},{"id":1584,"value":"temp_min = data.temp_min","pinned":true,"hidden":false,"mode":"ojs"},{"id":1587,"value":"wind = data.wind","pinned":true,"hidden":false,"mode":"ojs"},{"id":3,"value":"data = fetch(\"https://raw.githubusercontent.com/vega/vega-datasets/530b2f02ebcac4539d292d97da0b57f4d1fcf177/data/seattle-weather.csv\")\n .then(response => response.text())\n .then(csv)\n .then(([date, precipitation, temp_max, temp_min, wind, weather]) => {\n const length = date.length - 1;\n return {\n date: Array.from({length}, (_, i) => new Date(date[i + 1])),\n precipitation: Float64Array.from({length}, (_, i) => +precipitation[i + 1]),\n temp_max: Float64Array.from({length}, (_, i) => +temp_max[i + 1]),\n temp_min: Float64Array.from({length}, (_, i) => +temp_min[i + 1]),\n wind: Float64Array.from({length}, (_, i) => +wind[i + 1])\n };\n })","pinned":true,"hidden":false,"mode":"ojs"},{"id":6,"value":"$31","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}