31:T414,correlationHeatMap = () => { // Concatenate correlationAnalysis to itself, switching metric_1/metric_2 for the heatmap. const concatenation = correlationAnalysis().concat( correlationAnalysis().derive({ metric_2: ({ metric_1 }) => metric_1, metric_1: ({ metric_2 }) => metric_2 }) ); const lookup = vl.lookupData(stockCompany).key("stock").fields(["company"]); return vl .markBar() .title({ text: `DJIA 2020 Correlation Analysis`, subtitle: correlations.subtitle }) .data(concatenation) .transform( vl.lookup("metric_1").from(lookup).as("Stock 1"), vl.lookup("metric_2").from(lookup).as("Stock 2") ) .encode( vl.y().fieldN("metric_1").title("DJIA Stock"), vl.x().fieldN("metric_2").title("DJIA Stock"), vl.color().fieldQ("Correlation").scale({ scheme: colorScheme }), vl.tooltip(["Stock 1", "Stock 2", { field: "Correlation", format: "r" }]), vl.opacity().if(correlations.isolation, vl.value(1)).value(0.09) ) .render(); }32:T511,correlationHeatMapNewData = (data, title) => { // Concatenate correlationAnalysis to itself, switching metric_1/metric_2 for the heatmap. const concatenation = correlationAnalysis(data).concat( correlationAnalysis(data).derive({ metric_2: ({ metric_1 }) => metric_1, metric_1: ({ metric_2 }) => metric_2 }) ); // I removed the lookup table for this generic heatmap function // const lookup = vl.lookupData(stockCompany).key("stock").fields(["company"]); return ( vl .markBar() .title({ text: title // subtitle: correlations.subtitle }) .data(concatenation) // .transform( // vl.lookup("metric_1").from(lookup).as("Stock 1"), // vl.lookup("metric_2").from(lookup).as("Stock 2") // ) .encode( vl.y().fieldN("metric_1").title(""), vl.x().fieldN("metric_2").title(""), vl .color() .fieldQ("Correlation") .scale({ scheme: colorScheme, domain: [-1, 1] }), // I (jurb) opted for a set domain of -1 to 1 here vl.tooltip([ "metric_1", "metric_2", { field: "Correlation", format: "r" } ]) // vl.opacity().if(correlations.isolation, vl.value(1)).value(0.09) ) .render() ); }33:T402,md`[*Introducing Arquero*](https://observablehq.com/@uwdata/introducing-arquero) demonstrates the use of the [*op.corr*](https://uwdata.github.io/arquero/api/op#corr) aggregate function to determine the [product-moment correlation](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient) between two variables in a table. The notebook's example shows the correlation operation being invoked manually for two variables:

...
beers
  .rollup({
   ... 
    corr_abv_ibu: d => op.corr(d.abv, d.ibu),
   ...
  })
  .view()
...

This notebook implements a general, reusable pattern to calculate correlations for all measures in a dataset. To illustrate the pattern, the 30 stocks in the [Dow Jones Industrial Average (DJIA)](https://en.wikipedia.org/wiki/Dow_Jones_Industrial_Average) will be analyzed to generate this notebook's [heatmap](https://en.wikipedia.org/wiki/Heat_map) with [Vega-Lite](https://observablehq.com/@uwdata/introduction-to-vega-lite).`34:T7bd,stockCompany= [ {stock: 'AAPL', company: 'Apple Inc.'}, {stock: 'AXP', company: 'The American Express Company'}, {stock: 'BA', company: 'The Boeing Company'}, {stock: 'CAT', company: 'Caterpillar Inc.'}, {stock: 'CSCO', company: 'Cisco Systems, Inc.'}, {stock: 'CVX', company: 'Chevron Corporation'}, {stock: 'DIS', company: 'The Walt Disney Company'}, {stock: 'DOW', company: 'DowDuPont Inc.'}, {stock: 'GS', company: 'The Goldman Sachs Group, Inc.'}, {stock: 'HD', company: 'The Home Depot Inc.'}, {stock: 'IBM', company: 'International Business Machines Corporation'}, {stock: 'INTC', company: 'Intel Corporation'}, {stock: 'JNJ', company: 'Johnson & Johnson'}, {stock: 'JPM', company: 'JPMorgan Chase & Co.'}, {stock: 'KO', company: 'The Coca-Cola Company'}, {stock: 'MCD', company: 'McDonald’s Corporation'}, {stock: 'MMM', company: 'The 3M Company'}, {stock: 'MRK', company: 'Merck & Company, Inc.'}, {stock: 'MSFT', company: 'Microsoft Corporation'}, {stock: 'NKE', company: 'Nike, Inc.'}, {stock: 'PFE', company: 'Pfizer Inc.'}, {stock: 'PG', company: 'Proctor & Gamble Co.'}, {stock: 'TRV', company: 'The Travelers Companies, Inc.'}, {stock: 'UNH', company: 'UnitedHealth Group, Inc.'}, {stock: 'UTX', company: 'United Technologies Corporation'}, {stock: 'V', company: 'Visa Inc.'}, {stock: 'VZ', company: 'Verizon Communications Inc.'}, {stock: 'WBA', company: 'Walgreens Boots Alliance, Inc.'}, {stock: 'WMT', company: 'Walmart Inc.'}, {stock: 'XOM', company: 'Exxon Mobil Corporation'} ];35:T697,// I (jurb) merged all the data cells into one function, see we can calculate the correlation analysis with other data in another notebook. correlationAnalysis = (dataArg = data) => { // this only keep columns with numbers (using Datalib as `dl`) const dataProfile = dl.summary(dataArg); const metrics = dataProfile .filter(({ type, field }) => type == "number") .map(({ field }) => field); // combinations of variables function combination2(set) { const pair = []; for (let i = 0; i < set.length; i++) { for (let j = i + 1; j < set.length; j++) { pair.push([set[i], set[j]]); } } return pair; } // function const correlationPairs = combination2(metrics).map( ([metric_1, metric_2]) => ({ metric_1, metric_2 }) ); // programatically construct arquero rollup expressions const rollupObj = {}; for (const { metric_1, metric_2 } of correlationPairs) { // https://uwdata.github.io/arquero/api/expressions // https://observablehq.com/@mariodelgadosr/arquero-correlation-speed-test rollupObj[ `${metric_1} : ${metric_2}` ] = `d => op.corr(d["${metric_1}"], d["${metric_2}"])`; } // for return ( aq .from(dataArg) .rollup(rollupObj) .fold(aq.all(), { as: ["Correlation Pair", "Correlation"] }) // https://uwdata.github.io/arquero/api/expressions#limitations .derive( { metric_1: `d => op.split(d["Correlation Pair"],"${pairSeperator}")[0]`, metric_2: `d => op.split(d["Correlation Pair"],"${pairSeperator}")[1]` }, { before: "Correlation Pair" } ) .orderby(aq.desc("Correlation")) ); }36:[{"id":"3ecced0eb8cdeb388a6a2b076a5f758a0da91d00ce4eca327e84a680d3690ac8b8ee3c5fe0e4b70d548f6debf522e71d9db57bfa7b0696f66374d159fa465f25","href":"https://static.observableusercontent.com/files/3ecced0eb8cdeb388a6a2b076a5f758a0da91d00ce4eca327e84a680d3690ac8b8ee3c5fe0e4b70d548f6debf522e71d9db57bfa7b0696f66374d159fa465f25","name":"DJIA2020.csv","type":"text/csv","lastModified":1627304330816.111,"size":51508}] 2f:{"title":"Dow Jones Industrial Average Correlation Analysis with Arquero","cells":[{"id":0,"value":"md`# Dow Jones Industrial Average Correlation Analysis with Arquero\n\nFork that turns \\`correlationAnalysis\\` into a function that accepts data, and a new \\`correlationHeatMapNewData\\` function that accepts that data, for portability.\n\n*Update 2021: the notebook this is forked from now supports importing the heatmap with your own data! Import with:*\n\n\\`import {correlationHeatMap as newCorrelationHeatMap} with {testDataFork as data} from \"@mariodelgadosr/dow-jones-industrial-average-correlation-analysis-with-ar\"\\`\n\nAnd just put \\`newCorrelationHeatmap\\` in a cell. To create another heatmap, import the heatmap again with a new name and with new data.\n\n \n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1332,"value":"","pinned":true,"hidden":false,"mode":"ojs"},{"id":1322,"value":"import {correlationHeatMap as newCorrelationHeatMap} with {testDataFork as data} from \"@mariodelgadosr/dow-jones-industrial-average-correlation-analysis-with-ar\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":1318,"value":"newCorrelationHeatMap","pinned":true,"hidden":false,"mode":"ojs"},{"id":773,"value":"html`${viewof correlations}${viewof colorScheme}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1136,"value":"correlationHeatMap()","pinned":true,"hidden":false,"mode":"ojs"},{"id":496,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":1296,"value":"correlationHeatMapNewData(testDataFork, \"Title here\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":1292,"value":"$32","pinned":true,"hidden":false,"mode":"ojs"},{"id":246,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":465,"value":"md `## Usage\n\nTo use *correlationAnalysis* against your data in the Observable environment:\n\n~~~js\nimport {correlationAnalysis} with {YourData as data} from \"@mariodelgadosr/dow-jones-industrial-average-correlation-analysis-with-ar\"\n~~~\n\n***YourData*** is an array of standard JavaScript objects relative to your notebook, where: \n\n* Each object is interpreted as a table row with object keys serving as column names; \n* Each row has at least two measures;\n* Eg: [{[dimension1: value...], measure1: value, measure2: value...}, ...]\n\nThen reference ***correlationAnalysis*** in an Observable Cell. \n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":378,"value":"md`## Data\nFor this notebook, *data* is an array of objects created with [*GoogleFinance*](https://support.google.com/docs/answer/3093281?hl=en) that contains DJIA stock symbols and their respective closing prices; for each trading day in year 2020:\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":6,"value":"data = FileAttachment(\"DJIA2020.csv\").csv({typed: true})","pinned":true,"hidden":false,"mode":"ojs"},{"id":610,"value":"md `*stockCompany* translates stock symbols to their respective company names.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":605,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":255,"value":"md`## Data Profile\n[*Datalib*](https://github.com/vega/datalib)'s [*summary*](https://github.com/vega/datalib/wiki/Statistics#dl_summary) function is used to create a data profile of the data to identify all numeric columns:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":10,"value":"dataProfile = { \n \n // https://github.com/vega/datalib\n const dl = await require('datalib');\n\n // https://github.com/vega/datalib/wiki/Statistics#dl_summary\n return dl.summary(data); \n \n} ","pinned":true,"hidden":false,"mode":"ojs"},{"id":264,"value":"md `## Correlation Pairs \n*metrics* is an array that filters out just the numeric columns in the data profile. These will be the variables analyzed for the correlations.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":19,"value":"// we don't need to filter out non-numeric columns for my use case\nmetrics = dataProfile\n .filter(({ type, field }) => type == \"number\")\n .map(({ field }) => field)","pinned":true,"hidden":false,"mode":"ojs"},{"id":29,"value":"md `The metrics are transformed into correlation pairs for the analysis; the [*2-combination*](https://en.wikipedia.org/wiki/Combination) of the stock symbols:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1063,"value":"correlationPairs = {\n function combination2(set) {\n const pair = [];\n for (let i = 0; i < set.length; i++) {\n for (let j = i + 1; j < set.length; j++) {\n pair.push([set[i], set[j]]);\n }\n }\n return pair;\n } // function\n\n return combination2(Object.keys(data[0]).slice(1)).map(\n ([metric_1, metric_2]) => ({\n metric_1,\n metric_2\n })\n );\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":285,"value":"md `## Programmatically Building the Arquero Rollups\nWith the correlation pairings determined, the *rollupObj* is dynamically constructed. This object will be passed to the [*rollup*](https://observablehq.com/@uwdata/an-illustrated-guide-to-arquero-verbs#rollup) method to calculate the correlations for each metric pair. `","pinned":false,"hidden":false,"mode":"ojs"},{"id":979,"value":"pairSeperator = ` : `;","pinned":false,"hidden":false,"mode":"ojs"},{"id":77,"value":"rollupObj = {\n\n const rollupObj = {};\n\n for (const { metric_1, metric_2 } of correlationPairs) {\n // https://uwdata.github.io/arquero/api/expressions\n // https://observablehq.com/@mariodelgadosr/arquero-correlation-speed-test\n rollupObj[\n `${metric_1}${pairSeperator}${metric_2}`\n ] = `d => op.corr(d[\"${metric_1}\"], d[\"${metric_2}\"])`;\n } // for\n\n return rollupObj;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":321,"value":"md`## Correlation Analysis\n*correlationAnalysis* is a reference to the original dataset:\n\n* Loaded into an Arquero table;\n* With correlations calculated for each of the metric pairs using the programmatically generated *rollupObj*. \n\nThe results are then used as the input to the [Vega-lite](https://observablehq.com/@uwdata/introduction-to-vega-lite) visualization. \n\nThe analysis reveals:\n\n${maxMin.view()}\n\n* ${analysisTextMax};\n* ${analysisTextMin}.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"import { aq, op } from '@uwdata/arquero'","pinned":false,"hidden":false,"mode":"ojs"},{"id":1245,"value":"correlationAnalysis()","pinned":true,"hidden":false,"mode":"ojs"},{"id":983,"value":"$35","pinned":true,"hidden":false,"mode":"ojs"},{"id":305,"value":"correlationAnalysis().view(correlationAnalysis().numRows())","pinned":false,"hidden":false,"mode":"ojs"},{"id":315,"value":"maxMin = correlationAnalysis()\n .derive({\n maxPair: op.first_value(\"Correlation Pair\"),\n minPair: op.last_value(\"Correlation Pair\")\n })\n .rollup({\n max: (d) => op.max(d.Correlation),\n min: (d) => op.min(d.Correlation),\n maxPair: op.max(\"maxPair\"),\n minPair: op.min(\"minPair\")\n })","pinned":false,"hidden":false,"mode":"ojs"},{"id":789,"value":"md`## Texts, Viewofs and Imports`","pinned":false,"hidden":false,"mode":"ojs"},{"id":705,"value":"analysisTextMax = \n`Max correlation of ${maxMin.get(\"max\",0).toFixed(6)} for ${maxMin.get(\"maxPair\",0)}`;","pinned":false,"hidden":false,"mode":"ojs"},{"id":708,"value":"analysisTextMin = \n`Min correlation of ${maxMin.get(\"min\",0).toFixed(6)} for ${maxMin.get(\"minPair\",0)}`;","pinned":false,"hidden":false,"mode":"ojs"},{"id":782,"value":"md`🢇 Click cells to expand code for ***correlations*** and ***colorScheme***`","pinned":false,"hidden":false,"mode":"ojs"},{"id":906,"value":"viewof correlations = {\n\n const minMaxPairs= maxMin.get(\"maxPair\",0)\n .split(pairSeperator)\n .concat(maxMin.get(\"minPair\",0).split(pairSeperator)); \n const maxMinFilter =`(datum.metric_1 == '${minMaxPairs[0]}' && datum.metric_2 == '${minMaxPairs[1]}') || \n (datum.metric_1 == '${minMaxPairs[2]}' && datum.metric_2 == '${minMaxPairs[3]}')`;\n const isolation = [\"true\", maxMinFilter, \"datum.Correlation > 0.5\",\"datum.Correlation < - 0.5\"];\n const subtitle = [null, `${analysisTextMax}; ${analysisTextMin}`, null, null];\n const correlations = [\"All\", \"Max/Min\", \"> 0.5\", \"< -0.5\"]; \n \n const corrMap = new Map(correlations.map((corr,i) => [corr, {isolation: isolation[i], subtitle: subtitle[i]}])); \n \n return Radio(corrMap, {label: \"Correlations\", value: corrMap.get(\"All\")});\n \n} ","pinned":false,"hidden":false,"mode":"ojs"},{"id":585,"value":"// https://vega.github.io/vega/docs/schemes/#diverging\nviewof colorScheme = { \n \n const schemes = new Map( [[\"Blue-Orange\",\"blueorange\"],\n [\"Brown-Blue-Green\",\"brownbluegreen\"],\n [\"Puple-Green\",\"purplegreen\"],\n [\"Pink-Yellow-Green\",\"pinkyellowgreen\"],\n [\"Purple-Orange\",\"purpleorange\"],\n [\"Red-Blue\",\"redblue\"],\n [\"Red-Grey\",\"redgrey\"],\n [\"Red-Yellow-Blue\",\"redyellowblue\"],\n [\"Red-Yellow-Green\",\"redyellowgreen\"],\n [\"Spectral\",\"spectral\"]\n ]);\n \n return Select(schemes, {label: \"Color Scheme\", value: \"spectral\"});\n} \n","pinned":false,"hidden":false,"mode":"ojs"},{"id":492,"value":"import {vl} from \"@vega/vega-lite-api\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":582,"value":"import { Select, Radio} from \"@observablehq/inputs\"\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":1201,"value":"# Test with other data for fork","pinned":false,"hidden":false,"mode":"md"},{"id":1272,"value":"dl = require(\"datalib\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":1214,"value":"import { data as testDataFork } from \"@observablehq/plot-exploration-penguins\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":1205,"value":"test = correlationAnalysis(testDataFork).view()","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W36","stdlib":"1","titling":"auto"}