31:T6da,md`# Montreal 311 incidents in 2016-2018 Montreal is begining to open up its data: [http://donnees.ville.montreal.qc.ca/](http://donnees.ville.montreal.qc.ca/). In this notebook, I was interested to try out a few techniques. But first I needed an interesting dataset, so I chose the 311 incidents (people can call the city for any complaint, comment, etc.). Unfortunately, the data came in a single 650Mb file for the past three years ([source](http://donnees.ville.montreal.qc.ca/dataset/requete-311)). I first purged nonpertinent columns using [csvkit](https://github.com/wireservice/csvkit) and replaced the duplicate category names with keys and translated them into English (GoogleTranslate). Features: 1. You can visualize specific categories by selecting them in the multiselect list below 2. The map will **flyToBounds ✈️** of current selection of markers 3. At certain zoom levels, a heatmap of the markers is generated 4. Used **d3.nest** to aggregate the data for visualization with **vega-lite 📊** 5. Clusters of nodes are nicely generated at different zoom levels ${(navigator.userAgent.match('HeadlessChrome')) ? html`

` : ''} **Note: Please be patient while the data loads -- these are ~20Mb files for each year.** They should cache in your browser, but unfortunately the parsing does not cache. I wanted to include a progress bar for the data downloads, but this [solution](https://beta.observablehq.com/@mbostock/streaming-fetch) was still a little to convoluded for my taste and I was worried about performance issues.' ${(navigator.userAgent.match('HeadlessChrome')) ? html`

` : ''} `32:[] 2f:{"title":"Montreal 311 incidents in 2016-2018","cells":[{"id":0,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":400,"value":"viewof year = select({\n title: \"Year\",\n options: [2016, 2017, 2018],\n value: 2018\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":186,"value":"md`\nThere was a total of **${d3.format(',')(incidents.length)}** incidents in **${year}**. You selected the following categories which contain **${d3.format(',')(filtered_incidents.length)}** incidents.\n${incident_category_list.map(d => {return `1. ${translate(d)}\\n`})}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1105,"value":"viewof category_list_order = select({\n options: [{value:'Alphabetical', label: 'Order by category name (A-Z)'},\n {value:'Count', label: 'Order by number of incidents for category (Highest first)'}],\n value: 'Alphabetical'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":748,"value":"viewof language = select({\n title: \"Select language for categories\",\n options: [{value: \"english\", label: \"English\"},\n {value: \"french\", label: \"French\"}],\n value: \"english\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1070,"value":"viewof incident_category_list = select({\n title: \"Select one or more categories to view on the map below\",\n multiple: true,\n size: 15,\n options: nested_incident_categories.map(row => {\n return {\n value: row.key,\n label: `${translate(row.key)} (${d3.format(',')(row.value)})`,\n };\n }),\n value: \"1118\", // Pothole\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":415,"value":"viewof map = {\n const container = html`

`;\n yield container; // Give the container dimensions.\n // Now we create a map object and add a layer to it.\n const map = container.value = new L.map(container).setView([45.5535,-73.6696], 11)\n let osmLayer = map.osmLayer = L.tileLayer(map_styles[map_styling].url, {\n attribution: map_styles[map_styling].attribution\n }).addTo(map);\n\n try {\n yield invalidation;\n } finally {\n map.eachLayer(function (layer) {\n map.removeLayer(layer)\n }); \n map.remove();\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1772,"value":"//TODO: Fix -- changing the map_styling seems to leave behind a ghost marker layer despite efforts to remove layers on re-render.\nviewof map_styling = select({\n title: 'Choose a map styling (Careful! bug: changing the styling seems to cause duplicate cluster markers)',\n options: [\n {value: 'stamen', label: 'Stamen (no street names)'},\n {value: 'openStreetMap', label: 'OpenStreetMap (with street names)'},\n ],\n value: 'stamen'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":702,"value":"embed({\n data: {values: by_category},\n width: 700,\n title: `Top 20 categories in ${year}`,\n encoding: {\n y: {field: 'key', type: 'ordinal', title: \"\", \"sort\": {\"op\": \"mean\", \"field\": \"value\", \"order\": \"descending\"}},\n x: {type: 'quantitative', field: 'value', axis: {format: \",.0f\"}, title: \"Incident count\"},\n },\n layer: [{\n mark: {type: 'bar', interpolate: 'cardinal'},\n },\n ] \n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1485,"value":"// Using d3.nest is way quicker than aggregating with vega lite\nembed({\n \"title\": `When incidents were reported (${year})`, // doesn't appear to support centering across hconcat\n \"hconcat\": [\n {\"mark\": \"bar\", \"data\": {\"values\": by_month}, \"encoding\": encoding, \"title\": \"By month\"},\n {\"mark\": \"bar\", \"data\": {\"values\": by_dow}, \"encoding\": encoding, \"title\": \"By Weekday\"},\n {\"mark\": \"bar\", \"data\": {\"values\": by_hour}, \"encoding\": encoding, \"title\": \"By hour of day\"},\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1740,"value":"by_category = d3.nest()\n .key(d => translate(d.desc_key))\n .rollup(v => v.length)\n .entries(incidents)\n .sort((a,b) => d3.descending(a.value, b.value))\n .filter((d,i) => i<20)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1611,"value":"by_month = d3.nest()\n .key(d => d3.timeFormat('%b')(d.datetime))\n .rollup(v => v.length)\n .entries(incidents)","pinned":false,"hidden":false,"mode":"ojs"},{"id":564,"value":"on_zoom = {\n map.off('zoomend');\n map.on('zoomend', function () {\n if (map.getZoom() > 12 && map.hasLayer(map.heat_layer)) {\n map.removeLayer(map.heat_layer);\n map.heat_layer = null;\n }\n else if (map.getZoom() <= 12 && map.hasLayer(map.heat_layer) == false) {\n // TODO: find a better way to fire heat_layer\n mutable trigger_heat_layer = mutable trigger_heat_layer + 1;\n }\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1624,"value":"by_dow = d3.nest()\n .key(d => d3.timeFormat('%a')(d.datetime))\n .rollup(v => v.length)\n .entries(incidents)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1630,"value":"by_hour = d3.nest()\n .key(d => d3.timeFormat('%H')(d.datetime))\n .rollup(v => v.length)\n .entries(incidents)\n .sort((a,b) => d3.ascending(parseInt(a.key), parseInt(b.key)))","pinned":false,"hidden":false,"mode":"ojs"},{"id":1777,"value":"map_styles = ({\n 'stamen': {\n url: 'https://stamen-tiles-{s}.a.ssl.fastly.net/watercolor/{z}/{x}/{y}.png', \n attribution: 'Map tiles by Stamen Design, CC BY 3.0 — Map data © OpenStreetMap'\n },\n 'openStreetMap': {\n url: 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',\n attribution: '© OpenStreetMap'\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1569,"value":"// Using d3.nest is way quicker than aggregating with vega lite\nencoding = ({\n \"x\": {\n \"field\": \"key\",\n \"type\": \"ordinal\",\n \"title\": \"\",\n \"sort\": {\"field\": \"key\"}\n },\n \"y\": {\n \"field\": \"value\",\n \"type\": \"quantitative\",\n \"title\": \"\"\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":450,"value":"update_markers = {\n\n if (map.markers) { \n map.removeLayer(map.markers);\n }\n let markers = map.markers = L.markerClusterGroup({ \n disableClusteringAtZoom: 16, \n removeOutsideVisibleBounds: true, \n chunkedLoading: true, // prevents blocking while rendering\n });\n \n filtered_incidents.forEach((incident) => {\n let marker = L.marker([incident.lat, incident.long])\n marker.bindPopup(`ID: ${incident.id}
\n Date: ${formatTime(incident.datetime)}
\n Description: ${translate(incident.desc_key)}`)\n marker.addTo(markers)\n });\n map.addLayer(markers);\n map.flyToBounds(markers.getBounds());\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1146,"value":"// TODO: find better way to trigger heat_layer\nmutable trigger_heat_layer = 0","pinned":false,"hidden":false,"mode":"ojs"},{"id":742,"value":"translate = (key) => {\n let category = all_incident_categories.find(d => d.key === key);\n return category && category[language]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1226,"value":"update_heatmap = {\n trigger_heat_layer;\n \n if (map.heat_layer) { map.removeLayer(map.heat_layer) };\n\n const heatmapLayer = map.heat_layer = L.layerGroup();\n heatmapLayer.addTo(map);\n \n // Redraw heatLayer once flyToBounds has finished to avoid ghosting\n map.once('zoomend', function() {\n const heat_layer = heatLayer(incidentPoints, {\n minOpacity: 0.4,\n maxZoom: 18,\n max: 1.0,\n radius: 8,\n blur: 6,\n gradient: null\n }).addTo(heatmapLayer)\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1373,"value":"chart_data_categories = nested_incident_categories.map(d => {\n d.label = translate(d.key)\n return d;\n })","pinned":false,"hidden":false,"mode":"ojs"},{"id":153,"value":"// Count the number of incidents in each category and sort according to user preference\nnested_incident_categories = d3.nest()\n .key(function(d) { return d.desc_key; })\n .rollup(function(leaves) { return leaves.length; })\n .entries(incidents)\n .sort(function(a,b){\n return category_list_order === 'Alphabetical' ? \n d3.ascending(translate(a.key), translate(b.key)) : \n d3.descending(a.value, b.value)\n })","pinned":false,"hidden":false,"mode":"ojs"},{"id":191,"value":"filtered_incidents = incidents.filter(d => incident_category_list.includes(d.desc_key))","pinned":false,"hidden":false,"mode":"ojs"},{"id":95,"value":"// create points for the heatmap layer\nincidentPoints = filtered_incidents.filter(i => {return !isNaN(i.long)}).map(i =>[i.lat, i.long, 0.1])","pinned":true,"hidden":false,"mode":"ojs"},{"id":4,"value":"incidents = {\n const parseDate = d3.timeParse(\"%Y-%m-%dT%H:%M:%S\")\n \n // gist barely accepted each of these due to file size, so separated into separate gists\n const filepaths = {\n '2018': 'https://gist.githubusercontent.com/john-clarke/a5054b64aeac44adc135fe9e8397c6ff/raw/1f7acf7d65c6b7329be4de3111909c3bc646b5c2/2018-compact.csv',\n '2017': 'https://gist.githubusercontent.com/john-clarke/f64c2cdb21d0162826331f2c6fce738d/raw/fbb930fc66a15ceaf4b5f58cbe3cef9b1bbb0fd4/2017-compact.csv',\n '2016': 'https://gist.githubusercontent.com/john-clarke/356da8fae95f8112abde5bcf9deee189/raw/ce76b28ecda144bfd333415ac06fb103f790e995/2016-compact.csv',\n }\n \n return d3.csv(filepaths[year],\n {cache: \"force-cache\"}, row => {\n return {id: row.ID_UNIQUE,\n datetime: parseDate(row.DDS_DATE_CREATION),\n long: +row.LOC_LONG,\n lat: +row.LOC_LAT,\n desc_key: row.ACTI_NOM_KEY,\n }\n }).then(data => data.filter(d => d.lat !== 0 && d.long !== 0))\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":733,"value":"// A non-standard description for each incident is used but is often repeated, so we created a map and translated into English to reduce filesize and improve readability for English readers.\nall_incident_categories = d3.csv('https://gist.githubusercontent.com/john-clarke/1cc6470f624ba5bf7fa8b355b7b68b7f/raw/249a45db11874ca97972621120b24b174ba3c4f3/montreal-311-incident-descriptions.csv')","pinned":false,"hidden":false,"mode":"ojs"},{"id":321,"value":"formatTime = d3.timeFormat(\"%B %d, %Y\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":48,"value":"L = require('leaflet@1.3.4')\n//L = require('leaflet@1.2.0')","pinned":false,"hidden":false,"mode":"ojs"},{"id":38,"value":"_ = require('lodash')","pinned":false,"hidden":false,"mode":"ojs"},{"id":82,"value":"heatLayer = L, require('leaflet.heat').catch(() => L.heatLayer)","pinned":false,"hidden":false,"mode":"ojs"},{"id":80,"value":"markerCluster = L, require('leaflet.markercluster@1.4.1').catch(() => L.markerClusterGroup)","pinned":false,"hidden":false,"mode":"ojs"},{"id":110,"value":"markerClusterCSS = html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":63,"value":"leafletCSS = html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":2,"value":"d3 = require('d3@5')","pinned":false,"hidden":false,"mode":"ojs"},{"id":402,"value":"import { select, checkbox } from '@jashkenas/inputs'","pinned":false,"hidden":false,"mode":"ojs"},{"id":703,"value":"embed = require(\"vega-embed@3\")","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}