2a:T6b3,md`# Montreal 311 incidents in 2016-2018 Montreal is begining to open up its [data](http://donnees.ville.montreal.qc.ca/). In this notebook, I was interested to try out a few techniques. But first I needed an interesting dataset, so I chose the 311 incidents (people can call the city for any complaint, comment, etc.). Unfortunately, the data came in a single 650Mb file for the past three years ([source](http://donnees.ville.montreal.qc.ca/dataset/requete-311)). I first purged nonpertinent columns using [csvkit](https://github.com/wireservice/csvkit) and replaced the duplicate category names with keys and translated them into English (GoogleTranslate). Features: 1. You can visualize specific categories by selecting them in the multiselect list below 2. The map will **flyToBounds ✈️** of current selection of markers 3. At certain zoom levels, a heatmap of the markers is generated 4. Used **d3.nest** to aggregate the data for visualization with **vega-lite 📊** 5. Clusters of nodes are nicely generated at different zoom levels ${(navigator.userAgent.match('HeadlessChrome')) ? html`

` : ''} **Note: Please be patient while the data loads -- these are ~20Mb files for each year.** They should cache in your browser, but unfortunately the parsing does not cache. I wanted to include a progress bar for the data downloads, but this [solution](https://beta.observablehq.com/@mbostock/streaming-fetch) was still a little to convoluded for my taste and I was worried about performance issues. ${(navigator.userAgent.match('HeadlessChrome')) ? html`

` : ''} `2b:T437,viewof map = { const container = html`

`; yield container; const map = (container.value = new L.map(container).setView( [45.5535, -73.6696], 11 )); invalidation.then(() => map.remove()); const osm = (map.osm = L.tileLayer(map_styles[map_styling].url, { attribution: map_styles[map_styling].attribution }).addTo(map)); const markers = (map.markers = L.markerClusterGroup({ disableClusteringAtZoom: 16, removeOutsideVisibleBounds: true, chunkedLoading: true // prevents blocking while rendering }).addTo(map)); const heatmap = (map.heatmap = L.layerGroup().addTo(map)); const borough = L.geoJSON(boroughs, { style: function (feature) { if (non_included_cities.includes(feature.properties.NOM)) return { color: "gray", fillOpacity: 0.5 }; }, weight: 2, color: "steelblue", opacity: 0.4, fillOpacity: 0 }).addTo(map); map.on("zoomstart", () => map.removeLayer(heatmap)); map.on("zoomend", () => map.getZoom() < 12 && map.addLayer(heatmap)); yield container; }2c:[] 29:{"title":"Montreal 311 incidents in 2016-2018","cells":[{"id":0,"value":"$2a","pinned":false,"hidden":false,"mode":"ojs"},{"id":400,"value":"viewof year = select({\n title: \"Year\",\n options: [2016, 2017, 2018],\n value: 2018\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":186,"value":"md`\nThere was a total of **${d3.format(',')(incidents.length)}** incidents in **${year}**. You selected the following categories which contain **${d3.format(',')(filtered_incidents.length)}** incidents.\n${incident_category_list.map(d => {return `1. ${translate(d)}\\n`})}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1105,"value":"viewof category_list_order = select({\n options: [{value:'Alphabetical', label: 'Order by category name (A-Z)'},\n {value:'Count', label: 'Order by number of incidents for category (Highest first)'}],\n value: 'Alphabetical'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":748,"value":"viewof language = select({\n title: \"Select language for categories\",\n options: [{value: \"english\", label: \"English\"},\n {value: \"french\", label: \"French\"}],\n value: \"english\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1070,"value":"viewof incident_category_list = select({\n title: \"Select one or more categories to view on the map below\",\n multiple: true,\n size: 15,\n options: nested_incident_categories.map(row => {\n return {\n value: row.key,\n label: `${translate(row.key)} (${d3.format(',')(row.value)})`,\n };\n }),\n value: \"1118\", // Pothole\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1772,"value":"viewof map_styling = select({\n title: 'Choose a map styling',\n options: [\n {value: 'cartoLight', label: 'cartoLight'},\n {value: 'stamen', label: 'Stamen (no street names)'},\n {value: 'openStreetMap', label: 'OpenStreetMap (with street names)'},\n ],\n value: 'cartoLight'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":415,"value":"$2b","pinned":false,"hidden":false,"mode":"ojs"},{"id":450,"value":"{\n map.markers.clearLayers();\n map.heatmap.clearLayers();\n\n filtered_incidents.forEach((incident) => {\n const marker = L.marker([incident.lat, incident.long]);\n marker.bindPopup(`ID: ${incident.id}
\n Date: ${formatTime(incident.datetime)}
\n Description: ${translate(incident.desc_key)}`);\n marker.addTo(map.markers);\n });\n\n // create points for the heatmap layer\n L.heatLayer(filtered_incidents.filter(i => {return !isNaN(i.long)}).map(i =>[i.lat, i.long, 0.1]), {\n minOpacity: 0.4,\n maxZoom: 18,\n max: 1.0,\n radius: 8,\n blur: 6,\n gradient: null\n }).addTo(map.heatmap);\n \n map.flyToBounds(map.markers.getBounds());\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":702,"value":"embed({\n data: {values: by_category},\n width: 700,\n title: `Top 20 categories in ${year}`,\n encoding: {\n y: {field: 'key', type: 'ordinal', title: \"\", \"sort\": {\"op\": \"mean\", \"field\": \"value\", \"order\": \"descending\"}},\n x: {type: 'quantitative', field: 'value', axis: {format: \",.0f\"}, title: \"Incident count\"},\n },\n layer: [{\n mark: {type: 'bar', interpolate: 'cardinal'},\n },\n ] \n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1485,"value":"// Using d3.nest is way quicker than aggregating with vega lite\nembed({\n \"title\": `When incidents were reported (${year})`, // doesn't appear to support centering across hconcat\n \"hconcat\": [\n {\"mark\": \"bar\", \"data\": {\"values\": by_month}, \"encoding\": encoding, \"title\": \"By month\"},\n {\"mark\": \"bar\", \"data\": {\"values\": by_dow}, \"encoding\": encoding, \"title\": \"By Weekday\"},\n {\"mark\": \"bar\", \"data\": {\"values\": by_hour}, \"encoding\": encoding, \"title\": \"By hour of day\"},\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":2034,"value":"md`#### Acknowledgments\n\n*Thanks to [Mike Bostock](/@mbostock) for his help in fixing and improving this notebook.*`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1740,"value":"by_category = d3.nest()\n .key(d => translate(d.desc_key))\n .rollup(v => v.length)\n .entries(incidents)\n .sort((a,b) => d3.descending(a.value, b.value))\n .filter((d,i) => i<20)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1611,"value":"by_month = d3.nest()\n .key(d => d3.timeFormat('%b')(d.datetime))\n .rollup(v => v.length)\n .entries(incidents)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1624,"value":"by_dow = d3.nest()\n .key(d => d3.timeFormat('%a')(d.datetime))\n .rollup(v => v.length)\n .entries(incidents)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1630,"value":"by_hour = d3.nest()\n .key(d => d3.timeFormat('%H')(d.datetime))\n .rollup(v => v.length)\n .entries(incidents)\n .sort((a,b) => d3.ascending(parseInt(a.key), parseInt(b.key)))","pinned":false,"hidden":false,"mode":"ojs"},{"id":1777,"value":"map_styles = ({\n 'stamen': {\n url: 'https://stamen-tiles-{s}.a.ssl.fastly.net/watercolor/{z}/{x}/{y}.png', \n attribution: 'Map tiles by Stamen Design, CC BY 3.0 — Map data © OpenStreetMap'\n },\n 'openStreetMap': {\n url: 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',\n attribution: '© OpenStreetMap'\n },\n 'cartoLight': {\n url: 'https://cartodb-basemaps-{s}.global.ssl.fastly.net/light_all/{z}/{x}/{y}{r}.png',\n attribution: '© OpenStreetMap | © CartoDB'\n },\n \n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1569,"value":"// Using d3.nest is way quicker than aggregating with vega lite\nencoding = ({\n \"x\": {\n \"field\": \"key\",\n \"type\": \"ordinal\",\n \"title\": \"\",\n \"sort\": {\"field\": \"key\"}\n },\n \"y\": {\n \"field\": \"value\",\n \"type\": \"quantitative\",\n \"title\": \"\"\n }\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":742,"value":"translate = (key) => {\n let category = all_incident_categories.find(d => d.key === key);\n return category && category[language]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1373,"value":"chart_data_categories = nested_incident_categories.map(d => {\n d.label = translate(d.key)\n return d;\n })","pinned":false,"hidden":false,"mode":"ojs"},{"id":153,"value":"// Count the number of incidents in each category and sort according to user preference\nnested_incident_categories = d3.nest()\n .key(function(d) { return d.desc_key; })\n .rollup(function(leaves) { return leaves.length; })\n .entries(incidents)\n .sort(function(a,b){\n return category_list_order === 'Alphabetical' ? \n d3.ascending(translate(a.key), translate(b.key)) : \n d3.descending(a.value, b.value)\n })","pinned":false,"hidden":false,"mode":"ojs"},{"id":191,"value":"filtered_incidents = incidents.filter(d => incident_category_list.includes(d.desc_key))","pinned":false,"hidden":false,"mode":"ojs"},{"id":4,"value":"incidents = {\n const parseDate = d3.timeParse(\"%Y-%m-%dT%H:%M:%S\")\n \n // gist barely accepted each of these due to file size, so separated into separate gists\n const filepaths = {\n '2018': 'https://gist.githubusercontent.com/john-clarke/a5054b64aeac44adc135fe9e8397c6ff/raw/1f7acf7d65c6b7329be4de3111909c3bc646b5c2/2018-compact.csv',\n '2017': 'https://gist.githubusercontent.com/john-clarke/f64c2cdb21d0162826331f2c6fce738d/raw/fbb930fc66a15ceaf4b5f58cbe3cef9b1bbb0fd4/2017-compact.csv',\n '2016': 'https://gist.githubusercontent.com/john-clarke/356da8fae95f8112abde5bcf9deee189/raw/ce76b28ecda144bfd333415ac06fb103f790e995/2016-compact.csv',\n }\n \n return d3.csv(filepaths[year],\n {cache: \"force-cache\"}, row => {\n return {id: row.ID_UNIQUE,\n datetime: parseDate(row.DDS_DATE_CREATION),\n long: +row.LOC_LONG,\n lat: +row.LOC_LAT,\n desc_key: row.ACTI_NOM_KEY,\n }\n }).then(data => data.filter(d => d.lat !== 0 && d.long !== 0))\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":733,"value":"// A non-standard description for each incident is used but is often repeated, so we created a map and translated into English to reduce filesize and improve readability for English readers.\nall_incident_categories = d3.csv('https://gist.githubusercontent.com/john-clarke/1cc6470f624ba5bf7fa8b355b7b68b7f/raw/249a45db11874ca97972621120b24b174ba3c4f3/montreal-311-incident-descriptions.csv')","pinned":false,"hidden":false,"mode":"ojs"},{"id":2046,"value":"non_included_cities = ['Baie-d\\'Urfé', 'Montréal-Est', 'Pointe-Claire', 'Westmount', 'Beaconsfield', 'L\\'Île-Dorval', 'Dorval', 'Dollard-des-Ormeaux', 'Sainte-Anne-de-Bellevue', 'Mount Royal', \"L'Île-Bizard–Sainte-Geneviève\", 'Hampstead', 'Côte-Saint-Luc', 'Kirkland', 'Senneville']","pinned":false,"hidden":false,"mode":"ojs"},{"id":2048,"value":"boroughs = d3.json('https://gist.githubusercontent.com/john-clarke/f7c03ff2d4264987236d69b9b7cca832/raw/bd263403f7f9a607dcd04c33c12c62120adf43e2/arrondissements.geojson')","pinned":false,"hidden":false,"mode":"ojs"},{"id":321,"value":"formatTime = d3.timeFormat(\"%B %d, %Y\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":48,"value":"L = {\n const L = await require('leaflet@1.3.4');\n if (!L._css) {\n document.head.appendChild(L._css = html`\n \n \n `);\n }\n await require('leaflet.markercluster@1.4.1').catch(() => L.markerClusterGroup);\n await require('leaflet.heat').catch(() => L.heatLayer);\n return L;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2,"value":"d3 = require('d3@5')","pinned":false,"hidden":false,"mode":"ojs"},{"id":402,"value":"import { select, checkbox } from '@jashkenas/inputs'","pinned":false,"hidden":false,"mode":"ojs"},{"id":703,"value":"embed = require(\"vega-embed@3\")","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W2c","stdlib":"1","titling":"auto"}