2f:T7f1,# Sleep Visualization # * Oura sleep summary dates are keyed off the day before the wakeup date * The date reported in the Oura app is the wakeup date (day after the key date) ## TODO ## - Document what I've done so far: https://bost.ocks.org/mike/join/ - Investigate correlation of short deep sleep chunks with events, movement, or o2 drops - Did I get a shift in resting HR with the CPAP pressure change and pad install? - Shapes to cusum chart: https://github.com/d3/d3-shape#symbolDiamond - Add oura HR, met1, cls5 - Frequency of apnea events when listening to radio versus zoning out - Cusum: multi-variable support and option to filter gaps. Window based on data points or based on delta time (gap handling) - Bivariate analysis with histograms - Sleep stage bucket analysis (in old analysis) - Fix formatting: https://github.com/d3/d3-format - Correlation analysis - https://www.d3-graph-gallery.com/correlogram.html - https://jstat.github.io/all.html ## Resources ## - Using D3.JS book: http://using-d3js.com/ - User manual: https://observablehq.com/@observablehq/user-manual - Gallery: https://observablehq.com/@d3/gallery - https://www.d3-graph-gallery.com - Observable plot: - https://observablehq.com/@observablehq/plot - https://observablehq.com/@observablehq/plot-cheatsheets - file:///Users/dsa0079/Downloads/plot-cheatsheets.pdf - https://observablehq.com/@observablehq/plot-overview-for-matplotlib-users?collection=@observablehq/plot-for-matplotlib-users - Thinking about visualizations - https://www.data-to-viz.com - https://www.data-to-viz.com/caveat/overplotting.html - https://www.data-to-viz.com/graph/scatter.html - https://observablehq.com/@amazingdesign/observable-notebooks-basics - https://observablehq.com/@observablehq/plot-cheatsheets?collection=@observablehq/plot-cheatsheets - https://observablehq.com/@sorig/console-log-without-leaving-your-notebook - https://observablehq.com/@d3/d3v6-migration-guide - https://observablehq.com/@d3/learn-d3-interaction30:T136c,data = { const data = {}; /** * Unpack a little-endian packed number from a byte array */ const unpack = byteData => byteData.reduce((prev, curr, i) => prev + curr * Math.pow(2, i * 8)); // raw o2ring data data.o2raw = new Uint8Array(await FileAttachment("o2ring@2").arrayBuffer()); /** * Expand compressed raw o2ring data */ data.o2ring = []; function extractO2(ix) { const t0 = unpack(data.o2raw.slice(ix, ix + 8)); const n = unpack(data.o2raw.slice(ix + 8, ix + 10)); const out = []; for (let i = 0; i < n; i++) { const ref = ix + 10 + i * 3; out.push({ t: new Date(t0 + 4000 * i), spo2: data.o2raw[ref], pulse: data.o2raw[ref+1], motion: data.o2raw[ref+2] }); } return out } let ptr = 0; while (ptr < data.o2raw.length) { const arr = extractO2(ptr); ptr += 3 * arr.length + 8 + 2; arr.forEach(x => { data.o2ring.push(x); }); } /** * Process oura sleep metadata and convert to a map */ const sleep_meta = await FileAttachment("sleep_meta@5.json").json(); data.sleep_meta_map = {}; sleep_meta.forEach(sm => { // convert date key sm.Date = new Date(sm.summary_date); // convert bedtime start and end times with 5 hour time zone offset sm.bedtime_start = new Date(sm.bedtime_start - 5 * 60 * 60 * 1000); sm.bedtime_end = new Date(sm.bedtime_end - 5 * 60 * 60 * 1000); // convert sleep totals to hours sm.deep = sm.deep / 60 / 60; sm.total = sm.total / 60 / 60; sm.rem = sm.rem / 60 / 60; sm.light = sm.light / 60 / 60; sm.awake = sm.awake / 60 / 60; // attach to output map data.sleep_meta_map[sm.Date] = sm; }); /** * Process OSCAR summary data and convert to a map */ data.oscar_summary_map = {}; (await FileAttachment("oscar_summary@3.csv").csv()).forEach(o => { Object.keys(o).forEach(k => { const v = o[k]; if (v.match(/^\d{4}\-\d{2}\-\d{2}/)) { o[k] = new Date(o[k]); } else if (v.match(/^\d\d:\d\d:\d\d$/)) { const [h, m, s] = v.split(":"); o[k] = parseInt(h) + parseInt(m) / 60 + parseInt(s) / 60 / 60; } else { o[k] = parseFloat(o[k]); } }); data.oscar_summary_map[o.Date] = o; }); /** * Process OSCAR event data */ data.oscar_events = await FileAttachment("oscar_2@4.json").json(); const k = 24 * 60 * 60 * 1000; data.oscar_events.forEach(o => { o.FloorDate = new Date(k * (Math.round(o.DateTime / k) - 1)); o.DateTime = new Date(o.DateTime); o.FloorMonthYear = new Date(o.DateTime.getFullYear(), o.DateTime.getMonth(), 1); o.Hour = (5 + o.DateTime.getHours()) % 24; const sm = data.sleep_meta_map[o.FloorDate]; if (sm !== undefined) { o.BedtimeStart = sm.bedtime_start; o.BedtimeEnd = sm.bedtime_end; o.DeltaTime = (o.DateTime - sm.bedtime_start) / 60 / 60 / 1000; } else { o.BedtimeStart = null; o.BedtimeEnd = null; o.DeltaTime = null; } o.InAHI = o.Event == "Obstructive" || o.Event == "Hypopnea" || o.Event == "ClearAirway" || o.Event == "RERA" || o.Event == "FlowLimit"; }); /** * Calculate spacing between events */ // TODO: do within-session processing instead of cross-session (within date) processing const grouped_events = d3.group(data.oscar_events.filter(e => e.InAHI), d => d.FloorDate); data.event_spacing = []; for (let [event_date, events] of grouped_events) { let prev = null; const sorted_events = d3.sort(events, d => d.DateTime); sorted_events.forEach((event, index) => { if (prev !== null) { const dt = (event.DateTime - prev.DateTime) / 60000; // minutes event.dt = dt; const dt_event = { FloorDate: event.FloorDate, DateTime: event.DateTime, dt: dt, AHI: data.oscar_summary_map[event.FloorDate].AHI }; data.event_spacing.push(dt_event); } prev = event; }); data.oscar_summary_map[event_date].Event85 = d3.quantile(sorted_events.map(d => d.dt), 0.8); } /** * Merge summary maps */ data.summary_map = {}; for (const [key, value] of Object.entries(data.sleep_meta_map)) { data.summary_map[key] = {}; for (const [k, v] of Object.entries(value)) { data.summary_map[key][k] = v; } } for (const [key, value] of Object.entries(data.oscar_summary_map)) { data.summary_map[key] = data.summary_map[key] === undefined ? {} : data.summary_map[key]; for (const [k, v] of Object.entries(value)) { data.summary_map[key][k] = v; } } /** * Oura stage data */ data.oura_stage = await FileAttachment("oura@3.json").json(); data.oura_stage.forEach(s => { s.time = new Date(s.time); }); /** * Return processed data */ return data }31:Tc10,function get_overlay_meta(y, m, d) { const meta = {}; // summary date as a JavaScript Date (with 5 hour adjustment for time zone) if (y instanceof Date) { meta.overlay_key = y; } else { meta.overlay_key = new Date(y, m - 1, d, -5, 0, 0); } // key date as a string meta.date_string = d3.timeFormat("%x")(new Date(meta.overlay_key.getTime() + 5 * 60 * 60 * 1000)); // use 8PM - 10AM as sleep window meta.sleep_start = new Date(meta.overlay_key.getTime() + ( 20/24) * 24 * 60 * 60 * 1000); meta.sleep_stop = new Date(meta.overlay_key.getTime() + (1 + 10/24) * 24 * 60 * 60 * 1000); const eventAbbreviations = { "ClearAirway": "CA", "EPAP": "E", "FlowLimit": "FL", "Hypopnea": "H", "Obstructive": "OA", "PressurePulse": "PP", "RERA": "R", "VSnore": "VS", "VSnore2": "VS2" }; // filtered data sources meta.sleep_event = data.oscar_events .filter(e => e.DateTime >= meta.sleep_start && e.DateTime <= meta.sleep_stop) .map(x => {return { "time": x.DateTime, "dt": x.DateTime - meta.sleep_start, "event": x.Event, "shortEvent": eventAbbreviations[x.Event], "key": meta.overlay_key, "width": 1, "color": "black" }}); // oura hypnogram data const hypnogram_classes = {"1": "Deep", "2": "Light", "3": "REM", "4": "Awake"}; const hypnogram_colors = {"1": "royalblue", "2": "lightsteelblue", "3": "lightskyblue", "4": "wheat"}; meta.sleep_oura = []; if (data.sleep_meta_map[meta.overlay_key] !== undefined) { const smm = data.sleep_meta_map[meta.overlay_key]; smm.hypnogram_5min.split("").forEach((x, i) => { meta.sleep_oura.push({ "time": new Date(smm.bedtime_start.getTime() + i * 5 * 60 * 1000), "dt": i * 5 * 60 * 1000, "event": hypnogram_classes[x], "key": meta.overlay_key, "width": 8, "color": hypnogram_colors[x] }); }); } // combined event data meta.combined = []; meta.sleep_oura .forEach(x => { meta.combined.push(x) }); meta.sleep_event.forEach(x => { meta.combined.push(x) }); // event sort order meta.sort_order = ["Awake", "REM", "Light", "Deep", "Hypopnea", "Obstructive", "ClearAirway"]; const found = {}; meta.sort_order.forEach(x => { found[x] = 1 }); const event_map = d3.rollup(meta.combined, v => v.length, d => d.event); event_map.forEach((v, k) => { if (found[k] === undefined) { meta.sort_order.push(k); } }); meta.abbreviation_sort_order = meta.sort_order.map(x => eventAbbreviations[x] !== undefined ? eventAbbreviations[x] : x); // o2ring data meta.o2ring = data.o2ring .filter(x => x.t >= meta.sleep_start && x.t <= meta.sleep_stop) .filter(d => d.pulse < 255); // add heart rate to sleep events const o2ring_t = meta.o2ring.map(d => d.t); meta.sleep_event.forEach(e => { const ix = d3.bisectLeft(o2ring_t, e.time); e.pulse = data.o2ring[ix].pulse; }); return meta }32:T32bf,sleep_line = { // use slider value for date and use it to look up overlay data const meta = get_overlay_meta(everything_date); /* * Layout */ // dimensions (in pixels) const dimensions = { innerWidth: 900, // width of displays mainHeight: 400, // height of main display laneHeight: 10, // height of event lane display padLeft: 80, padRight: 20, padTop: 30, padBottom: 30, padBetween: 30, // padding between displays }; // computed dimensions: // inner (shared width and padding for regions) dimensions.innerLeft = dimensions.padLeft; dimensions.innerRight = dimensions.padLeft + dimensions.innerWidth; dimensions.width = dimensions.innerWidth + dimensions.padLeft + dimensions.padRight; // main: main display panel with o2ring data dimensions.mainTop = dimensions.padTop; dimensions.mainBottom = dimensions.mainHeight + dimensions.padTop; // event: event data paired with main panel dimensions.eventHeight = dimensions.laneHeight * (meta.sort_order.length - 3) // collapse stages dimensions.eventTop = dimensions.mainBottom + dimensions.padBetween; dimensions.eventBottom = dimensions.eventTop + dimensions.eventHeight; // zoom: independent event data for zooming dimensions.zoomHeight = dimensions.laneHeight * meta.sort_order.length; dimensions.zoomTop = dimensions.eventBottom + dimensions.padBetween; dimensions.zoomBottom = dimensions.zoomTop + dimensions.zoomHeight; // total height dimensions.height = dimensions.zoomBottom + dimensions.padBottom; // domains are minimum and maximum data values that map to the minimum // and maximum SVG values dimensions.xDomain = d3.extent(meta.o2ring.map(d => d.t)); dimensions.yDomainM = [0, 100]; // ranges are minimum and maximum values in SVG internal (viewBox) coordinates dimensions.xRange = [0, dimensions.innerWidth]; dimensions.yRangeM = [dimensions.mainHeight , 0]; dimensions.yRangeE = [dimensions.eventBottom, dimensions.eventTop]; dimensions.yRangeZ = [dimensions.zoomBottom , dimensions.zoomTop]; // scales map domains to ranges const xScale1 = d3.scaleUtc(dimensions.xDomain, dimensions.xRange); const xScale2 = d3.scaleUtc().domain(dimensions.xDomain).range(dimensions.xRange); const yScaleM = d3.scaleLinear(dimensions.yDomainM, dimensions.yRangeM); const yScaleE = d3.scaleOrdinal(meta.sort_order.slice(4), d3.range(0, 10 * (meta.sort_order.length - 4), 10)); const yScaleZ = d3.scaleOrdinal(meta.sort_order, meta.sort_order.map((d, i) => 10 * i)); // mapping from event types to lane position const eventScaleE = {}; meta.sort_order.forEach((e, i) => { eventScaleE[e] = 10 * (i < 4 ? 0 : i - 3); }); // create svg element // make internal coordinates match external coordinates with origin at top left const svg = d3.create("svg") .attr("width", dimensions.width) .attr("height", dimensions.height) .attr("viewBox", [0, 0, dimensions.width, dimensions.height]) // min-x, min-y, width, height .attr("style", "max-width: 100%; height: auto; height: intrinsic;"); // main container const mainGroup = svg.append("g") .attr("transform", `translate(${dimensions.innerLeft}, ${dimensions.mainTop})`); const mainRect = mainGroup.append("rect") .attr("width", dimensions.innerWidth) .attr("height", dimensions.mainHeight) .attr("stroke", "none") .attr("fill", "white"); // container for event data const eventGroup = svg.append("g") .attr("transform", `translate(${dimensions.innerLeft}, ${dimensions.eventTop})`); eventGroup.append("rect") .attr("stroke", "none") .attr("fill", "white") .attr("height", dimensions.eventHeight) .attr("width", dimensions.innerWidth); // clippath to keep zoomed line from moving outside of region mainGroup.append("defs").append("clipPath") .attr("id", "main-clip") .append("rect") .attr("height", dimensions.mainHeight) .attr("width", dimensions.innerWidth); eventGroup.append("defs").append("clipPath") .attr("id", "event-clip") .append("rect") .attr("height", dimensions.eventHeight) .attr("width", dimensions.innerWidth); // container for zoom data const zoomGroup = svg.append("g") .attr("transform", `translate(${dimensions.innerLeft}, ${dimensions.zoomTop})`); zoomGroup.append("rect") .attr("stroke", "none") .attr("fill", "white") .attr("height", dimensions.zoomHeight) .attr("width", dimensions.zoomWidth); // create a title const titleContainer = svg.append("g"); titleContainer.append("text") .attr("x", 500) .attr("y", 20) .attr("font-family", "Arial, Helvetica, sans-serif") .text(meta.date_string); // line colors const lineColors = { "SpO2": "blue", "Pulse": "black", "Motion": "green" }; // legend const legend = svg.append("g").attr("transform", `translate(${dimensions.padLeft}, 5)`); legend.append("rect").attr("width", 160).attr("height", 20).attr("stroke", "black").attr("fill", "white"); legend.append("circle").attr("cx", 10).attr("cy", 10).attr("r", 5).style("fill", lineColors.SpO2); legend.append("circle").attr("cx", 60).attr("cy", 10).attr("r", 5).style("fill", lineColors.Pulse); legend.append("circle").attr("cx", 110).attr("cy", 10).attr("r", 5).style("fill", lineColors.Motion); legend.append("text").attr("x", 20).attr("y", 10).text("SpO2").style("font-size", "10px").attr("alignment-baseline", "middle"); legend.append("text").attr("x", 70).attr("y", 10).text("Pulse").style("font-size", "10px").attr("alignment-baseline", "middle"); legend.append("text").attr("x", 120).attr("y", 10).text("Motion").style("font-size", "10px").attr("alignment-baseline", "middle"); /* * Create axes */ const xAxis = d3.axisBottom(xScale1).ticks(width / 80).tickSizeOuter(0); mainGroup.append("g") .attr("class", "x-axis-main") .attr("transform", `translate(0, ${dimensions.mainHeight})`) .call(xAxis); const xAxis2 = d3.axisBottom(xScale2).ticks(width / 80).tickSizeOuter(0); zoomGroup.append("g") .attr("transform", `translate(0, ${dimensions.zoomHeight})`) .call(xAxis2); const yFormat = undefined; const yAxis = d3.axisLeft(yScaleM).ticks(dimensions.height / 40, yFormat); mainGroup.append("g") .call(yAxis) .call(g => g.select(".domain").remove()) .call(g => g.selectAll(".tick line").clone() .attr("x2", dimensions.width - dimensions.padLeft - dimensions.padRight) .attr("stroke-opacity", 0.1)); const yAxisE = d3.axisLeft(yScaleE); eventGroup.append("g") .call(yAxisE) .attr("transform", `translate(0, ${dimensions.laneHeight / 2})`) .call(g => g.selectAll(".tick line").clone() .attr("x2", dimensions.width - dimensions.padLeft - dimensions.padRight) .attr("stroke-opacity", 0.1)); const yAxisZ = d3.axisLeft(yScaleZ); zoomGroup.append("g") .call(yAxisZ) .attr("transform", `translate(0, ${dimensions.laneHeight / 2})`) .call(g => g.selectAll(".tick line").clone() .attr("x2", dimensions.width - dimensions.padLeft - dimensions.padRight) .attr("stroke-opacity", 0.1)); /* * Main data */ // line generator for pulse data const pulseLineGenerator = d3.line().x(d => xScale1(d.t)).y(d => yScaleM(d.pulse)); mainGroup.append("path") .datum(meta.o2ring) .attr("class", "pulse-line") .attr("fill", "none") .attr("stroke", lineColors.Pulse) .attr("clip-path", "url(#main-clip)") .attr("d", pulseLineGenerator); const colorScale = d3.scaleOrdinal(d3.schemeTableau10); const motionLineGenerator = d3.line().x(d => xScale1(d.t)).y(d => yScaleM(d.motion / 4)); mainGroup.append("path") .datum(meta.o2ring) .attr("class", "motion-line") .attr("fill", "none") .attr("stroke", lineColors.Motion) .attr("clip-path", "url(#main-clip)") .attr("d", motionLineGenerator); const oxygenLineGenerator = d3.line().x(d => xScale1(d.t)).y(d => yScaleM(d.spo2)); mainGroup.append("path") .datum(meta.o2ring) .attr("class", "oxygen-line") .attr("fill", "none") .attr("stroke", lineColors.SpO2) .attr("clip-path", "url(#main-clip)") .attr("d", oxygenLineGenerator); const eventColorScale = d3.scaleOrdinal(meta.sort_order.filter(x => x !== "Awake" && x !== "Light" && x !== "REM" && x !== "Deep"), d3.schemeCategory10); const mainEventLines = mainGroup.append("g"); /* * Sleep stage data */ const eventGroupData = eventGroup.append("g"); eventGroupData .selectAll("rect") .data(meta.sleep_oura) .join("rect") .attr("clip-path", "url(#event-clip)") .attr("fill", d => d.color) .attr("x", d => xScale1(d.time)) .attr("y", d => dimensions.laneHeight * (meta.sort_order.length - 4)) .attr("width", 10) // todo: compute .attr("height", dimensions.laneHeight); eventGroupData .selectAll("line") .data(meta.sleep_event) .join("line") .attr("clip-path", "url(#event-clip)") .attr("stroke", d => eventColorScale(d.event)) .attr("x1", d => xScale1(d.time)) .attr("x2", d => xScale1(d.time)) .attr("y1", d => eventScaleE[d.event]-10) .attr("y2", d => eventScaleE[d.event]); zoomGroup.append("g") .selectAll("rect") .data(meta.sleep_oura) .join("rect") .attr("fill", d => d.color) .attr("x", d => xScale2(d.time)) // .attr("y", d => eventScaleZ[d.event] - 10) .attr("y", d => yScaleZ(d.event)) .attr("width", 10) // todo: compute .attr("height", dimensions.laneHeight); zoomGroup.append("g") .selectAll("line") .data(meta.sleep_event) .join("line") .attr("stroke", d => eventColorScale(d.event)) .attr("x1", d => xScale2(d.time)) .attr("x2", d => xScale2(d.time)) .attr("y1", d => yScaleZ(d.event)) .attr("y2", d => yScaleZ(d.event) + 10); /* * Brushing */ // brush: handle 'end' event and call detail() when brush is done const brush = d3.brushX().on("end", detail); zoomGroup.call(brush); function detail(event) { console.log(event); if (event.selection !== null) { const [x0, x1] = event.selection; console.log("inversion", [xScale2.invert(x0), xScale2.invert(x1)]); xScale1.domain([xScale2.invert(x0), xScale2.invert(x1)]); } else { xScale1.domain(d3.extent(meta.o2ring.map(d => d.t))); } const t = d3.transition().duration(1500).ease(d3.easeLinear); d3.select(".pulse-line") .transition(t) .attr("d", pulseLineGenerator.x(d => xScale1(d.t))); d3.select(".motion-line") .transition(t) .attr("d", motionLineGenerator.x(d => xScale1(d.t))); d3.select(".oxygen-line") .transition(t) .attr("d", oxygenLineGenerator.x(d => xScale1(d.t))); d3.select(".x-axis-main") .transition(t) .call(xAxis); eventGroupData.selectAll("line") .transition(t) .attr("x1", d => xScale1(d.time)) .attr("x2", d => xScale1(d.time)) .attr("y1", d => eventScaleE[d.event] - 10) .attr("y2", d => eventScaleE[d.event]); eventGroupData .selectAll("rect") .transition(t) .attr("x", d => xScale1(d.time)) .attr("y", d => dimensions.laneHeight * (meta.sort_order.length - 4)) .attr("width", d => xScale1(new Date(d.time.getTime() + 5 * 60 * 1000)) - xScale1(d.time)) // todo: compute .attr("height", dimensions.laneHeight); } /* * Vertical ruler */ const fmt = d3.timeFormat("%X"); // const hoverLineGroup = svg.append("g").attr("class", "hover-line"); const hoverLine1 = mainGroup.append("line").attr("stroke", "black").attr("opacity", 0.2) .attr("x1", 100).attr("x2", 100).attr("y1", 0).attr("y2", dimensions.mainHeight); const hoverLine2 = eventGroup.append("line").attr("stroke", "black").attr("opacity", 0.2) .attr("x1", 100).attr("x2", 100).attr("y1", 0).attr("y2", dimensions.eventHeight); const hoverLine3 = zoomGroup.append("line").attr("stroke", "black").attr("opacity", 0.2) .attr("x1", 100).attr("x2", 100).attr("y1", 0).attr("y2", dimensions.zoomHeight); const hoverDate = mainGroup.append("g").append("text").attr("x", 100 + 10).attr("y", 100) .attr("font-size", 10).attr("font-family", "Arial, Helvetica, sans-serif").text(""); mainRect.on("mousemove", function(event){ const [mouse_x, mouse_y] = d3.pointer(event); // todo: d3 time format const mouse_date = new Date(xScale1.invert(mouse_x).getTime() + 5 * 60 * 60 * 1000); hoverLine1.attr("x1", mouse_x).attr("x2", mouse_x); hoverLine2.attr("x1", mouse_x).attr("x2", mouse_x); hoverDate.attr("x", mouse_x + 10).attr("y", mouse_y).text(fmt(mouse_date)); const inverted = xScale1.invert(mouse_x); hoverLine3.attr("x1", xScale2(inverted)).attr("x2", xScale2(inverted)); }); // return the generated svg return svg.node(); }33:T149e,{ const yVar = cusum_select; const window = 10; function moving(data) { const out = []; data.forEach((d, i) => { const a = i - window > 0 ? i - window : 0; const b = i + window < data.length ? i + window : data.length; out.push({"x": d.x, "y": d3.mean(data.slice(a, b).map(d => d.y))}); }); return out; } function cusum(data) { const out = []; const ymean = d3.mean(data.map(d => d.y)); var cumulative = 0; data.forEach(d => { cumulative += d.y - ymean; out.push({"x": d.x, "y": cumulative}); }); return out; } const _data = Object.values(data.summary_map) .map(d => ({"x": d.Date, "y": d[yVar]})) .filter(d => d.y !== undefined); const sorted_data = d3.sort(_data, d => d.x); const dimensions = { "height" : 600, "width" : 800, "padLeft" : 30, "padRight" : 20, "padTop" : 20, "padBottom": 20 }; dimensions.top = dimensions.padTop; dimensions.bottom = dimensions.height - dimensions.padBottom; dimensions.left = dimensions.padLeft; dimensions.right = dimensions.width - dimensions.padRight; dimensions.innerHeight = dimensions.bottom - dimensions.top; dimensions.innerWidth = dimensions.right - dimensions.left; const xDomain = d3.extent(sorted_data.map(d => d.x)); const yDomain = d3.extent(sorted_data.map(d => d.y)); const xScale = d3.scaleUtc(xDomain, [0, dimensions.innerWidth]); const yScale = d3.scaleLinear(yDomain, [dimensions.innerHeight, 0]); const div = d3.create("div"); div.append("style").text(` path { fill: none; } path.main { stroke: black; stroke-width: 1; opacity: 0.3; } path.cusum { stroke: blue; stroke-width: 3; } circle { fill: black; opacity: 0.1; r: 3; } path.moving { stroke: black; stroke-width: 3; } #tooltip { background: cornsilk; border: 1px solid black; border-radius: 5px; padding: 5px; } `); const tooltip = div.append("div").attr("id", "tooltip").style("position", "absolute").text("tooltip"); const svg = div.append("svg") .attr("width", dimensions.width) .attr("height", dimensions.height) .attr("viewBox", [0, 0, dimensions.width, dimensions.height]) .attr("style", "max-width: 100%; height: auto; height: intrinsic;"); const plotGroup = svg.append("g") .attr("transform", `translate(${dimensions.left}, ${dimensions.top})`); plotGroup.append("rect") .attr("stroke", "none") .attr("fill", "white") .attr("height", dimensions.innerHeight) .attr("width", dimensions.innerWidth); const xAxis = d3.axisBottom(xScale).ticks(dimensions.width / 80).tickSizeOuter(0); plotGroup.append("g").attr("class", "x-axis").attr("transform", `translate(0, ${dimensions.innerHeight})`).call(xAxis); const yAxis = d3.axisLeft(yScale); plotGroup.append("g").attr("class", "y-axis").call(yAxis); const lineGenerator = d3.line().x(d => xScale(d.x)).y(d => yScale(d.y)); plotGroup.append("path").datum(sorted_data).attr("class", "main").attr("d", lineGenerator); plotGroup.append("path").datum(moving(sorted_data)).attr("class", "moving").attr("d", lineGenerator); const cData = cusum(sorted_data); const cScale = d3.scaleLinear(d3.extent(cData.map(d => d.y)), [dimensions.innerHeight, 0]); const cGenerator = d3.line().x(d => xScale(d.x)).y(d => cScale(d.y)); plotGroup.append("path").datum(cusum(sorted_data)).attr("class", "cusum").attr("d", cGenerator); function redraw() { const t = d3.transition().duration(1500).ease(d3.easeLinear); d3.select("path.main").transition(t).attr("d", lineGenerator.x(d => xScale(d.x))); d3.select("path.moving").transition(t).attr("d", lineGenerator.x(d => xScale(d.x))); d3.select("path.cusum").transition(t).attr("d", cGenerator.x(d => xScale(d.x))); d3.selectAll("circle.marker").transition(t).attr("cx", d => xScale(d.x)); d3.select(".x-axis").transition(t).call(xAxis); } const brush = d3.brushX().on("end", detail); plotGroup.call(brush); function detail(event) { const [x0, x1] = event.selection; // console.log("newbrush", event.selection); // if (event.selection !== null) { xScale.domain([xScale.invert(x0), xScale.invert(x1)]); // } else { // xScale.domain(xDomain); // } redraw(); plotGroup.call(brush.move, null); // remove brush after zooming } // has to be after brush for both to function plotGroup.selectAll("circle") .data(sorted_data) .join("circle") .attr("class", "marker") .attr("cx", d => xScale(d.x)) .attr("cy", d => yScale(d.y)) .on("mouseover", event => { tooltip.style("visibility", "visible") }) .on("mousemove", (event, d) => { const [mouse_x, mouse_y] = d3.pointer(event); tooltip .text(`${d3.timeFormat("%x")(d.x)}, ${d.y}`) .style("left", `${mouse_x+dimensions.left}px`) .style("top", `${mouse_y+dimensions.top}px`); }) .on("mouseout", event => { tooltip.style("visibility", "hidden") }) // reset zoom on click plotGroup.on("click", function(event) { xScale.domain(xDomain); redraw(); }); return div.node(); }34:[{"id":"774660f8a7ee7dda17e47216e0472a73d3ececa40a3e1df486ca2603a684cc185fce107f50567caa1b6502a11e41f449d449545c614e477bf3f9328a15ce67b6","href":"https://static.observableusercontent.com/files/774660f8a7ee7dda17e47216e0472a73d3ececa40a3e1df486ca2603a684cc185fce107f50567caa1b6502a11e41f449d449545c614e477bf3f9328a15ce67b6","name":"oscar_2@4.json","type":"application/json","lastModified":1651885833328.757,"size":6268401},{"id":"62a939d4becf1fc938a8e4543b5f192a9bb88beaa52d60453e9b5791b5a9afba062dc7160a251ee757e8e1395700fa91709dab8ba67d8a8f4139148b74d3eeff","href":"https://static.observableusercontent.com/files/62a939d4becf1fc938a8e4543b5f192a9bb88beaa52d60453e9b5791b5a9afba062dc7160a251ee757e8e1395700fa91709dab8ba67d8a8f4139148b74d3eeff","name":"o2ring@2","type":"application/octet-stream","lastModified":1645661124833.619,"size":1571885},{"id":"3763a273d09015f35e25f3ae7619af514bfa730280bcf960b572f69c9c831fdb0c4b9c425ae5027b794a36f06f66c21364fbde9ba730271b3267fbc2bd425436","href":"https://static.observableusercontent.com/files/3763a273d09015f35e25f3ae7619af514bfa730280bcf960b572f69c9c831fdb0c4b9c425ae5027b794a36f06f66c21364fbde9ba730271b3267fbc2bd425436","name":"oscar_summary@3.csv","type":"text/csv","lastModified":1651885856373.466,"size":116817},{"id":"701d819ee64c313f65e91b8a99443a7256996b66367e2efb760a3e9e8f5e99ef5eab0a9f43b7f0e751dbc1a8474a9aa8898cc650140cf600588fb3aa9765344b","href":"https://static.observableusercontent.com/files/701d819ee64c313f65e91b8a99443a7256996b66367e2efb760a3e9e8f5e99ef5eab0a9f43b7f0e751dbc1a8474a9aa8898cc650140cf600588fb3aa9765344b","name":"oura@3.json","type":"application/json","lastModified":1651885867311.796,"size":4256378},{"id":"c52f8707a2c10f0c4fcd66d5d5a41520867ceb5d05234cc9ac4a8af7765f0e71fc2e90f1e79b491c411b9c6a40a6bb000610d75cb7604d4e0d919fa5caf3c807","href":"https://static.observableusercontent.com/files/c52f8707a2c10f0c4fcd66d5d5a41520867ceb5d05234cc9ac4a8af7765f0e71fc2e90f1e79b491c411b9c6a40a6bb000610d75cb7604d4e0d919fa5caf3c807","name":"sleep_meta@5.json","type":"application/json","lastModified":1651885843789.586,"size":415652}] 2d:{"title":"Sleep Visualization","cells":[{"id":0,"value":"$2f","pinned":false,"hidden":false,"mode":"md"},{"id":4496,"value":"jstat = require('jstat')","pinned":true,"hidden":false,"mode":"ojs"},{"id":4494,"value":"","pinned":true,"hidden":false,"mode":"ojs"},{"id":3950,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":493,"value":"## Event Aggregation ##","pinned":false,"hidden":false,"mode":"md"},{"id":506,"value":"function event_filter(dt) {\n return e => (\n e.Event == \"Hypopnea\" || e.Event == \"ClearAirway\" || e.Event == \"Obstructive\"\n ) && e.DeltaTime >= 0 && e.DeltaTime < dt\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":865,"value":"early_events = {\n const out = [];\n\n [1, 2, 3, 4].forEach(n => {\n const rollup = d3.rollup(data.oscar_events, v => v.filter(event_filter(n)).length, e => e.FloorDate);\n Array.from(rollup, ([key, value]) => ({\"d\": key, \"n\":value})).forEach(e => {\n out.push({\n \"d\": e.d,\n \"n\": e.n,\n \"deep\": (data.sleep_meta_map[e.d] || {}).deep,\n \"threshold\": n\n });\n });\n });\n\n return out\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":690,"value":"## Overlay Plot ##\n\n* https://github.com/observablehq/plot/blob/main/README.md","pinned":false,"hidden":false,"mode":"md"},{"id":982,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":1258,"value":"plot_overlay = function(y, m, d) {\n const meta = get_overlay_meta(y, m, d);\n \n return Plot.plot({\n y: {domain: meta.sort_order},\n marks: [\n Plot.tickX(\n meta.combined, \n {\n x: \"time\", \n y: \"event\", \n strokeWidth: \"width\", \n stroke: \"color\"\n }\n )\n ],\n marginLeft: 100,\n width: 800\n }) \n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1355,"value":"viewof overlay_date = Inputs.date({label: \"Start\", value: \"2021-12-19\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1357,"value":"plot_overlay(overlay_date)","pinned":false,"hidden":false,"mode":"ojs"},{"id":929,"value":"best_deep = {\n const _oscar_rollup = d3.rollup(data.oscar_events, v => v.length, d => d.FloorDate)\n return Object.values(data.sleep_meta_map)\n .filter(x => _oscar_rollup.get(x.Date) !== undefined)\n .slice()\n .sort((a, b) => d3.descending(a.deep, b.deep))\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1329,"value":"viewof best_selection = Inputs.range([0, best_deep.length - 1], {value: 0, label: \"Amount\", step: 1})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1337,"value":"plot_overlay(best_deep[best_selection].Date)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1522,"value":"## Everything Plot ##","pinned":false,"hidden":false,"mode":"md"},{"id":2813,"value":"// https://observablehq.com/@sorig/console-log-without-leaving-your-notebook\nimport {log, logTail} from '@sorig/console-log-without-leaving-your-notebook'","pinned":false,"hidden":false,"mode":"ojs"},{"id":2814,"value":"logTail(5)","pinned":true,"hidden":false,"mode":"ojs"},{"id":2992,"value":"get_overlay_meta(2021, 12, 19)","pinned":true,"hidden":false,"mode":"ojs"},{"id":3440,"value":"## TODO ##\n\n* Cleanup\n* Documentation\n* Fall back on plot_overlay if we don't have O2Ring data. Clear plot_overlay section.","pinned":false,"hidden":false,"mode":"md"},{"id":2233,"value":"viewof everything_date = Inputs.date({label: \"Start\", value: \"2021-12-19\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1529,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":2173,"value":"keyDate = new Date(2021, 11, 19, -5, 0, 0)","pinned":true,"hidden":false,"mode":"ojs"},{"id":3325,"value":"new Date(keyDate.getTime() + 5 * 60 * 1000)","pinned":true,"hidden":false,"mode":"ojs"},{"id":3349,"value":"keyDate + 5 * 60 * 1000","pinned":true,"hidden":false,"mode":"ojs"},{"id":2246,"value":"d3.timeFormat(\"%x\")(keyDate)","pinned":true,"hidden":false,"mode":"ojs"},{"id":2353,"value":"debugMeta = get_overlay_meta(new Date(2021, 11, 19, -5, 0, 0));","pinned":true,"hidden":false,"mode":"ojs"},{"id":2354,"value":"(debugMeta.sleep_event[5].time - debugMeta.sleep_start) / (debugMeta.sleep_stop - debugMeta.sleep_start)","pinned":true,"hidden":false,"mode":"ojs"},{"id":2362,"value":"debugMeta.sleep_stop - debugMeta.sleep_start","pinned":true,"hidden":false,"mode":"ojs"},{"id":3550,"value":"## Cusum Plots ##","pinned":false,"hidden":false,"mode":"md"},{"id":3557,"value":"trend_fields = {\n const found = {};\n Object.values(data.summary_map).forEach(value => {\n Object.keys(value).filter(k => typeof value[k] === 'number').forEach(k => {\n found[k] = 1;\n });\n });\n return Object.keys(found)\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":3575,"value":"viewof cusum_select = Inputs.select(trend_fields, {label: \"Select one\", value: \"deep\"})","pinned":true,"hidden":false,"mode":"ojs"},{"id":3562,"value":"Plot.plot({\n marks: [\n Plot.line(d3.sort(Object.values(data.summary_map), d => d.Date), {x: \"Date\", y: cusum_select})\n ]\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":3586,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":4037,"value":"## Scatter Plots ##","pinned":false,"hidden":false,"mode":"md"},{"id":4044,"value":"viewof scatter_xvar = Inputs.select(trend_fields, {label: \"X variable\", value: \"AHI\"})","pinned":true,"hidden":false,"mode":"ojs"},{"id":4048,"value":"viewof scatter_yvar = Inputs.select(trend_fields, {label: \"Y variable\", value: \"deep\"})","pinned":true,"hidden":false,"mode":"ojs"},{"id":4052,"value":"Plot.plot({\n marks: [\n Plot.dot(\n d3.sort(Object.values(data.summary_map), d => d.Date), \n {x: scatter_xvar, y: scatter_yvar})\n ]\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":455,"value":"## Event distribution ##","pinned":false,"hidden":false,"mode":"md"},{"id":2421,"value":"{\n const hypnogram_classes = {\"1\": \"Deep\", \"2\": \"Light\", \"3\": \"REM\", \"4\": \"Awake\"};\n const hypnogram_colors = {\"1\": \"royalblue\", \"2\": \"lightsteelblue\", \"3\": \"lightskyblue\", \"4\": \"wheat\"};\n const oura_hypno = [];\n Object.values(data.sleep_meta_map).forEach(v => {\n v.hypnogram_5min.split(\"\").forEach((x, i) => {\n oura_hypno.push({\"time\": i * 5, \"event\": hypnogram_classes[x], \"color\": hypnogram_colors[x]});\n });\n });\n return Plot.plot({\n marks: [\n Plot.rectY(\n oura_hypno.filter(d => d.time <= 9 * 60), \n Plot.binX({y: \"count\"}, {x: \"time\", fill: \"color\"})\n )\n ]\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":301,"value":"Plot.plot({\n marks: [\n Plot.rectY(\n data.oscar_events.filter(e => e.InAHI && e.DeltaTime >= 0 && e.DeltaTime < 10), \n Plot.binX({y: \"count\"}, {x: \"DeltaTime\", fill: \"Event\"})\n )\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":306,"value":"{\n const _data = data.oscar_events.filter(e => e.InAHI && e.DeltaTime >= 0 && e.DeltaTime < 10); // oscar_events\n return Plot.plot({\n grid: true,\n facet: {\n data: _data,\n y: \"Event\"\n },\n marks: [\n Plot.rectY(_data, Plot.binX({y: \"count\"}, {x: \"DeltaTime\", fill: \"Event\"})),\n Plot.ruleY([0])\n ]\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4387,"value":"## Event Clustering ##","pinned":false,"hidden":false,"mode":"md"},{"id":4399,"value":"viewof event_bin_count = Inputs.range([0, 20], {label: \"Number of event bins\", step: 1})","pinned":true,"hidden":false,"mode":"ojs"},{"id":4406,"value":"grouped_by_ahi = (event_bin_count > 0 ? d3.bin().thresholds(event_bin_count) : d3.bin()).value(d => d.AHI)(\n data.event_spacing.filter(d => d.dt < 100)\n )","pinned":false,"hidden":false,"mode":"ojs"},{"id":4414,"value":"ahi_quantile = {\n const out = [];\n d3.range(0, 1, 0.01).forEach(p => {\n grouped_by_ahi.forEach((d, i) => {\n out.push({pct: p, bin: i, y: d3.quantile(d.map(_d => _d.dt), p)});\n });\n });\n return out\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4438,"value":"binned_by_ahi = {\n const out = [];\n grouped_by_ahi.forEach((values, index) => {\n values.forEach(v => {\n v.AHI_Bin = index;\n out.push(v);\n });\n });\n return out\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4421,"value":"Plot.plot({\n marks: [\n Plot.line(ahi_quantile, {x: \"pct\", y: \"y\", stroke: \"bin\"}),\n ],\n color: {type: \"linear\", scheme: \"turbo\", legend: true },\n y: {grid: true},\n x: {grid: true}\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4428,"value":"Plot.plot({\n marks: [\n Plot.line(binned_by_ahi, Plot.binX({y: \"sum\"}, {x: \"dt\", stroke: \"AHI_Bin\"}))\n ],\n color: {type: \"linear\", scheme: \"turbo\", legend: true },\n y: {type: \"log\", grid: true},\n x: {grid: true}\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4390,"value":"## Experiments ##","pinned":false,"hidden":false,"mode":"md"},{"id":4033,"value":"Plot.plot({\n grid: true,\n facet: {\n data: early_events,\n y: \"threshold\"\n },\n marks: [\n Plot.dot(early_events, {x: \"n\", y: \"deep\"}),\n Plot.ruleY([0])\n ],\n height: 1000\n})","pinned":true,"hidden":false,"mode":"ojs"},{"id":863,"value":"","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W34","stdlib":"1","titling":"auto"}