30:T758,learnersBySessions = learners.map((l) => l .reduce((acc, current) => { const currentTs = new Date(current.course_progress_ts).getTime(); if (acc.length === 0) { return [[current]]; } const lastSession = acc[acc.length - 1]; const lastTs = new Date( lastSession[lastSession.length - 1].course_progress_ts ).getTime(); const timeDiff = currentTs - lastTs; const thirtyMinutesMs = 30 * 60 * 1000; if (timeDiff <= thirtyMinutesMs) { lastSession.push(current); return acc; } else { return [...acc, [current]]; } }, []) .map((s) => ({ duration: new Date(s[s.length - 1].course_progress_ts).getTime() - new Date(s[0].course_progress_ts).getTime(), averageGrade: d3.mean( s .map((d) => parseFloat(d.courseItemData.grade?.course_item_grade_overall) ) .filter((d) => d) ), averageProgress: d3.mean( s .map((d) => d.courseItemData.grade ? parseFloat(d.courseItemData.grade?.course_item_grade_overall) : parseInt(d.course_progress_state_type_id) - 1 ) .filter((d) => d) ), grades: s .map((d) => parseFloat(d.courseItemData.grade?.course_item_grade_overall) ) .filter((d) => d), progress: s .map((d) => d.courseItemData.grade ? parseFloat(d.courseItemData.grade?.course_item_grade_overall) : parseInt(d.course_progress_state_type_id) - 1 ) .filter((d) => d), expectation: s.reduce( (a, v) => (a += parseInt(v.courseItemData.item.global_item_time_commitment)), 0 ), events: s })) .map((s) => ({ ...s, delta: s.expectation - s.duration })) )31:T1a76,lessonDBScan = learnersByLessons .map((student, ii) => { return student.map((lesson) => { // Sort the lesson's checkpoints by timestamp const sortedLesson = [...lesson].sort( (a, b) => new Date(a.course_progress_ts) - new Date(b.course_progress_ts) ); const allTimestamps = sortedLesson.map( (d) => new Date(d.course_progress_ts) ); // Calculate intervals between consecutive timestamps const intervals = allTimestamps .slice(1) .map((ts, i) => ts - allTimestamps[i]); // If no intervals (only one timestamp), return null metrics if (intervals.length === 0) { return { studentId: sortedLesson[0].illinois_tech_user_id, totalTimestamps: allTimestamps.length, totalLessons: student.length, intervals: [], clusters: [], coefficientOfVariation: null, pacingPattern: "insufficient data" }; } // Calculate mean and standard deviation for coefficient of variation const mean = intervals.reduce((a, b) => a + b, 0) / intervals.length; const variance = intervals.reduce( (sum, interval) => sum + Math.pow(interval - mean, 2), 0 ) / intervals.length; const stdDev = Math.sqrt(variance); const coefficientOfVariation = mean > 0 ? stdDev / mean : null; const eps = stdDev * 0.5; // epsilon: maximum distance between points in same cluster const minPts = 2; // minimum points to form a cluster const clusters = []; const visited = new Array(intervals.length).fill(false); const noise = new Array(intervals.length).fill(false); intervals.forEach((interval, idx) => { if (visited[idx]) return; visited[idx] = true; // Find neighbors within eps distance const neighbors = intervals .map((int, i) => (Math.abs(int - interval) <= eps ? i : -1)) .filter((i) => i !== -1); if (neighbors.length < minPts) { noise[idx] = true; } else { // Create new cluster const cluster = { id: clusters.length, points: [], intervals: [] }; // Expand cluster const toProcess = [...neighbors]; while (toProcess.length > 0) { const pointIdx = toProcess.shift(); if (!visited[pointIdx]) { visited[pointIdx] = true; const pointNeighbors = intervals .map((int, i) => Math.abs(int - intervals[pointIdx]) <= eps ? i : -1 ) .filter((i) => i !== -1); if (pointNeighbors.length >= minPts) { toProcess.push(...pointNeighbors.filter((n) => !visited[n])); } } if (!cluster.points.includes(pointIdx)) { cluster.points.push(pointIdx); cluster.intervals.push(intervals[pointIdx]); } } clusters.push(cluster); } }); // Calculate cluster statistics const clusterStats = clusters.map((cluster) => ({ clusterId: cluster.id, size: cluster.points.length, meanInterval: cluster.intervals.reduce((a, b) => a + b, 0) / cluster.intervals.length, minInterval: Math.min(...cluster.intervals), maxInterval: Math.max(...cluster.intervals), // Convert to readable time units meanIntervalHours: cluster.intervals.reduce((a, b) => a + b, 0) / cluster.intervals.length / (1000 * 60 * 60) })); const sortedIntervals = [...intervals].sort((a, b) => a - b); const p25 = sortedIntervals[Math.floor(sortedIntervals.length * 0.25)]; const p50 = sortedIntervals[Math.floor(sortedIntervals.length * 0.5)]; const p75 = sortedIntervals[Math.floor(sortedIntervals.length * 0.75)]; const iqr = p75 - p25; const outlierThresholdLow = p25 - 1.5 * iqr; const outlierThresholdHigh = p75 + 1.5 * iqr; const outliers = intervals .map((interval, idx) => ({ index: idx, interval, hoursGap: interval / (1000 * 60 * 60), isOutlier: interval < outlierThresholdLow || interval > outlierThresholdHigh })) .filter((item) => item.isOutlier); const totalDuration = allTimestamps[allTimestamps.length - 1] - allTimestamps[0]; // Ensure minimum of 1 day to avoid division issues const totalDurationDays = Math.max( 1, totalDuration / (1000 * 60 * 60 * 24) ); const averageEventsPerDay = allTimestamps.length / totalDurationDays; const outcome = learnerGrades[ sortedLesson[0].illinois_tech_user_id + "-" + sortedLesson[0].courseIndex ]; return { studentId: sortedLesson[0].illinois_tech_user_id, outcome, totalTimestamps: allTimestamps.length, totalLessons: student.length, courseDurationDays: totalDurationDays, averageEventsPerDay, intervals, intervalsInHours: intervals.map((i) => i / (1000 * 60 * 60)), meanIntervalHours: mean / (1000 * 60 * 60), medianIntervalHours: p50 / (1000 * 60 * 60), coefficientOfVariation, quartiles: { p25Hours: p25 / (1000 * 60 * 60), p50Hours: p50 / (1000 * 60 * 60), p75Hours: p75 / (1000 * 60 * 60), iqrHours: iqr / (1000 * 60 * 60) }, clusterCount: clusters.length, noisePointCount: noise.filter((n) => n).length, clusters: clusterStats, outlierCount: outliers.length, outlierPercentage: (outliers.length / intervals.length) * 100, largestGapHours: Math.max(...intervals) / (1000 * 60 * 60), smallestGapHours: Math.min(...intervals) / (1000 * 60 * 60), pacingPattern: coefficientOfVariation > 1.5 ? "severely irregular" : coefficientOfVariation > 1 ? "highly irregular" : coefficientOfVariation > 0.5 ? "moderately irregular" : coefficientOfVariation !== null ? "regular" : "insufficient data", hasBingeSessions: clusters.some( (c) => c.meanIntervalHours < 1 && c.size > 3 ), hasLongBreaks: outliers.some((o) => o.hoursGap > 168), // breaks > 1 week consistency: stdDev / mean < 0.3 ? "highly consistent" : stdDev / mean < 0.7 ? "moderate consistency" : "low consistency" }; }); }) .flat(Infinity)32:T1806,studentDBScan = learnersByLessons.map((student, ii) => { const allTimestamps = student .flat() .map((d) => new Date(d.course_progress_ts)) .sort((a, b) => a - b); // Calculate intervals between consecutive timestamps const intervals = allTimestamps .slice(1) .map((ts, i) => ts - allTimestamps[i]); // If no intervals (only one timestamp), return null metrics if (intervals.length === 0) { return { studentId: student[0][0].illinois_tech_user_id, totalTimestamps: allTimestamps.length, totalLessons: student.length, intervals: [], clusters: [], coefficientOfVariation: null, pacingPattern: "insufficient data" }; } // Calculate mean and standard deviation for coefficient of variation const mean = intervals.reduce((a, b) => a + b, 0) / intervals.length; const variance = intervals.reduce((sum, interval) => sum + Math.pow(interval - mean, 2), 0) / intervals.length; const stdDev = Math.sqrt(variance); const coefficientOfVariation = mean > 0 ? stdDev / mean : null; // DBSCAN-like clustering parameters const eps = stdDev * 0.5; // epsilon: maximum distance between points in same cluster const minPts = 2; // minimum points to form a cluster // Simple DBSCAN implementation for 1D intervals const clusters = []; const visited = new Array(intervals.length).fill(false); const noise = new Array(intervals.length).fill(false); intervals.forEach((interval, idx) => { if (visited[idx]) return; visited[idx] = true; // Find neighbors within eps distance const neighbors = intervals .map((int, i) => (Math.abs(int - interval) <= eps ? i : -1)) .filter((i) => i !== -1); if (neighbors.length < minPts) { noise[idx] = true; } else { // Create new cluster const cluster = { id: clusters.length, points: [], intervals: [] }; // Expand cluster const toProcess = [...neighbors]; while (toProcess.length > 0) { const pointIdx = toProcess.shift(); if (!visited[pointIdx]) { visited[pointIdx] = true; const pointNeighbors = intervals .map((int, i) => Math.abs(int - intervals[pointIdx]) <= eps ? i : -1 ) .filter((i) => i !== -1); if (pointNeighbors.length >= minPts) { toProcess.push(...pointNeighbors.filter((n) => !visited[n])); } } if (!cluster.points.includes(pointIdx)) { cluster.points.push(pointIdx); cluster.intervals.push(intervals[pointIdx]); } } clusters.push(cluster); } }); // Calculate cluster statistics const clusterStats = clusters.map((cluster) => ({ clusterId: cluster.id, size: cluster.points.length, meanInterval: cluster.intervals.reduce((a, b) => a + b, 0) / cluster.intervals.length, minInterval: Math.min(...cluster.intervals), maxInterval: Math.max(...cluster.intervals), // Convert to readable time units meanIntervalHours: cluster.intervals.reduce((a, b) => a + b, 0) / cluster.intervals.length / (1000 * 60 * 60) })); // Calculate percentiles for additional insights const sortedIntervals = [...intervals].sort((a, b) => a - b); const p25 = sortedIntervals[Math.floor(sortedIntervals.length * 0.25)]; const p50 = sortedIntervals[Math.floor(sortedIntervals.length * 0.5)]; const p75 = sortedIntervals[Math.floor(sortedIntervals.length * 0.75)]; const iqr = p75 - p25; // Identify outlier intervals (potential pacing disruptions) const outlierThresholdLow = p25 - 1.5 * iqr; const outlierThresholdHigh = p75 + 1.5 * iqr; const outliers = intervals .map((interval, idx) => ({ index: idx, interval, hoursGap: interval / (1000 * 60 * 60), isOutlier: interval < outlierThresholdLow || interval > outlierThresholdHigh })) .filter((item) => item.isOutlier); // Calculate total course duration with minimum of 1 day const totalDuration = allTimestamps[allTimestamps.length - 1] - allTimestamps[0]; // Ensure minimum of 1 day to avoid division issues const totalDurationDays = Math.max(1, totalDuration / (1000 * 60 * 60 * 24)); const averageEventsPerDay = allTimestamps.length / totalDurationDays; const outcome = learnerGrades[ student[0][0].illinois_tech_user_id + "-" + student[0][0].courseIndex ]; return { studentId: student[0][0].illinois_tech_user_id, outcome, totalTimestamps: allTimestamps.length, totalLessons: student.length, courseDurationDays: totalDurationDays, averageEventsPerDay, intervals, intervalsInHours: intervals.map((i) => i / (1000 * 60 * 60)), meanIntervalHours: mean / (1000 * 60 * 60), medianIntervalHours: p50 / (1000 * 60 * 60), coefficientOfVariation, quartiles: { p25Hours: p25 / (1000 * 60 * 60), p50Hours: p50 / (1000 * 60 * 60), p75Hours: p75 / (1000 * 60 * 60), iqrHours: iqr / (1000 * 60 * 60) }, clusterCount: clusters.length, noisePointCount: noise.filter((n) => n).length, clusters: clusterStats, outlierCount: outliers.length, outlierPercentage: (outliers.length / intervals.length) * 100, largestGapHours: Math.max(...intervals) / (1000 * 60 * 60), smallestGapHours: Math.min(...intervals) / (1000 * 60 * 60), // Pacing diagnosis pacingPattern: coefficientOfVariation > 1.5 ? "severely irregular" : coefficientOfVariation > 1 ? "highly irregular" : coefficientOfVariation > 0.5 ? "moderately irregular" : coefficientOfVariation !== null ? "regular" : "insufficient data", // Additional diagnostic flags hasBingeSessions: clusters.some( (c) => c.meanIntervalHours < 1 && c.size > 3 ), hasLongBreaks: outliers.some((o) => o.hoursGap > 168), // breaks > 1 week consistency: stdDev / mean < 0.3 ? "highly consistent" : stdDev / mean < 0.7 ? "moderate consistency" : "low consistency" }; })33:T14d8,{ // Convert timestamps to Date objects and calculate intervals const processedData = learnersByLessons.map((student) => ({ ...student, lessons: student.map((lesson) => { const timestamps = lesson.map( (item) => new Date(item.course_progress_ts) ); timestamps.sort((a, b) => a - b); const intervals = []; for (let i = 1; i < timestamps.length; i++) { intervals.push((timestamps[i] - timestamps[i - 1]) / (1000 * 60)); } return { timestamps, intervals }; }) })); // Define pacing states based on interval thresholds (in minutes) const getState = (interval) => { if (interval < 1) return "rapid"; // less than 1 hour if (interval < 24) return "normal"; // 1-24 hours if (interval < 168) return "slow"; // 1-7 days return "stalled"; // more than a week }; // Convert intervals to state sequences for each student const stateSequences = processedData.map((student) => ({ studentId: student[0][0].illinois_tech_user_id, sequences: student.lessons .map((lesson) => lesson.intervals.map((interval) => getState(interval))) .flat() })); // Build transition count matrix const states = ["rapid", "normal", "slow", "stalled"]; const transitionCounts = {}; states.forEach((from) => { transitionCounts[from] = {}; states.forEach((to) => { transitionCounts[from][to] = 0; }); }); // Count all transitions across all students stateSequences.forEach((student) => { const seq = student.sequences; for (let i = 0; i < seq.length - 1; i++) { transitionCounts[seq[i]][seq[i + 1]]++; } }); // Convert counts to probabilities const transitionMatrix = {}; states.forEach((from) => { const totalTransitions = Object.values(transitionCounts[from]).reduce( (a, b) => a + b, 0 ); transitionMatrix[from] = {}; states.forEach((to) => { transitionMatrix[from][to] = totalTransitions > 0 ? transitionCounts[from][to] / totalTransitions : 0; }); }); // Calculate steady-state distribution (eigenvector approach) const iterations = 100; let steadyState = states.reduce((acc, state) => { acc[state] = 0.25; // initial uniform distribution return acc; }, {}); for (let iter = 0; iter < iterations; iter++) { const newState = {}; states.forEach((to) => { newState[to] = 0; states.forEach((from) => { newState[to] += steadyState[from] * transitionMatrix[from][to]; }); }); steadyState = newState; } // Calculate per-student metrics const studentMetrics = stateSequences.map((student) => { const seq = student.sequences; // Calculate state distribution for this student const stateCount = {}; states.forEach((s) => (stateCount[s] = 0)); seq.forEach((s) => stateCount[s]++); const stateDistribution = {}; const total = seq.length; states.forEach((s) => { stateDistribution[s] = total > 0 ? stateCount[s] / total : 0; }); // Calculate entropy of student's pacing const entropy = -Object.values(stateDistribution) .filter((p) => p > 0) .reduce((sum, p) => sum + p * Math.log2(p), 0); // Calculate deviation from steady state const klDivergence = states.reduce((sum, state) => { const p = stateDistribution[state] || 0.001; const q = steadyState[state] || 0.001; return sum + p * Math.log(p / q); }, 0); // Count self-transitions (staying in same state) let selfTransitions = 0; for (let i = 0; i < seq.length - 1; i++) { if (seq[i] === seq[i + 1]) selfTransitions++; } const selfTransitionRate = seq.length > 1 ? selfTransitions / (seq.length - 1) : 0; return { studentId: student.studentId, stateDistribution, entropy, klDivergence, selfTransitionRate, sequenceLength: seq.length, dominantState: Object.entries(stateDistribution).sort((a, b) => b[1] - a[1])[0]?.[0] || "unknown" }; }); // Identify potentially disordered pacing const disorderThresholds = { highEntropy: studentMetrics.map((m) => m.entropy).sort((a, b) => b - a)[ Math.floor(studentMetrics.length * 0.2) ], highDivergence: studentMetrics .map((m) => m.klDivergence) .sort((a, b) => b - a)[Math.floor(studentMetrics.length * 0.2)] }; const pacingIrregularity = studentMetrics .filter( (student) => student.entropy > disorderThresholds.highEntropy || student.klDivergence > disorderThresholds.highDivergence ) .map((s) => s.studentId); // Final results return { transitionMatrix, steadyState, studentMetrics: studentMetrics.sort((a, b) => b.entropy - a.entropy), pacingIrregularity, summary: { avgEntropy: studentMetrics.reduce((sum, s) => sum + s.entropy, 0) / studentMetrics.length, avgSelfTransitionRate: studentMetrics.reduce((sum, s) => sum + s.selfTransitionRate, 0) / studentMetrics.length, statePrevalence: states.map((state) => ({ state, prevalence: studentMetrics.reduce( (sum, s) => sum + s.stateDistribution[state], 0 ) / studentMetrics.length })) } }; }34:T2d9f,latentMarkov = { // Load data and convert timestamps const processedData = learnersByLessons.map((student) => student.map((lesson) => lesson.map((item) => ({ ...item, timestamp: new Date(item.course_progress_ts) })) ) ); // Extract all inter-event intervals for each student const studentIntervals = processedData.map((student) => { const allEvents = student.flat().sort((a, b) => a.timestamp - b.timestamp); const intervals = []; for (let i = 1; i < allEvents.length; i++) { intervals.push( (allEvents[i].timestamp - allEvents[i - 1].timestamp) / (1000 * 60 * 60) ); // hours } return intervals; }); // Define pacing states based on interval quartiles const allIntervals = studentIntervals.flat(); const sortedIntervals = [...allIntervals].sort((a, b) => a - b); const q1 = sortedIntervals[Math.floor(sortedIntervals.length * 0.25)]; const q2 = sortedIntervals[Math.floor(sortedIntervals.length * 0.5)]; const q3 = sortedIntervals[Math.floor(sortedIntervals.length * 0.75)]; // Map intervals to discrete states const states = ["rapid", "normal", "slow", "stalled"]; const getState = (interval) => { if (interval < q1) return 0; // rapid if (interval < q2) return 1; // normal if (interval < q3) return 2; // slow return 3; // stalled }; // Convert intervals to state sequences for each student const studentStateSequences = studentIntervals.map((intervals) => intervals.map(getState) ); // Initialize transition matrix with pseudocounts (Laplace smoothing) const pseudocount = 0.1; // Small value to avoid zeros const transitionCounts = Array(4) .fill() .map(() => Array(4).fill(pseudocount)); const stateCounts = Array(4).fill(pseudocount * 4); // Account for pseudocounts in denominator // Count transitions - fixing the counting logic studentStateSequences.forEach((sequence) => { for (let i = 0; i < sequence.length - 1; i++) { const fromState = sequence[i]; const toState = sequence[i + 1]; transitionCounts[fromState][toState]++; stateCounts[fromState]++; } }); // Normalize to get transition probabilities const transitionMatrix = transitionCounts.map((row, i) => row.map((count) => count / stateCounts[i]) ); // Debug: Check state distribution const stateDistribution = Array(4).fill(0); studentStateSequences.forEach((sequence) => { sequence.forEach((state) => { stateDistribution[state]++; }); }); const totalStates = stateDistribution.reduce((a, b) => a + b, 0); const stateProportions = stateDistribution.map( (count) => count / totalStates ); // Baum-Welch algorithm for HMM (simplified version) const numHiddenStates = 3; // underlying cognitive states const numObservableStates = 4; // pacing states we observe // Initialize HMM parameters with slight bias toward self-transitions let hiddenTransitions = Array(numHiddenStates) .fill() .map((_, i) => { const row = Array(numHiddenStates) .fill() .map((_, j) => i === j ? 0.5 + Math.random() * 0.3 : Math.random() * 0.3 ); const sum = row.reduce((a, b) => a + b, 0); return row.map((v) => v / sum); }); // Initialize emissions with some structure let emissionProbs = Array(numHiddenStates) .fill() .map((_, hiddenState) => { const row = Array(numObservableStates) .fill() .map((_, obsState) => { // Bias emissions: hidden state 0 -> rapid/normal, 1 -> normal/slow, 2 -> slow/stalled const bias = Math.exp(-Math.abs(hiddenState - obsState * 0.75)); return bias + Math.random() * 0.5; }); const sum = row.reduce((a, b) => a + b, 0); return row.map((v) => v / sum); }); let initialProbs = Array(numHiddenStates) .fill() .map(() => Math.random()); initialProbs = initialProbs.map( (v) => v / initialProbs.reduce((a, b) => a + b, 0) ); // Run EM iterations with regularization const numIterations = 50; const allSequences = studentStateSequences.filter((seq) => seq.length > 1); // Need at least 2 states for (let iter = 0; iter < numIterations; iter++) { // E-step accumulators with pseudocounts const transAccum = Array(numHiddenStates) .fill() .map(() => Array(numHiddenStates).fill(1e-5)); const emissAccum = Array(numHiddenStates) .fill() .map(() => Array(numObservableStates).fill(1e-5)); const initAccum = Array(numHiddenStates).fill(1e-5); let totalSeqs = 0; allSequences.forEach((sequence) => { if (sequence.length === 0) return; // Forward pass with scaling to prevent underflow const T = sequence.length; const alpha = Array(T) .fill() .map(() => Array(numHiddenStates).fill(0)); const scales = Array(T).fill(0); // Initialize for (let i = 0; i < numHiddenStates; i++) { alpha[0][i] = initialProbs[i] * emissionProbs[i][sequence[0]]; } scales[0] = 1 / (alpha[0].reduce((a, b) => a + b, 0) + 1e-10); for (let i = 0; i < numHiddenStates; i++) { alpha[0][i] *= scales[0]; } // Forward recursion for (let t = 1; t < T; t++) { for (let j = 0; j < numHiddenStates; j++) { for (let i = 0; i < numHiddenStates; i++) { alpha[t][j] += alpha[t - 1][i] * hiddenTransitions[i][j]; } alpha[t][j] *= emissionProbs[j][sequence[t]]; } scales[t] = 1 / (alpha[t].reduce((a, b) => a + b, 0) + 1e-10); for (let j = 0; j < numHiddenStates; j++) { alpha[t][j] *= scales[t]; } } // Backward pass with scaling const beta = Array(T) .fill() .map(() => Array(numHiddenStates).fill(0)); for (let i = 0; i < numHiddenStates; i++) { beta[T - 1][i] = scales[T - 1]; } for (let t = T - 2; t >= 0; t--) { for (let i = 0; i < numHiddenStates; i++) { for (let j = 0; j < numHiddenStates; j++) { beta[t][i] += hiddenTransitions[i][j] * emissionProbs[j][sequence[t + 1]] * beta[t + 1][j]; } beta[t][i] *= scales[t]; } } // Update accumulators for (let i = 0; i < numHiddenStates; i++) { initAccum[i] += (alpha[0][i] * beta[0][i]) / scales[0]; } for (let t = 0; t < T - 1; t++) { const denominator = alpha[t].map((a, i) => a * beta[t][i]).reduce((a, b) => a + b, 0) + 1e-10; for (let i = 0; i < numHiddenStates; i++) { for (let j = 0; j < numHiddenStates; j++) { const xi = (alpha[t][i] * hiddenTransitions[i][j] * emissionProbs[j][sequence[t + 1]] * beta[t + 1][j]) / denominator; transAccum[i][j] += xi; } } } for (let t = 0; t < T; t++) { const denominator = alpha[t].map((a, i) => a * beta[t][i]).reduce((a, b) => a + b, 0) + 1e-10; for (let i = 0; i < numHiddenStates; i++) { const gamma = (alpha[t][i] * beta[t][i]) / denominator; emissAccum[i][sequence[t]] += gamma; } } totalSeqs++; }); // M-step: update parameters with normalization const initSum = initAccum.reduce((a, b) => a + b, 0); initialProbs = initAccum.map((v) => v / initSum); for (let i = 0; i < numHiddenStates; i++) { const transSum = transAccum[i].reduce((a, b) => a + b, 0); hiddenTransitions[i] = transAccum[i].map((v) => v / transSum); const emissSum = emissAccum[i].reduce((a, b) => a + b, 0); emissionProbs[i] = emissAccum[i].map((v) => v / emissSum); } } // Viterbi algorithm to decode most likely hidden states for each student const decodedSequences = allSequences.map((sequence) => { if (sequence.length === 0) return []; const T = sequence.length; const viterbi = Array(T) .fill() .map(() => Array(numHiddenStates).fill(-Infinity)); const path = Array(T) .fill() .map(() => Array(numHiddenStates).fill(0)); // Initialize (log space) for (let i = 0; i < numHiddenStates; i++) { viterbi[0][i] = Math.log(initialProbs[i] + 1e-10) + Math.log(emissionProbs[i][sequence[0]] + 1e-10); } // Recursion (log space) for (let t = 1; t < T; t++) { for (let j = 0; j < numHiddenStates; j++) { let maxLogProb = -Infinity; let maxState = 0; for (let i = 0; i < numHiddenStates; i++) { const logProb = viterbi[t - 1][i] + Math.log(hiddenTransitions[i][j] + 1e-10); if (logProb > maxLogProb) { maxLogProb = logProb; maxState = i; } } viterbi[t][j] = maxLogProb + Math.log(emissionProbs[j][sequence[t]] + 1e-10); path[t][j] = maxState; } } // Backtrack const states = Array(T); let maxLogProb = -Infinity; let maxState = 0; for (let i = 0; i < numHiddenStates; i++) { if (viterbi[T - 1][i] > maxLogProb) { maxLogProb = viterbi[T - 1][i]; maxState = i; } } states[T - 1] = maxState; for (let t = T - 2; t >= 0; t--) { states[t] = path[t + 1][states[t + 1]]; } return states; }); // Calculate disorder metrics const disorderMetrics = decodedSequences.map((hiddenSeq, idx) => { const observedSeq = allSequences[idx]; // Entropy of hidden state transitions const hiddenTransitionCounts = Array(numHiddenStates) .fill() .map(() => Array(numHiddenStates).fill(0)); for (let i = 0; i < hiddenSeq.length - 1; i++) { hiddenTransitionCounts[hiddenSeq[i]][hiddenSeq[i + 1]]++; } let hiddenEntropy = 0; const totalTransitions = Math.max(hiddenSeq.length - 1, 1); for (let i = 0; i < numHiddenStates; i++) { for (let j = 0; j < numHiddenStates; j++) { if (hiddenTransitionCounts[i][j] > 0) { const p = hiddenTransitionCounts[i][j] / totalTransitions; hiddenEntropy -= p * Math.log2(p); } } } // Switching frequency let switches = 0; for (let i = 1; i < hiddenSeq.length; i++) { if (hiddenSeq[i] !== hiddenSeq[i - 1]) switches++; } const switchRate = hiddenSeq.length > 1 ? switches / (hiddenSeq.length - 1) : 0; return { studentIdx: idx, hiddenEntropy, switchRate, sequenceLength: hiddenSeq.length, dominantState: hiddenSeq.length > 0 ? hiddenSeq.reduce((a, b, i, arr) => arr.filter((v) => v === a).length >= arr.filter((v) => v === b).length ? a : b ) : null }; }); // Results with diagnostics return { observedTransitionMatrix: transitionMatrix, stateDistribution: { counts: stateDistribution, proportions: stateProportions, labels: states }, hiddenStatesModel: { transitions: hiddenTransitions, emissions: emissionProbs, initial: initialProbs }, stateLabels: states, disorderMetrics: disorderMetrics.sort( (a, b) => b.hiddenEntropy - a.hiddenEntropy ), intervalQuartiles: { q1, q2, q3 }, diagnostics: { numStudents: processedData.length, numSequences: allSequences.length, avgSequenceLength: allSequences.reduce((sum, seq) => sum + seq.length, 0) / allSequences.length } }; }35:T1162,// Create the heatmap visualization { const width = 600; const height = 400; const margin = { top: 50, right: 80, bottom: 100, left: 100 }; // Transform emission probabilities into heatmap data const data = []; const hiddenStateLabels = [ "Cognitive State 0", "Cognitive State 1", "Cognitive State 2" ]; const observedStateLabels = ["Rapid", "Normal", "Slow", "Stalled"]; for (let i = 0; i < latentMarkov.hiddenStatesModel.emissions.length; i++) { for ( let j = 0; j < latentMarkov.hiddenStatesModel.emissions[i].length; j++ ) { data.push({ hidden: i, observed: j, value: latentMarkov.hiddenStatesModel.emissions[i][j], hiddenLabel: hiddenStateLabels[i], observedLabel: observedStateLabels[j] }); } } // Create scales const xScale = d3 .scaleBand() .domain(observedStateLabels) .range([margin.left, width - margin.right]) .padding(0.05); const yScale = d3 .scaleBand() .domain(hiddenStateLabels) .range([margin.top, height - margin.bottom]) .padding(0.05); const colorScale = d3 .scaleSequential(d3.interpolateViridis) .domain([0, d3.max(data, (d) => d.value)]); // Create SVG const svg = d3 .create("svg") .attr("width", width) .attr("height", height) .attr("viewBox", [0, 0, width, height]) .style("max-width", "100%"); // Add rectangles svg .selectAll("rect") .data(data) .join("rect") .attr("x", (d) => xScale(d.observedLabel)) .attr("y", (d) => yScale(d.hiddenLabel)) .attr("width", xScale.bandwidth()) .attr("height", yScale.bandwidth()) .attr("fill", (d) => colorScale(d.value)) .attr("stroke", "#fff") .attr("stroke-width", 2); // Add text values svg .selectAll(".value-text") .data(data) .join("text") .attr("class", "value-text") .attr("x", (d) => xScale(d.observedLabel) + xScale.bandwidth() / 2) .attr("y", (d) => yScale(d.hiddenLabel) + yScale.bandwidth() / 2) .attr("text-anchor", "middle") .attr("dominant-baseline", "middle") .attr("fill", (d) => (d.value > 0.5 ? "white" : "black")) .attr("font-size", "14px") .attr("font-weight", "bold") .text((d) => d.value.toFixed(2)); // Add X axis svg .append("g") .attr("transform", `translate(0,${height - margin.bottom})`) .call(d3.axisBottom(xScale)) .append("text") .attr("x", width / 2) .attr("y", 40) .attr("fill", "black") .style("text-anchor", "middle") .style("font-size", "14px") .text("Observed Pacing States"); // Add Y axis svg .append("g") .attr("transform", `translate(${margin.left},0)`) .call(d3.axisLeft(yScale)) .append("text") .attr("transform", "rotate(-90)") .attr("x", -(height / 2)) .attr("y", -60) .attr("fill", "black") .style("text-anchor", "middle") .style("font-size", "14px") .text("Hidden Cognitive States"); // Add title svg .append("text") .attr("x", width / 2) .attr("y", 25) .attr("text-anchor", "middle") .style("font-size", "16px") .style("font-weight", "bold") .text("Emission Probabilities: Hidden States → Observed Pacing"); // Add color legend const legendWidth = 20; const legendHeight = 200; const legendScale = d3 .scaleLinear() .domain(colorScale.domain()) .range([legendHeight, 0]); const legendAxis = d3 .axisRight(legendScale) .ticks(5) .tickFormat(d3.format(".2f")); const legend = svg .append("g") .attr( "transform", `translate(${width - margin.right + 20}, ${margin.top})` ); // Create gradient for legend const gradientId = "legend-gradient"; const gradient = svg .append("defs") .append("linearGradient") .attr("id", gradientId) .attr("x1", "0%") .attr("x2", "0%") .attr("y1", "100%") .attr("y2", "0%"); const steps = 20; for (let i = 0; i <= steps; i++) { gradient .append("stop") .attr("offset", `${(i / steps) * 100}%`) .attr( "stop-color", colorScale((i / steps) * d3.max(data, (d) => d.value)) ); } legend .append("rect") .attr("width", legendWidth) .attr("height", legendHeight) .style("fill", `url(#${gradientId})`); legend .append("g") .attr("transform", `translate(${legendWidth}, 0)`) .call(legendAxis); return svg.node(); }36:T240d,fourierAnalysis = { // Convert timestamps to Date objects and extract time series const processedData = learnersByLessons.map((student) => ({ ...student, interactionCount: student.flat().length, lessons: student.map((lesson) => lesson.map((item) => ({ ...item, date: new Date(item.course_progress_ts) })) ) })); const count = learnersByLessons.map((student) => { return student.flat().length; }); // Flatten to get all timestamps per student and convert to time intervals const studentIntervals = processedData.map((student) => { const allTimestamps = student.lessons .flat() .map((item) => item.date.getTime()) .sort((a, b) => a - b); const intervals = []; for (let i = 1; i < allTimestamps.length; i++) { intervals.push( (allTimestamps[i] - allTimestamps[i - 1]) / (1000 * 60 * 60) ); // Convert to hours } return { studentId: student.id || student.name, intervals: intervals, timestamps: allTimestamps }; }); // Resample intervals to regular grid for Fourier analysis const resampleToRegularGrid = studentIntervals.map((student) => { if (student.intervals.length < 2) return { ...student, regularSamples: [] }; const totalDuration = (student.timestamps[student.timestamps.length - 1] - student.timestamps[0]) / (1000 * 60 * 60); const numSamples = Math.min(256, student.timestamps.length * 2); // Power of 2 for FFT const sampleRate = totalDuration / numSamples; const regularSamples = new Array(numSamples).fill(0); let currentEventIndex = 0; for (let i = 0; i < numSamples; i++) { const currentTime = student.timestamps[0] + i * sampleRate * 60 * 60 * 1000; while ( currentEventIndex < student.timestamps.length - 1 && student.timestamps[currentEventIndex + 1] <= currentTime ) { currentEventIndex++; } if (currentEventIndex < student.timestamps.length - 1) { const nextInterval = student.timestamps[currentEventIndex + 1] - student.timestamps[currentEventIndex]; regularSamples[i] = 1 / (nextInterval / (1000 * 60 * 60)); // Activity rate (events per hour) } } return { ...student, regularSamples: regularSamples, sampleRate: sampleRate }; }); // Simple FFT implementation const fft = (x) => { const N = x.length; if (N <= 1) return x; if (N % 2 !== 0) throw "N must be power of 2"; const even = fft(x.filter((_, i) => i % 2 === 0)); const odd = fft(x.filter((_, i) => i % 2 === 1)); const result = new Array(N); for (let k = 0; k < N / 2; k++) { const t = (-2 * Math.PI * k) / N; const wk = { re: Math.cos(t), im: Math.sin(t) }; const oddK = { re: wk.re * odd[k].re - wk.im * odd[k].im, im: wk.re * odd[k].im + wk.im * odd[k].re }; result[k] = { re: even[k].re + oddK.re, im: even[k].im + oddK.im }; result[k + N / 2] = { re: even[k].re - oddK.re, im: even[k].im - oddK.im }; } return result; }; // Apply FFT to each student's regular samples const fourierResults = resampleToRegularGrid.map((student) => { if (student.regularSamples.length === 0) return { ...student, spectrum: [], frequencies: [] }; // Pad to nearest power of 2 const nextPow2 = Math.pow( 2, Math.ceil(Math.log2(student.regularSamples.length)) ); const padded = [ ...student.regularSamples, ...new Array(nextPow2 - student.regularSamples.length).fill(0) ]; // Convert to complex numbers for FFT const complexSignal = padded.map((v) => ({ re: v, im: 0 })); const fftResult = fft(complexSignal); // Calculate magnitude spectrum (only positive frequencies) const spectrum = fftResult .slice(0, Math.floor(fftResult.length / 2)) .map((c) => Math.sqrt(c.re * c.re + c.im * c.im)); // Calculate corresponding frequencies (in cycles per day) const frequencies = spectrum.map( (_, i) => i * (1 / (student.sampleRate * spectrum.length * 2)) * 24 ); return { ...student, spectrum: spectrum, frequencies: frequencies }; }); // Identify dominant frequencies for each student const dominantFrequencies = fourierResults.map((student, ii) => { if (student.spectrum.length === 0) return { dominantFreqs: [], regularityScore: 0 }; // Find peaks in spectrum (frequencies with higher amplitude than neighbors) const peaks = []; for (let i = 1; i < student.spectrum.length - 1; i++) { if ( student.spectrum[i] > student.spectrum[i - 1] && student.spectrum[i] > student.spectrum[i + 1] && student.spectrum[i] > Math.max(...student.spectrum) * 0.1 ) { // Threshold peaks.push({ frequency: student.frequencies[i], amplitude: student.spectrum[i], period: student.frequencies[i] > 0 ? 1 / student.frequencies[i] : Infinity }); } } return { dominantFreqs: peaks .sort((a, b) => b.amplitude - a.amplitude) .slice(0, 3), regularityScore: student.spectrum.length > 0 ? Math.max(...student.spectrum) / (student.spectrum.reduce((a, b) => a + b, 0) / student.spectrum.length) : 0 }; }); // Simple Morlet wavelet transform for time-frequency analysis const waveletTransform = studentIntervals.map((student) => { if (student.timestamps.length < 3) return { studentId: student.studentId, wavelet: [] }; const scales = [1, 2, 4, 8, 16, 32, 64]; // Hours const omega0 = 6; // Wavelet parameter const coefficients = scales.map((scale) => { const scaleCoeffs = []; for (let i = 0; i < student.timestamps.length; i++) { let coeff = 0; const t = student.timestamps[i]; for (let j = 0; j < student.timestamps.length; j++) { const tau = (student.timestamps[j] - t) / (1000 * 60 * 60 * scale); const wavelet = Math.exp((-tau * tau) / 2) * Math.cos(omega0 * tau); coeff += wavelet; } scaleCoeffs.push(Math.abs(coeff) / Math.sqrt(scale)); } return scaleCoeffs; }); return { studentId: student.studentId, scales: scales, coefficients: coefficients, timestamps: student.timestamps }; }); // Calculate pacing disorder metrics const pacingMetrics = dominantFrequencies.map((freq, i) => ({ regularityScore: freq.regularityScore, studentId: learners[i][0].illinois_tech_user_id, count: learnersByLessons[i].flat(Infinity).length, outcome: courseGrades[learners[i][0].courseIndex][ learners[i][0].illinois_tech_user_id ]?.course_branch_grade_overall ? parseFloat( courseGrades[learners[i][0].courseIndex][ learners[i][0].illinois_tech_user_id ]?.course_branch_grade_overall ) > 1 ? 1 : parseFloat( courseGrades[learners[i][0].courseIndex][ learners[i][0].illinois_tech_user_id ]?.course_branch_grade_overall ) : 0, fourierResults, dominantFrequencies, waveletTransform, hasDailyPattern: freq.dominantFreqs && freq.dominantFreqs.length > 0 ? freq.dominantFreqs.some((f) => f.period > 0.8 && f.period < 1.2) : false, hasWeeklyPattern: freq.dominantFreqs && freq.dominantFreqs.length > 0 ? freq.dominantFreqs.some((f) => f.period > 6 && f.period < 8) : false, dominantPeriods: freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period) : [], dominantPeriod0: freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period)[0] : [], dominantPeriod1: freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period)[1] : [], dominantPeriod2: freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period)[2] : [], dominantPeriodDifference: Math.abs( (freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period)[0] : 0) - (freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period)[1] : 0) ) + Math.abs( (freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period)[1] : 0) - (freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period)[2] : 0) ) + Math.abs( (freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period)[2] : 0) - (freq.dominantFreqs ? freq.dominantFreqs.map((f) => f.period)[0] : 0) ), intervalVariability: studentIntervals[i].intervals.length > 0 ? Math.sqrt( studentIntervals[i].intervals.reduce((a, b) => a + b * b, 0) / studentIntervals[i].intervals.length - Math.pow( studentIntervals[i].intervals.reduce((a, b) => a + b, 0) / studentIntervals[i].intervals.length, 2 ) ) : 0 })); return pacingMetrics.filter((d) => d.outcome > 0); }37:T579,learnerIntervals = processedLearners.map((student) => { const intervals = []; const timestamps = student.allTimestamps; for (let i = 1; i < timestamps.length; i++) { intervals.push({ hours: (timestamps[i] - timestamps[i - 1]) / (1000 * 60 * 60), startTime: timestamps[i - 1], endTime: timestamps[i] }); } // Extract just the interval lengths (in hours) const intervalHours = intervals.map((d) => d.hours); // Compute mean, standard deviation, coefficient of variation let mean = null, standardDeviation = null, coefficientOfVariation = null; if (intervalHours.length > 0) { mean = intervalHours.reduce((a, b) => a + b, 0) / intervalHours.length; const variance = intervalHours.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / intervalHours.length; standardDeviation = Math.sqrt(variance); coefficientOfVariation = mean !== 0 ? standardDeviation / mean : null; } return { studentId: student.studentId, outcome: student.outcome, timestamps: timestamps, timestampCount: timestamps.length, intervals: intervals, totalDays: timestamps.length > 1 ? (timestamps[timestamps.length - 1] - timestamps[0]) / (1000 * 60 * 60 * 24) : 0, meanInterval: mean, standardDeviation: standardDeviation, coefficientOfVariation: coefficientOfVariation }; })38:T627,velocityMetrics = processedLearners.map((student) => { const lessonProgress = student.lessons .map((lesson) => ({ timestamp: lesson[0]?.timestamp, itemsCompleted: lesson.length })) .filter((l) => l.timestamp); const velocities = []; const decelerations = []; for (let i = 1; i < lessonProgress.length; i++) { const timeDiff = (lessonProgress[i].timestamp - lessonProgress[i - 1].timestamp) / (1000 * 60 * 60); if (timeDiff > 0.05) { const velocity = lessonProgress[i].itemsCompleted / timeDiff; velocities.push(velocity); // Track deceleration points if (i > 0 && velocities.length > 1) { const prevVelocity = velocities[velocities.length - 2]; if (velocity < prevVelocity * 0.5) { decelerations.push({ index: i, timestamp: lessonProgress[i].timestamp, drop: prevVelocity - velocity }); } } } } const mean = velocities.length > 0 ? velocities.reduce((a, b) => a + b, 0) / velocities.length : 0; const variance = velocities.length > 0 ? velocities.reduce((acc, v) => acc + Math.pow(v - mean, 2), 0) / velocities.length : 0; return { studentId: student.studentId, outcome: student.outcome, velocities: velocities, minimumVelocity: d3.min(velocities), maximumVelocity: d3.max(velocities), averageVelocity: mean, velocityVariance: variance, decelerationPoints: decelerations, decelerationCount: decelerations.length }; })39:T43c,// Cell 5: Analyze pause patterns pauseAnalysis = learnerIntervals.map((student) => { const pauseThreshold = 0.25; // 15 minutes in hours const longPauseThreshold = 24; // 1 day in hours const pauses = student.intervals.filter((i) => i.hours > pauseThreshold); const longPauses = student.intervals.filter( (i) => i.hours > longPauseThreshold ); return { studentId: student.studentId, outcome: student.outcome, totalPauses: pauses.length, longPauses: longPauses.length, averagePauseDuration: d3.mean(pauses, (d) => d.hours), minPauseDuration: d3.min(pauses, (d) => d.hours), maxPauseDuration: d3.max(pauses, (d) => d.hours), pauseCount: pauses.length, averageLongPauseDuration: d3.mean(longPauses, (d) => d.hours), minLongPauseDuration: d3.min(longPauses, (d) => d.hours), maxLongPauseDuration: d3.max(longPauses, (d) => d.hours), longPauseCount: longPauses.length, pauseFrequency: student.intervals.length > 0 ? pauses.length / student.intervals.length : 0, pauseIntervals: pauses }; })3a:T4de,viewof pauseAxes = Inputs.form({ x: Inputs.radio( [ "averagePauseDuration", "minPauseDuration", "maxPauseDuration", "pauseCount", "averageLongPauseDuration", "minLongPauseDuration", "maxLongPauseDuration", "longPauseCount", "pauseFrequency", "outcome" ], { label: "X-Axis", value: "pauseCount" } ), y: Inputs.radio( [ "averagePauseDuration", "minPauseDuration", "maxPauseDuration", "pauseCount", "averageLongPauseDuration", "minLongPauseDuration", "maxLongPauseDuration", "longPauseCount", "pauseFrequency", "outcome" ], { label: "Y-Axis", value: "outcome" } ), c: Inputs.radio( [ "averagePauseDuration", "minPauseDuration", "maxPauseDuration", "pauseCount", "averageLongPauseDuration", "minLongPauseDuration", "maxLongPauseDuration", "longPauseCount", "pauseFrequency", "outcome" ], { label: "Color-Axis", value: "pauseFrequency" } ), k: Inputs.range([0, 20], { label: "How Many Standard Deviations?", value: 2, step: 0.5 }), clip: Inputs.toggle({ label: "Clip to only students that finished?" }) })3b:T5d2,// Cell 6: Calculate study regularity patterns regularityMetrics = learnerIntervals.map((student) => { // Group activity by day const dayActivity = {}; student.timestamps.forEach((ts) => { const day = new Date(ts).toDateString(); dayActivity[day] = (dayActivity[day] || 0) + 1; }); // Group by hour of day const hourActivity = new Array(24).fill(0); student.timestamps.forEach((ts) => { hourActivity[new Date(ts).getHours()]++; }); // Calculate streaks const dates = Object.keys(dayActivity).sort(); const streaks = []; let currentStreak = 1; for (let i = 1; i < dates.length; i++) { const dayDiff = (new Date(dates[i]) - new Date(dates[i - 1])) / (1000 * 60 * 60 * 24); if (dayDiff === 1) { currentStreak++; } else { streaks.push(currentStreak); currentStreak = 1; } } if (currentStreak > 0) streaks.push(currentStreak); const studyDays = dates.length; const totalDays = student.totalDays; return { studentId: student.studentId, consistency: totalDays > 0 ? studyDays / Math.ceil(totalDays) : 0, studyDays: studyDays, longestStreak: Math.max(...streaks, 0), averageStreak: streaks.length > 0 ? streaks.reduce((a, b) => a + b, 0) / streaks.length : 0, preferredHours: hourActivity .map((count, hour) => ({ hour, count })) .sort((a, b) => b.count - a.count) .slice(0, 3) .map((h) => h.hour), dayActivity: dayActivity }; })3c:Td26,fastFourierAnalysis = fftData.map((data) => { if (!data.fftReady) { return { studentId: data.studentId, spectrum: [], frequencies: [], dominantPeriods: [], primaryFrequencies: [null, null, null], frequencyDifferences: null }; } // Perform FFT const phasors = FFT.fft(data.signal); const magnitudes = FFT.util.fftMag(phasors); // Calculate frequencies (in cycles per day) const frequencies = []; for (let i = 0; i < magnitudes.length / 2; i++) { frequencies.push(i / (data.sampleRate * magnitudes.length)); } // Find peaks in spectrum const peaks = []; const threshold = Math.max(...magnitudes) * 0.1; for (let i = 1; i < magnitudes.length / 2 - 1; i++) { if ( magnitudes[i] > magnitudes[i - 1] && magnitudes[i] > magnitudes[i + 1] && magnitudes[i] > threshold ) { peaks.push({ frequency: frequencies[i], amplitude: magnitudes[i], period: frequencies[i] > 0 ? 1 / frequencies[i] : Infinity }); } } // Sort peaks by amplitude and take top 3 const dominantPeriods = peaks .sort((a, b) => b.amplitude - a.amplitude) .slice(0, 3); // Extract the three primary periods in days const primaryPeriods = [ dominantPeriods[0]?.period || null, dominantPeriods[1]?.period || null, dominantPeriods[2]?.period || null ]; // Calculate differences between periods const periodDifferences = { diff_1_2: primaryPeriods[0] && primaryPeriods[1] ? Math.abs(primaryPeriods[0] - primaryPeriods[1]) : null, diff_2_3: primaryPeriods[1] && primaryPeriods[2] ? Math.abs(primaryPeriods[1] - primaryPeriods[2]) : null, diff_1_3: primaryPeriods[0] && primaryPeriods[2] ? Math.abs(primaryPeriods[0] - primaryPeriods[2]) : null, totalDifference: primaryPeriods[0] && primaryPeriods[1] && primaryPeriods[2] ? Math.abs(primaryPeriods[0] - primaryPeriods[1]) + Math.abs(primaryPeriods[1] - primaryPeriods[2]) + Math.abs(primaryPeriods[0] - primaryPeriods[2]) : null, maxDifference: Math.max( primaryPeriods[0] && primaryPeriods[1] ? Math.abs(primaryPeriods[0] - primaryPeriods[1]) : 0, primaryPeriods[1] && primaryPeriods[2] ? Math.abs(primaryPeriods[1] - primaryPeriods[2]) : 0, primaryPeriods[0] && primaryPeriods[2] ? Math.abs(primaryPeriods[0] - primaryPeriods[2]) : 0 ) || null }; // Calculate spectral entropy (measure of regularity) const totalPower = magnitudes.reduce((a, b) => a + b, 0); const spectralEntropy = magnitudes.reduce((entropy, mag) => { const p = mag / totalPower; return p > 0 ? entropy - p * Math.log2(p) : entropy; }, 0); return { studentId: data.studentId, outcome: data.outcome, spectrum: magnitudes.slice(0, Math.floor(magnitudes.length / 2)), frequencies: frequencies, dominantPeriods: dominantPeriods, primaryPeriod1: primaryPeriods[0], primaryPeriod2: primaryPeriods[1], primaryPeriod3: primaryPeriods[2], ...periodDifferences, spectralEntropy: spectralEntropy, regularityScore: dominantPeriods.length > 0 ? dominantPeriods[0].amplitude / Math.max(...magnitudes) : 0 }; })3d:T406,viewof fftAxes = Inputs.form({ x: Inputs.radio( [ "primaryPeriod1", "primaryPeriod2", "primaryPeriod3", "totalDifference", "maxDifference", "spectralEntropy", "regularityScore", "outcome" ], { label: "X-Axis", value: "primaryPeriod1" } ), y: Inputs.radio( [ "primaryPeriod1", "primaryPeriod2", "primaryPeriod3", "totalDifference", "maxDifference", "spectralEntropy", "regularityScore", "outcome" ], { label: "Y-Axis", value: "outcome" } ), c: Inputs.radio( [ "primaryPeriod1", "primaryPeriod2", "primaryPeriod3", "totalDifference", "maxDifference", "spectralEntropy", "regularityScore", "outcome" ], { label: "Color-Axis", value: "regularityScore" } ), k: Inputs.range([0, 20], { label: "How Many Standard Deviations?", value: 2, step: 0.5 }), clip: Inputs.toggle({ label: "Clip to only students that finished?" }) })3e:T8ef,waveletAnalysis = learnerIntervals.map((student) => { if (student.intervals.length < 8) { return { studentId: student.studentId, coefficients: [], scaleAnalysis: [], waveletEnergy: 0 }; } // Prepare signal (interval durations in hours) const signal = student.intervals.map((i) => i.hours); // Perform multi-level wavelet decomposition using 'db4' wavelet // wavedec returns coefficients at different decomposition levels const coeffs = DWT.wavedec(signal, "db4"); // Calculate energy at each decomposition level const scaleAnalysis = coeffs.map((levelCoeffs, level) => { // Calculate energy for this level const energy = DWT.energy(levelCoeffs); // Estimate the time scale this level represents // Higher levels = lower frequencies = longer time scales const approxScaleDays = Math.pow(2, coeffs.length - level - 1) * (student.totalDays / signal.length); return { level: level, scalePeriod: approxScaleDays, // in days energy: energy, coefficients: levelCoeffs, numCoeffs: levelCoeffs.length, maxCoeff: Math.max(...levelCoeffs.map(Math.abs)), meanCoeff: levelCoeffs.reduce((a, b) => a + Math.abs(b), 0) / levelCoeffs.length }; }); // Calculate total energy across all scales const totalEnergy = DWT.energy(coeffs); // Find scales with highest energy (most important patterns) const topScales = [...scaleAnalysis] .sort((a, b) => b.energy - a.energy) .slice(0, 3); // Calculate energy distribution (how concentrated vs spread the energy is) const energyDistribution = scaleAnalysis.map( (scale) => scale.energy / totalEnergy ); // Entropy of energy distribution (measure of pattern complexity) const energyEntropy = energyDistribution.reduce((entropy, p) => { return p > 0 ? entropy - p * Math.log2(p) : entropy; }, 0); return { studentId: student.studentId, outcome: student.outcome, coefficients: coeffs, scaleAnalysis: scaleAnalysis, topScales: topScales, totalEnergy: totalEnergy, energyDistribution: energyDistribution, energyEntropy: energyEntropy, dominantScale: topScales[0]?.scalePeriod || null, timestampCount: student.intervals.length + 1 }; })3f:T83f,// Cell 11: Calculate final pacing metrics for analysis finalPacingMetrics = interventionMapping.map((student, i) => { const vm = velocityMetrics[i]; const pause = pauseAnalysis[i]; const regularity = regularityMetrics[i]; const fourier = fastFourierAnalysis[i]; const wavelet = waveletAnalysis[i]; // Get the original student data to access user ID const originalStudent = learnersByLessons[i]; const userId = originalStudent[0]?.[0]?.illinois_tech_user_id || student.studentId; // Get grade if available - need to check the structure of courseGrades const gradeInfo = courseGrades?.[0]?.[userId]; return { studentId: userId, outcome: student.outcome, // Core metrics averageVelocity: vm.averageVelocity, velocityVariance: vm.velocityVariance, decelerationCount: vm.decelerationPoints.length, // Pause behavior pauseFrequency: pause.pauseFrequency, averagePauseDuration: pause.averagePauseDuration, // Regularity consistency: regularity.consistency, longestStreak: regularity.longestStreak, // Frequency domain - THREE PRIMARY PERIODS primaryPeriod1: fourier.primaryPeriod1, primaryPeriod2: fourier.primaryPeriod2, primaryPeriod3: fourier.primaryPeriod3, // PERIOD DIFFERENCES (key for understanding pacing disorder) ...fourier.periodDifferences, totalDifference: fourier.totalDifference, maxDifference: fourier.maxDifference, spectralEntropy: fourier.spectralEntropy, // Pattern indicators regularityScore: fourier.regularityScore, spectralEntropy: fourier.spectralEntropy, hasDailyPattern: fourier.primaryPeriod1 > 0.8 && fourier.primaryPeriod1 < 1.2, hasWeeklyPattern: fourier.dominantPeriods.some( (p) => p.period > 6 && p.period < 8 ), // Wavelet analysis waveletTopScale: wavelet.topScales?.[0]?.scalePeriod, waveletTotalEnergy: wavelet.totalEnergy, // Interventions interventionNeeds: student.interventionNeeds, riskScore: student.riskScore }; }) //.filter((d) => d.outcome !== null) // Only include students with grades40:T6eb,viewof finalAxes = Inputs.form({ x: Inputs.radio( [ "outcome", "averageVelocity", "velocityVariance", "decelerationCount", "pauseFrequency", "averagePauseDuration", "consistency", "longestStreak", "primaryPeriod1", "primaryPeriod2", "primaryPeriod3", "totalDifference", "maxDifference", "spectralEntropy", "regularityScore", "waveletTopScale", "waveletTotalEnergy", "riskScore" ], { label: "X-Axis", value: "velocityVariance" } ), y: Inputs.radio( [ "outcome", "averageVelocity", "velocityVariance", "decelerationCount", "pauseFrequency", "averagePauseDuration", "consistency", "longestStreak", "primaryPeriod1", "primaryPeriod2", "primaryPeriod3", "totalDifference", "maxDifference", "spectralEntropy", "regularityScore", "waveletTopScale", "waveletTotalEnergy", "riskScore" ], { label: "Y-Axis", value: "outcome" } ), c: Inputs.radio( [ "outcome", "averageVelocity", "velocityVariance", "decelerationCount", "pauseFrequency", "averagePauseDuration", "consistency", "longestStreak", "primaryPeriod1", "primaryPeriod2", "primaryPeriod3", "totalDifference", "maxDifference", "spectralEntropy", "regularityScore", "waveletTopScale", "waveletTotalEnergy", "riskScore" ], { label: "Color-Axis", value: "consistency" } ), k: Inputs.range([0, 20], { label: "How Many Standard Deviations?", value: 2, step: 0.5 }), clip: Inputs.toggle({ label: "Clip to only students that finished?" }) })41:Ta07,// Cell 12: Summary statistics for frequency differences frequencyDifferenceStats = Inputs.table([ { // Calculate statistics on period differences avgPrimaryPeriod1: d3.mean(finalPacingMetrics, (d) => d.primaryPeriod1), avgPrimaryPeriod2: d3.mean(finalPacingMetrics, (d) => d.primaryPeriod2), avgPrimaryPeriod3: d3.mean(finalPacingMetrics, (d) => d.primaryPeriod3), avgTotalDifference: d3.mean( finalPacingMetrics, (d) => d.totalPeriodDifference ), medianTotalDifference: d3.median( finalPacingMetrics, (d) => d.totalPeriodDifference ), // Calculate correlation between period differences and outcomes correlationDiffOutcome: (() => { const validData = finalPacingMetrics.filter( (d) => d.totalPeriodDifference !== null && d.outcome !== null ); if (validData.length < 10) return null; const x = validData.map((d) => d.totalPeriodDifference); const y = validData.map((d) => d.outcome); // Pearson correlation coefficient const meanX = d3.mean(x); const meanY = d3.mean(y); const stdX = d3.deviation(x); const stdY = d3.deviation(y); if (stdX === 0 || stdY === 0) return 0; const covariance = d3.mean( x.map((xi, i) => (xi - meanX) * (y[i] - meanY)) ); return covariance / (stdX * stdY); })(), // Students with consistent vs inconsistent periods consistentPacers: finalPacingMetrics.filter( (d) => d.maxPeriodDifference !== null && d.maxPeriodDifference < 2 ).length, inconsistentPacers: finalPacingMetrics.filter( (d) => d.maxPeriodDifference !== null && d.maxPeriodDifference > 5 ).length, // Success rates by pacing consistency consistentSuccess: d3.mean( finalPacingMetrics.filter( (d) => d.maxPeriodDifference !== null && d.maxPeriodDifference < 2 ), (d) => d.outcome ), inconsistentSuccess: d3.mean( finalPacingMetrics.filter( (d) => d.maxPeriodDifference !== null && d.maxPeriodDifference > 5 ), (d) => d.outcome ), // Additional statistics stdDevTotalDifference: d3.deviation( finalPacingMetrics.filter((d) => d.totalPeriodDifference !== null), (d) => d.totalPeriodDifference ), // Quartiles for period differences periodDiffQuartiles: d3.quantile( finalPacingMetrics .filter((d) => d.totalPeriodDifference !== null) .map((d) => d.totalPeriodDifference) .sort(d3.ascending), [0.25, 0.5, 0.75] ) } ])42:T801,// Cell 10: Map patterns to intervention needs interventionMapping = velocityMetrics.map((vm, i) => { const studentId = vm.studentId; const pause = pauseAnalysis[i]; const regularity = regularityMetrics[i]; const fourier = fastFourierAnalysis[i]; const wavelet = waveletAnalysis[i]; // Determine intervention needs const interventions = []; const tiTypes = [ "onboarding_guidance", "trajectory_mapping", "cognitive_buffering", "dynamic_pace_coaching", "review_and_assess", "reflection_prompts" ]; // Onboarding guidance needed if early velocty is low - this keeps students in the system and facilitates upfront data collectoin if (vm.velocities.slice(0, 3).some((v) => v < vm.averageVelocity * 0.3)) { interventions.push("onboarding_guidance"); } // Trajectory mapping if consistency is low — demonstrating how current patterns will lead to future success.failure if (regularity.consistency < 0.3) { interventions.push("trajectory_mapping"); } // Cognitive buffering if no obseved pause behaviors if (pause.pauseFrequency < 0.1 || pause.averagePauseDuration > 48) { interventions.push("cognitive_buffering"); } // Dynamic pace coaching if many accel/decel intervals if (vm.decelerationPoints.length > 2) { interventions.push("dynamic_pace_coaching"); } // Reflection needed if high speed variance? if (vm.velocityVariance > vm.averageVelocity * 0.5) { interventions.push("reflection_prompts"); } // diagnose if irregular pacing patterns orjust large frequency differences between dominant periods if ( fourier.spectralEntropy > 3 || fourier.regularityScore < 0.2 || fourier.periodDifferences?.totalDifference > 10 ) { interventions.push("review_and_assess"); } return { studentId: studentId, outcome: vm.outcome, interventionNeeds: interventions, riskScore: interventions.length / 6, primaryNeed: interventions[0] || "none", primaryIndex: tiTypes.findIndex((d) => d == interventions[0]) }; })43:[{"id":"b01f03a0e4517290b602939045349ce92ee56980ad4da77ba652ff28daea99b19fabba39ac700ac984c3bf950b1c4eec3c3445710b7079c2f521cdc3a6f448bb","href":"https://static.observableusercontent.com/files/b01f03a0e4517290b602939045349ce92ee56980ad4da77ba652ff28daea99b19fabba39ac700ac984c3bf950b1c4eec3c3445710b7079c2f521cdc3a6f448bb","name":"deep-learning.zip","type":"application/zip","lastModified":1757884528202.939,"size":2347685},{"id":"c6c13e421af779b8067b65134ced075d558c92cbbe44023d889756046c0a6a3d599b06ccfa8aca6f61d5dc60f6a7cfcebfb78ca4c786c159e375d0e1617a3a06","href":"https://static.observableusercontent.com/files/c6c13e421af779b8067b65134ced075d558c92cbbe44023d889756046c0a6a3d599b06ccfa8aca6f61d5dc60f6a7cfcebfb78ca4c786c159e375d0e1617a3a06","name":"industrial-psychology.zip","type":"application/zip","lastModified":1757884527536.066,"size":1015017},{"id":"e0429e094059e7ff09766207000d11dede693fc8bb1bed309c44c11bbc58e31d600efbe290243448314e516a219e376ca4bb8ea98589005cc99f0c74386da717","href":"https://static.observableusercontent.com/files/e0429e094059e7ff09766207000d11dede693fc8bb1bed309c44c11bbc58e31d600efbe290243448314e516a219e376ca4bb8ea98589005cc99f0c74386da717","name":"mba-506.zip","type":"application/zip","lastModified":1757884528323.937,"size":3027436},{"id":"6e251adad3a5389378e6aa270384744195149293081fba559a379dffbe469b5ae7c4d4bfa80e404e9c2070e2674a676e8f19265c150115e8baad8d47b5d88ffb","href":"https://static.observableusercontent.com/files/6e251adad3a5389378e6aa270384744195149293081fba559a379dffbe469b5ae7c4d4bfa80e404e9c2070e2674a676e8f19265c150115e8baad8d47b5d88ffb","name":"excel.zip","type":"application/zip","lastModified":1757884530462.208,"size":9051555},{"id":"3cc3bae8026f3a19520ad4c16fe800b77679964284338d9d7f3c5f0b4674f473dc72a216d3d2329e4200abe520617ec13bbd9c4aceda4cdcb9979a1efef01232","href":"https://static.observableusercontent.com/files/3cc3bae8026f3a19520ad4c16fe800b77679964284338d9d7f3c5f0b4674f473dc72a216d3d2329e4200abe520617ec13bbd9c4aceda4cdcb9979a1efef01232","name":"itm-705.zip","type":"application/zip","lastModified":1757884531849.552,"size":18268950}] 2e:{"title":"Coursera Stats","cells":[{"id":0,"value":"# Coursera Stats","pinned":false,"hidden":false,"mode":"md"},{"id":4,"value":"viewof courseNames = Inputs.checkbox(\n [\n \"Hardware and Operating System Foundations I\",\n \"Leadership & Organization Design\",\n \"Industrial Psychology\",\n \"Deep Learning\",\n \"Comprehensive Excel Masterclass\"\n ],\n {\n value: [\"Hardware and Operating System Foundations I\"],\n label: \"Courses to Include\",\n columns: 3\n }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":340,"value":"---\n### Load Data","pinned":false,"hidden":false,"mode":"md"},{"id":8,"value":"files = await Promise.all(\n courseNames.map(\n async (c) =>\n ({\n \"Hardware and Operating System Foundations I\": await FileAttachment(\n \"itm-705.zip\"\n ).zip(),\n \"Leadership & Organization Design\": await FileAttachment(\n \"mba-506.zip\"\n ).zip(),\n\n \"Industrial Psychology\": await FileAttachment(\n \"industrial-psychology.zip\"\n ).zip(),\n \"Deep Learning\": await FileAttachment(\"deep-learning.zip\").zip(),\n \"Comprehensive Excel Masterclass\": await FileAttachment(\n \"excel.zip\"\n ).zip()\n }[c])\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":12,"value":"users = await Promise.all(\n files.map(async (f) => await f.file(\"users.csv\").csv())\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"globalItems = await Promise.all(\n files.map(async (f) =>\n (\n await f.file(\"global_items.csv\").csv()\n ).reduce((a, v) => {\n a[v.global_item_id] = v;\n return a;\n }, {})\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"globalItemTypes = await Promise.all(\n files.map(async (f) =>\n (\n await f.file(\"global_item_types.csv\").csv()\n ).reduce((a, v) => {\n a[v.global_item_content_type_id] = v;\n return a;\n }, {})\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":95,"value":"courseLessons = await Promise.all(\n files.map(async (f) =>\n (\n await f.file(\"course_branch_lessons.csv\").csv()\n ).reduce((a, v) => {\n a[v.course_lesson_id] = v;\n return a;\n }, {})\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":109,"value":"courseModules = await Promise.all(\n files.map(async (f) =>\n (\n await f.file(\"course_branch_modules.csv\").csv()\n ).reduce((a, v) => {\n a[v.course_module_id] = v;\n return a;\n }, {})\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":18,"value":"courseItems = await Promise.all(\n files.map(async (f) =>\n (\n await f.file(\"course_branch_items.csv\").csv()\n ).reduce((a, v) => {\n a[v.course_item_id] = v;\n return a;\n }, {})\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":141,"value":"courseItemPassingStates = await Promise.all(\n files.map(async (f) =>\n (\n await f.file(\"course_item_passing_states.csv\").csv()\n ).reduce((a, v) => {\n a[v.course_item_passing_state_id] = v;\n return a;\n }, {})\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":131,"value":"coursePassingStates = await Promise.all(\n files.map(async (f) =>\n (\n await f.file(\"course_passing_states.csv\").csv()\n ).reduce((a, v) => {\n a[v.course_passing_state_id] = v;\n return a;\n }, {})\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":68,"value":"courseItemGrades = await Promise.all(\n files.map(async (f, i) =>\n (\n await f.file(\"course_item_grades.csv\").csv()\n ).reduce((a, v) => {\n a[v.illinois_tech_user_id + \"-\" + v.course_item_id] = {\n ...v,\n course_passing_state_desc:\n courseItemPassingStates[i][v.course_item_passing_state_id]\n .course_item_passing_state_desc\n };\n return a;\n }, {})\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":88,"value":"courseGrades = await Promise.all(\n files.map(async (f, i) =>\n (\n await f.file(\"course_branch_grades.csv\").csv()\n ).reduce((a, v) => {\n a[v.illinois_tech_user_id] = {\n ...v,\n course_passing_state_desc:\n coursePassingStates[i][v.course_passing_state_id]\n .course_passing_state_desc\n };\n return a;\n }, {})\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":15,"value":"courseProgress = await Promise.all(\n files.map(async (f) =>\n (\n await f.file(\"course_progress.csv\").csv()\n ).sort((a, b) =>\n d3.ascending(\n new Date(a.course_progress_ts),\n new Date(b.course_progress_ts)\n )\n )\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":342,"value":"---\n### Connect References","pinned":false,"hidden":false,"mode":"md"},{"id":242,"value":"getItemData = (item_id, user_id, i) => {\n let obj = {\n item: courseItems[i][item_id],\n grade: courseItemGrades[i][user_id + \"-\" + item_id],\n globalItem: globalItems[i][courseItems[i][item_id].global_item_id],\n globalItemType:\n globalItemTypes[i][\n globalItems[i][courseItems[i][item_id].global_item_id]\n .global_item_content_type_id\n ],\n lesson: courseLessons[i][courseItems[i][item_id].course_lesson_id],\n module:\n courseModules[i][\n courseLessons[i][courseItems[i][item_id].course_lesson_id]\n .course_module_id\n ]\n };\n\n obj.courseSort =\n obj.module.course_branch_module_order.padStart(2, \"0\") +\n \"-\" +\n obj.lesson.course_branch_lesson_order.padStart(2, \"0\") +\n \"-\" +\n obj.item.course_branch_item_order.padStart(2, \"0\");\n\n return obj;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":79,"value":"progress = courseProgress.map((c, i) =>\n c.map((p) => ({\n ...p,\n courseItemData: getItemData(p.course_item_id, p.illinois_tech_user_id, i)\n }))\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":345,"value":"---\n### Group and Sort","pinned":false,"hidden":false,"mode":"md"},{"id":282,"value":"learners = progress\n .map((c, i) =>\n Object.values(Object.groupBy(c, (p) => p.illinois_tech_user_id))\n .map((l) =>\n l.reduce(\n (a, v) => {\n if (!a.ts.hasOwnProperty(v.course_progress_ts)) {\n a.items.push({ ...v, courseIndex: i });\n a.ts[v.course_progress_ts] = 1;\n }\n\n return a;\n },\n { ts: {}, items: [], originalCount: l.length }\n )\n )\n .map((d) => d.items)\n )\n .flat()","pinned":false,"hidden":false,"mode":"ojs"},{"id":1659,"value":"learnerGrades = learners.reduce((a, l) => {\n let outcome = parseFloat(\n courseGrades[l[0].courseIndex][l[0].illinois_tech_user_id]\n ?.course_branch_grade_overall\n );\n\n outcome = outcome ? outcome : 0;\n outcome = outcome >= 1 ? 1 : outcome;\n\n a[l[0].illinois_tech_user_id + \"-\" + l[0].courseIndex] = outcome;\n\n return a;\n}, {})","pinned":false,"hidden":false,"mode":"ojs"},{"id":289,"value":"learnersByModules = learners.map((l) =>\n Object.values(\n Object.groupBy(l, (i) => i.courseItemData.module.course_module_id)\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":295,"value":"learnersByLessons = learners.map((l) =>\n Object.values(\n Object.groupBy(l, (i) => i.courseItemData.lesson.course_lesson_id)\n )\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1794,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":1962,"value":"learnersBySessions.flat().length","pinned":false,"hidden":false,"mode":"ojs"},{"id":1911,"value":"---\n### Hidden Markov Model","pinned":false,"hidden":false,"mode":"md"},{"id":349,"value":"---\n### Calculate Time Deltas and Offsets","pinned":false,"hidden":false,"mode":"md"},{"id":317,"value":"learnersLessonTimes = learnersByLessons.map((l) =>\n l\n .map((m) => ({\n start: new Date(m[0].course_progress_ts),\n end: new Date(m[m.length - 1].course_progress_ts),\n duration:\n new Date(m[m.length - 1].course_progress_ts) -\n new Date(m[0].course_progress_ts),\n events: m\n }))\n .map((m, i, r) => {\n m.intervalBefore = 0;\n m.intervalAfter = 0;\n\n if (i) {\n m.intervalBefore = m.start - r[i - 1].end;\n }\n\n if (i < r.length - 1) {\n m.intervalAfter = r[i + 1].start - m.end;\n }\n return m;\n })\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":304,"value":"learnersModuleTimes = learnersByModules.map((l) =>\n l\n .map((m) => ({\n start: new Date(m[0].course_progress_ts),\n end: new Date(m[m.length - 1].course_progress_ts),\n duration:\n new Date(m[m.length - 1].course_progress_ts) -\n new Date(m[0].course_progress_ts),\n events: m\n }))\n .map((m, i, r) => {\n m.intervalBefore = 0;\n m.intervalAfter = 0;\n\n if (i) {\n m.intervalBefore = m.start - r[i - 1].end;\n }\n\n if (i < r.length - 1) {\n m.intervalAfter = r[i + 1].start - m.end;\n }\n return m;\n })\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":410,"value":"---\n\n### Lesson DBScan Analysis","pinned":false,"hidden":false,"mode":"md"},{"id":753,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":886,"value":"lessonDBScanFiltered = {\n const k = lessonDBScanK;\n\n const xMean = d3.mean(lessonDBScan.map((d) => d[lessonDBScanX]));\n const xStd = d3.deviation(lessonDBScan.map((d) => d[lessonDBScanX]));\n\n const yMean = d3.mean(lessonDBScan.map((d) => d[lessonDBScanY]));\n const yStd = d3.deviation(lessonDBScan.map((d) => d[lessonDBScanY]));\n\n const cMean = d3.mean(lessonDBScan.map((d) => d[lessonDBScanC]));\n const cStd = d3.deviation(lessonDBScan.map((d) => d[lessonDBScanC]));\n\n return lessonDBScan\n .filter((x) => Math.abs(x[lessonDBScanX] - xMean) <= k * xStd)\n .filter((y) => Math.abs(y[lessonDBScanY] - yMean) <= k * yStd)\n .filter((c) => Math.abs(c[lessonDBScanC] - cMean) <= k * cStd);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":913,"value":"viewof lessonDBScanK = Inputs.range([0, 20], {\n label: \"How Many Standard Deviations?\",\n value: 8,\n step: 0.5\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":716,"value":"viewof lessonDBScanX = Inputs.radio(\n [\n \"outcome\",\n \"totalTimestamps\",\n \"totalLessons\",\n \"courseDurationDays\",\n \"averageEventsPerDay\",\n \"meanIntervalHours\",\n \"medianIntervalHours\",\n \"coefficientOfVariation\",\n \"outlierCount\",\n \"outlierPercentage\",\n \"largestGapHours\",\n \"smallestGapHours\"\n ],\n { label: \"X-Axis\", value: \"totalTimestamps\" }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":720,"value":"viewof lessonDBScanY = Inputs.radio(\n [\n \"outcome\",\n \"totalTimestamps\",\n \"totalLessons\",\n \"courseDurationDays\",\n \"averageEventsPerDay\",\n \"meanIntervalHours\",\n \"medianIntervalHours\",\n \"coefficientOfVariation\",\n \"outlierCount\",\n \"outlierPercentage\",\n \"largestGapHours\",\n \"smallestGapHours\"\n ],\n { label: \"Y-Axis\", value: \"coefficientOfVariation\" }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":729,"value":"viewof lessonDBScanC = Inputs.radio(\n [\n \"outcome\",\n \"totalTimestamps\",\n \"totalLessons\",\n \"courseDurationDays\",\n \"averageEventsPerDay\",\n \"meanIntervalHours\",\n \"medianIntervalHours\",\n \"coefficientOfVariation\",\n \"outlierCount\",\n \"outlierPercentage\",\n \"largestGapHours\",\n \"smallestGapHours\"\n ],\n { label: \"Color-Axis\", value: \"outcome\" }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":770,"value":"Plot.plot({\n inset: 8,\n grid: true,\n width: 900,\n height: 900 / 2,\n color: {\n legend: true,\n scheme: \"Cool\"\n },\n marks: [\n Plot.dot(lessonDBScanFiltered, {\n x: lessonDBScanX,\n y: lessonDBScanY,\n fill: lessonDBScanC,\n r: 1.5,\n opacity: 0.5\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1170,"value":"viewof regressLessons = Inputs.toggle({ label: \"Run regression?\" })","pinned":false,"hidden":false,"mode":"ojs"},{"id":1173,"value":"{\n if (regressLessons) {\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: lessonDBScanX,\n y: lessonDBScanY,\n type: r[0],\n r2: r[1].x((d) => d[lessonDBScanX]).y((d) => d[lessonDBScanY])(\n lessonDBScanFiltered\n ).rSquared\n }))\n .flat(Infinity)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1071,"value":"---\n### Student DBScan Analysis","pinned":false,"hidden":false,"mode":"md"},{"id":418,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":934,"value":"studentDBScanFiltered = {\n const k = studentDBScanK;\n\n const xMean = d3.mean(studentDBScan.map((d) => d[studentDBScanX]));\n const xStd = d3.deviation(studentDBScan.map((d) => d[studentDBScanX]));\n\n const yMean = d3.mean(studentDBScan.map((d) => d[studentDBScanY]));\n const yStd = d3.deviation(studentDBScan.map((d) => d[studentDBScanY]));\n\n const cMean = d3.mean(studentDBScan.map((d) => d[studentDBScanC]));\n const cStd = d3.deviation(studentDBScan.map((d) => d[studentDBScanC]));\n\n return studentDBScan\n .filter((x) => Math.abs(x[studentDBScanX] - xMean) <= k * xStd)\n .filter((y) => Math.abs(y[studentDBScanY] - yMean) <= k * yStd)\n .filter((c) => Math.abs(c[studentDBScanC] - cMean) <= k * cStd);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":930,"value":"viewof studentDBScanK = Inputs.range([0, 20], {\n label: \"How Many Standard Deviations?\",\n value: 4,\n step: 0.5\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":511,"value":"viewof studentDBScanX = Inputs.radio(\n [\n \"outcome\",\n \"totalTimestamps\",\n \"totalLessons\",\n \"courseDurationDays\",\n \"averageEventsPerDay\",\n \"meanIntervalHours\",\n \"medianIntervalHours\",\n \"coefficientOfVariation\",\n \"outlierCount\",\n \"outlierPercentage\",\n \"largestGapHours\",\n \"smallestGapHours\"\n ],\n { label: \"X-Axis\", value: \"totalTimestamps\" }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":519,"value":"viewof studentDBScanY = Inputs.radio(\n [\n \"outcome\",\n \"totalTimestamps\",\n \"totalLessons\",\n \"courseDurationDays\",\n \"averageEventsPerDay\",\n \"meanIntervalHours\",\n \"medianIntervalHours\",\n \"coefficientOfVariation\",\n \"outlierCount\",\n \"outlierPercentage\",\n \"largestGapHours\",\n \"smallestGapHours\"\n ],\n { label: \"Y-Axis\", value: \"coefficientOfVariation\" }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":523,"value":"viewof studentDBScanC = Inputs.radio(\n [\n \"outcome\",\n \"totalTimestamps\",\n \"totalLessons\",\n \"courseDurationDays\",\n \"averageEventsPerDay\",\n \"meanIntervalHours\",\n \"medianIntervalHours\",\n \"coefficientOfVariation\",\n \"outlierCount\",\n \"outlierPercentage\",\n \"largestGapHours\",\n \"smallestGapHours\"\n ],\n { label: \"Color-Axis\", value: \"outcome\" }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":453,"value":"Plot.plot({\n inset: 8,\n grid: true,\n width: 900,\n height: 900 / 2,\n color: {\n legend: true,\n scheme: \"Cool\",\n label: \"Final Graded Outcome\"\n },\n x: { label: \"Total Interaction\" },\n y: {},\n marks: [\n Plot.dot(studentDBScanFiltered, {\n x: studentDBScanX,\n y: studentDBScanY,\n fill: studentDBScanC,\n r: 1.5,\n opacity: 0.75\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1154,"value":"viewof regressStudents = Inputs.toggle({ label: \"Run regression?\" })","pinned":false,"hidden":false,"mode":"ojs"},{"id":1152,"value":"{\n if (regressStudents) {\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: studentDBScanX,\n y: studentDBScanY,\n type: r[0],\n r2: r[1].x((d) => d[studentDBScanX]).y((d) => d[studentDBScanY])(\n studentDBScanFiltered\n ).rSquared\n }))\n .flat(Infinity)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":542,"value":"---\n### Markov Model Entropy Analysis","pinned":false,"hidden":false,"mode":"md"},{"id":544,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":776,"value":"---\n### Latent Markov Analysis","pinned":false,"hidden":false,"mode":"md"},{"id":778,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":786,"value":"$35","pinned":false,"hidden":false,"mode":"ojs"},{"id":590,"value":"---\n ### Fourier Analysis\n","pinned":false,"hidden":false,"mode":"md"},{"id":594,"value":"$36","pinned":false,"hidden":false,"mode":"ojs"},{"id":1179,"value":"**Primary Insight**\n\n- x: `dominantPeriodDifference`\n- y: `outcome`\n\nStudents who have meaningfully different dominant periods (such as hourly, daily, and weekly interactions)","pinned":false,"hidden":false,"mode":"md"},{"id":947,"value":"viewof fourierK = Inputs.range([0, 20], {\n label: \"How Many Standard Deviations?\",\n value: 4,\n step: 0.5\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":944,"value":"fourierFiltered = {\n const k = fourierK;\n\n const xMean = d3.mean(fourierAnalysis.map((d) => d[fourierX]));\n const xStd = d3.deviation(fourierAnalysis.map((d) => d[fourierX]));\n\n const yMean = d3.mean(fourierAnalysis.map((d) => d[fourierY]));\n const yStd = d3.deviation(fourierAnalysis.map((d) => d[fourierY]));\n\n const cMean = d3.mean(fourierAnalysis.map((d) => d[fourierC]));\n const cStd = d3.deviation(fourierAnalysis.map((d) => d[fourierC]));\n\n return fourierAnalysis\n .filter((x) => Math.abs(x[fourierX] - xMean) <= k * xStd)\n .filter((y) => Math.abs(y[fourierY] - yMean) <= k * yStd)\n .filter((c) => Math.abs(c[fourierC] - cMean) <= k * cStd);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":633,"value":"viewof fourierX = Inputs.radio(\n [\n \"regularityScore\",\n \"dominantPeriod0\",\n \"dominantPeriod1\",\n \"dominantPeriod2\",\n 'dominantPeriodDifference',\n \"intervalVariability\",\n \"outcome\",\"count\"\n ],\n { label: \"X-Axis\", value: \"regularityScore\" }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":642,"value":"viewof fourierY = Inputs.radio(\n [\n \"regularityScore\",\n \"dominantPeriod0\",\n \"dominantPeriod1\",\n \"dominantPeriod2\",\n \"dominantPeriodDifference\",\n \"intervalVariability\",\n \"outcome\",\"count\"\n ],\n { label: \"Y-Axis\", value: \"intervalVariability\" }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":646,"value":"viewof fourierC = Inputs.radio(\n [\n \"regularityScore\",\n \"dominantPeriod0\",\n \"dominantPeriod1\",\n \"dominantPeriod2\",\n \"dominantPeriodDifference\",\n \"intervalVariability\",\n \"outcome\",\n \"count\"\n ],\n { label: \"C-Axis\", value: \"outcome\" }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":620,"value":"Plot.plot({\n width: 1200,\n height: 800,\n inset: 8,\n grid: true,\n color: {\n legend: true,\n scheme: \"Cool\",\n label: \"Learning Session Count\"\n },\n x: { label: \"Dominant Learning Frequency Variability\" },\n y: { label: \"Final Graded Outcome\" },\n marks: [\n Plot.dot(fourierFiltered, {\n x: fourierX,\n y: fourierY,\n fill: fourierC,\n r: 2,\n opacity: 0.75\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1146,"value":"viewof regressFourier = Inputs.toggle({ label: \"Run regression?\" })","pinned":false,"hidden":false,"mode":"ojs"},{"id":1103,"value":"{\n if (regressFourier) {\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: fourierX,\n y: fourierY,\n type: r[0],\n r2: r[1].x((d) => d[fourierX]).y((d) => d[fourierY])(fourierFiltered)\n .rSquared\n }))\n .flat(Infinity)\n .filter((k) => k.x !== k.y)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":540,"value":"---","pinned":false,"hidden":false,"mode":"md"},{"id":961,"value":"---\n ### Bucketing","pinned":false,"hidden":false,"mode":"md"},{"id":357,"value":"data = learnersLessonTimes\n .flat(Infinity)\n .filter((d) => d.duration < 1000 * 60 * 60)","pinned":false,"hidden":false,"mode":"ojs"},{"id":351,"value":"// Plot.plot({\n// y: { grid: true },\n// color: { legend: true },\n// marks: [\n// Plot.rectY(\n// data,\n// Plot.binX({ y: \"count\" }, { x: \"duration\", fill: (d) => d.events.length })\n// ),\n// Plot.ruleY([0])\n// ]\n// })","pinned":false,"hidden":false,"mode":"ojs"},{"id":1096,"value":"d3 = require(\"d3\", \"d3-regression\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":1231,"value":"---\n## Signal Processing New Approaches\n","pinned":false,"hidden":false,"mode":"md"},{"id":1207,"value":"FFT = require(\"https://bundle.run/fft-js@0.0.12\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":1210,"value":"DWT\n= require(\"discrete-wavelets@5.0.15/dist/discrete-wavelets.umd.js\")","pinned":true,"hidden":false,"mode":"ojs"},{"id":1398,"value":"---\n #### Prep Data","pinned":false,"hidden":false,"mode":"md"},{"id":1221,"value":"processedLearners = learnersByLessons.map((studentLessons, studentIndex) => ({\n studentId:\n studentLessons[0]?.[0]?.illinois_tech_user_id || `student_${studentIndex}`,\n outcome:\n learnerGrades[\n studentLessons[0][0].illinois_tech_user_id +\n \"-\" +\n studentLessons[0][0].courseIndex\n ],\n allTimestamps: studentLessons\n .flat()\n .map((item) => new Date(item.course_progress_ts).getTime())\n .sort((a, b) => a - b),\n lessons: studentLessons.map((lesson) =>\n lesson.map((item) => ({\n ...item,\n timestamp: new Date(item.course_progress_ts).getTime()\n }))\n )\n}))","pinned":false,"hidden":false,"mode":"ojs"},{"id":1213,"value":"$37","pinned":false,"hidden":false,"mode":"ojs"},{"id":1632,"value":"viewof learnerAxes = Inputs.form({\n x: Inputs.radio(\n [\n \"outcome\",\n \"timestampCount\",\n \"totalDays\",\n \"meanInterval\",\n \"standardDeviation\",\n \"coefficientOfVariation\"\n ],\n { label: \"X-Axis\", value: \"coefficientOfVariation\" }\n ),\n y: Inputs.radio(\n [\n \"outcome\",\n \"timestampCount\",\n \"totalDays\",\n \"meanInterval\",\n \"standardDeviation\",\n \"coefficientOfVariation\"\n ],\n { label: \"Y-Axis\", value: \"outcome\" }\n ),\n c: Inputs.radio(\n [\n \"outcome\",\n \"timestampCount\",\n \"totalDays\",\n \"meanInterval\",\n \"standardDeviation\",\n \"coefficientOfVariation\"\n ],\n { label: \"Color-Axis\", value: \"totalDays\" }\n ),\n k: Inputs.range([0, 20], {\n label: \"How Many Standard Deviations?\",\n value: 2,\n step: 0.5\n }),\n clip: Inputs.toggle({ label: \"Clip to only students that finished?\" })\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1641,"value":"learnerIntervalsFiltered = {\n const xMean = d3.mean(learnerIntervals.map((d) => d[learnerAxes.x]));\n\n const xStd = d3.deviation(learnerIntervals.map((d) => d[learnerAxes.x]));\n\n const yMean = d3.mean(learnerIntervals.map((d) => d[learnerAxes.y]));\n const yStd = d3.deviation(learnerIntervals.map((d) => d[learnerAxes.y]));\n\n const cMean = d3.mean(learnerIntervals.map((d) => d[learnerAxes.c]));\n const cStd = d3.deviation(learnerIntervals.map((d) => d[learnerAxes.c]));\n\n return learnerIntervals\n .filter((d) => (learnerAxes.clip ? d.outcome : 1))\n .filter((x) => Math.abs(x[learnerAxes.x] - xMean) <= learnerAxes.k * xStd)\n .filter((y) => Math.abs(y[learnerAxes.y] - yMean) <= learnerAxes.k * yStd)\n .filter((c) => Math.abs(c[learnerAxes.c] - cMean) <= learnerAxes.k * cStd);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1630,"value":"Plot.plot({\n width: width,\n height: 500,\n inset: 8,\n grid: true,\n color: {\n legend: true,\n scheme: \"Cool\"\n },\n marks: [\n Plot.dot(learnerIntervalsFiltered, {\n x: learnerAxes.x,\n y: learnerAxes.y,\n fill: learnerAxes.c,\n r: 1\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1651,"value":"{\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: learnerAxes.x,\n y: learnerAxes.y,\n type: r[0],\n r2: r[1].x((d) => d[learnerAxes.x]).y((d) => d[learnerAxes.c])(\n learnerIntervalsFiltered\n ).rSquared\n }))\n .flat(Infinity)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1625,"value":"----\n#### Intervals","pinned":false,"hidden":false,"mode":"md"},{"id":1532,"value":"intervalData = learnerIntervals\n .map((d) =>\n d.intervals.map((e, i) => ({ ...e, outcome: d.outcome, position: i }))\n )\n .flat()","pinned":false,"hidden":false,"mode":"ojs"},{"id":1551,"value":"viewof intervalAxes = Inputs.form({\n x: Inputs.radio([\"hours\", \"outcome\", \"position\"], {\n label: \"X-Axis\",\n value: \"hours\"\n }),\n c: Inputs.radio([\"hours\", \"outcome\", \"position\"], {\n label: \"Color-Axis\",\n value: \"outcome\"\n }),\n k: Inputs.range([0, 20], {\n label: \"How Many Standard Deviations?\",\n value: 2,\n step: 0.5\n }),\n clip: Inputs.toggle({ label: \"Clip to only students that finished?\" })\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1548,"value":"filteredIntervalData = {\n const xVals = intervalData.map((d) => d[intervalAxes.x]);\n const cVals = intervalData.map((d) => d[intervalAxes.c]);\n\n const xQ1 = d3.quantile(xVals, 0.25);\n const xQ3 = d3.quantile(xVals, 0.75);\n const xIqr = xQ3 - xQ1;\n\n const cQ1 = d3.quantile(cVals, 0.25);\n const cQ3 = d3.quantile(cVals, 0.75);\n const cIqr = cQ3 - cQ1;\n\n return intervalData\n .filter((d) => (intervalAxes.clip ? d.outcome : 1))\n .filter(\n (x) =>\n x[intervalAxes.x] >= xQ1 - intervalAxes.k * xIqr &&\n x[intervalAxes.x] <= xQ3 + intervalAxes.k * xIqr\n )\n .filter(\n (c) =>\n c[intervalAxes.c] >= cQ1 - intervalAxes.k * cIqr &&\n c[intervalAxes.c] <= cQ3 + intervalAxes.k * cIqr\n )\n .sort((a, b) => d3.ascending(a[intervalAxes.c], b[intervalAxes.c]));\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1542,"value":"Plot.plot({\n marginLeft: 60,\n y: { grid: true },\n color: { scheme: \"cool\", legend: true },\n\n marks: [\n Plot.rectY(\n filteredIntervalData,\n Plot.binX(\n { y: \"count\" },\n { x: intervalAxes.x, fill: intervalAxes.c, sort: \"z\" }\n )\n ),\n Plot.ruleY([0])\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1598,"value":"{\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: intervalAxes.x,\n y: intervalAxes.y,\n type: r[0],\n r2: r[1].x((d) => d[intervalAxes.x]).y((d) => d[intervalAxes.c])(\n filteredIntervalData\n ).rSquared\n }))\n .flat(Infinity)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1402,"value":"---\n#### Velocity","pinned":false,"hidden":false,"mode":"md"},{"id":1236,"value":"$38","pinned":false,"hidden":false,"mode":"ojs"},{"id":1299,"value":"viewof velocityAxes = Inputs.form({\n x: Inputs.radio(\n [\n \"minimumVelocity\",\n \"maximumVelocity\",\n \"averageVelocity\",\n \"velocityVariance\",\n\n \"decelerationCount\",\n \"outcome\"\n ],\n { label: \"X-Axis\", value: \"velocityVariance\" }\n ),\n y: Inputs.radio(\n [\n \"minimumVelocity\",\n \"maximumVelocity\",\n \"averageVelocity\",\n \"velocityVariance\",\n\n \"decelerationCount\",\n \"outcome\"\n ],\n { label: \"Y-Axis\", value: \"outcome\" }\n ),\n c: Inputs.radio(\n [\n \"minimumVelocity\",\n \"maximumVelocity\",\n \"averageVelocity\",\n \"velocityVariance\",\n\n \"decelerationCount\",\n \"outcome\"\n ],\n { label: \"Color-Axis\", value: \"averageVelocity\" }\n ),\n k: Inputs.range([0, 20], {\n label: \"How Many Standard Deviations?\",\n value: 2,\n step: 0.5\n }),\n clip: Inputs.toggle({ label: \"Clip to only students that finished?\" })\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1342,"value":"velocityMetricsFiltered = {\n const xMean = d3.mean(velocityMetrics.map((d) => d[velocityAxes.x]));\n\n const xStd = d3.deviation(velocityMetrics.map((d) => d[velocityAxes.x]));\n\n const yMean = d3.mean(velocityMetrics.map((d) => d[velocityAxes.y]));\n const yStd = d3.deviation(velocityMetrics.map((d) => d[velocityAxes.y]));\n\n const cMean = d3.mean(velocityMetrics.map((d) => d[velocityAxes.c]));\n const cStd = d3.deviation(velocityMetrics.map((d) => d[velocityAxes.c]));\n\n return velocityMetrics\n .filter((d) => (velocityAxes.clip ? d.outcome : 1))\n .filter((x) => Math.abs(x[velocityAxes.x] - xMean) <= velocityAxes.k * xStd)\n .filter((y) => Math.abs(y[velocityAxes.y] - yMean) <= velocityAxes.k * yStd)\n .filter(\n (c) => Math.abs(c[velocityAxes.c] - cMean) <= velocityAxes.k * cStd\n );\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1292,"value":"Plot.plot({\n width: width,\n height: 500,\n inset: 8,\n grid: true,\n color: {\n legend: true,\n scheme: \"Cool\"\n },\n marks: [\n Plot.dot(velocityMetricsFiltered, {\n x: velocityAxes.x,\n y: velocityAxes.y,\n fill: velocityAxes.c,\n r: 1\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1590,"value":"{\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: velocityAxes.x,\n y: velocityAxes.y,\n type: r[0],\n r2: r[1].x((d) => d[velocityAxes.x]).y((d) => d[velocityAxes.y])(\n velocityMetricsFiltered\n ).rSquared\n }))\n .flat(Infinity)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1405,"value":"----\n#### Pause Analysis","pinned":false,"hidden":false,"mode":"md"},{"id":1242,"value":"$39","pinned":false,"hidden":false,"mode":"ojs"},{"id":1408,"value":"$3a","pinned":false,"hidden":false,"mode":"ojs"},{"id":1425,"value":"pauseAnalysisFiltered = {\n const xMean = d3.mean(pauseAnalysis.map((d) => d[pauseAxes.x]));\n\n const xStd = d3.deviation(pauseAnalysis.map((d) => d[pauseAxes.x]));\n\n const yMean = d3.mean(pauseAnalysis.map((d) => d[pauseAxes.y]));\n const yStd = d3.deviation(pauseAnalysis.map((d) => d[pauseAxes.y]));\n\n const cMean = d3.mean(pauseAnalysis.map((d) => d[pauseAxes.c]));\n const cStd = d3.deviation(pauseAnalysis.map((d) => d[pauseAxes.c]));\n\n return pauseAnalysis\n .filter((d) => (pauseAxes.clip ? d.outcome : 1))\n .filter((x) => Math.abs(x[pauseAxes.x] - xMean) <= pauseAxes.k * xStd)\n .filter((y) => Math.abs(y[pauseAxes.y] - yMean) <= pauseAxes.k * yStd)\n .filter((c) => Math.abs(c[pauseAxes.c] - cMean) <= pauseAxes.k * cStd);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1434,"value":"Plot.plot({\n width: width,\n height: 500,\n inset: 8,\n grid: true,\n color: {\n legend: true,\n scheme: \"Cool\"\n },\n marks: [\n Plot.dot(pauseAnalysisFiltered, {\n x: pauseAxes.x,\n y: pauseAxes.y,\n fill: pauseAxes.c,\n r: 1\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1604,"value":"{\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: pauseAxes.x,\n y: pauseAxes.y,\n type: r[0],\n r2: r[1].x((d) => d[pauseAxes.x]).y((d) => d[pauseAxes.y])(\n pauseAnalysisFiltered\n ).rSquared\n }))\n .flat(Infinity)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1440,"value":"----\n#### Regularity Scoring","pinned":false,"hidden":false,"mode":"md"},{"id":1245,"value":"$3b","pinned":false,"hidden":false,"mode":"ojs"},{"id":1448,"value":"----\n#### FFT Deconstruction ","pinned":false,"hidden":false,"mode":"md"},{"id":1248,"value":"// Cell 7: Prepare data for FFT analysis\nfftData = learnerIntervals.map((student) => {\n if (student.intervals.length < 4) {\n return {\n studentId: student.studentId,\n signal: [],\n fftReady: false\n };\n }\n\n // Extract interval durations as signal\n const signal = student.intervals.map((i) => i.hours);\n\n // Pad to nearest power of 2\n const nextPow2 = Math.pow(2, Math.ceil(Math.log2(signal.length)));\n const padded = [...signal, ...new Array(nextPow2 - signal.length).fill(0)];\n\n // Apply Hanning window\n const windowed = padded.map((value, i) => {\n const window =\n 0.5 * (1 - Math.cos((2 * Math.PI * i) / (padded.length - 1)));\n return value * window;\n });\n\n return {\n studentId: student.studentId,\n outcome: student.outcome,\n signal: windowed,\n originalLength: signal.length,\n fftLength: nextPow2,\n sampleRate: student.totalDays / signal.length,\n fftReady: true\n };\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1251,"value":"$3c","pinned":false,"hidden":false,"mode":"ojs"},{"id":1453,"value":"$3d","pinned":false,"hidden":false,"mode":"ojs"},{"id":1462,"value":"fftFiltered = {\n const xMean = d3.mean(fastFourierAnalysis.map((d) => d[fftAxes.x]));\n\n const xStd = d3.deviation(fastFourierAnalysis.map((d) => d[fftAxes.x]));\n\n const yMean = d3.mean(fastFourierAnalysis.map((d) => d[fftAxes.y]));\n const yStd = d3.deviation(fastFourierAnalysis.map((d) => d[fftAxes.y]));\n\n const cMean = d3.mean(fastFourierAnalysis.map((d) => d[fftAxes.c]));\n const cStd = d3.deviation(fastFourierAnalysis.map((d) => d[fftAxes.c]));\n\n return fastFourierAnalysis\n .filter((d) => (fftAxes.clip ? d.outcome : 1))\n .filter((x) => Math.abs(x[fftAxes.x] - xMean) <= fftAxes.k * xStd)\n .filter((y) => Math.abs(y[fftAxes.y] - yMean) <= fftAxes.k * yStd)\n .filter((c) => Math.abs(c[fftAxes.c] - cMean) <= fftAxes.k * cStd);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1466,"value":"Plot.plot({\n width: width,\n height: 500,\n inset: 8,\n grid: true,\n color: {\n legend: true,\n scheme: \"Cool\"\n },\n marks: [\n Plot.dot(fftFiltered, {\n x: fftAxes.x,\n y: fftAxes.y,\n fill: fftAxes.c,\n r: 1\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1610,"value":"{\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: fftAxes.x,\n y: fftAxes.y,\n type: r[0],\n r2: r[1].x((d) => d[fftAxes.x]).y((d) => d[fftAxes.y])(fftFiltered)\n .rSquared\n }))\n .flat(Infinity)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1483,"value":"---\n#### Wavelet (DWT) Analysis","pinned":false,"hidden":false,"mode":"md"},{"id":1790,"value":"$3e","pinned":false,"hidden":false,"mode":"ojs"},{"id":1255,"value":"","pinned":false,"hidden":false,"mode":"ojs"},{"id":1474,"value":"viewof waveletAxes = Inputs.form({\n x: Inputs.radio(\n [\n \"totalEnergy\",\n \"energyEntropy\",\n \"dominantScale\",\n \"outcome\",\n \"timestampCount\"\n ],\n { label: \"X-Axis\", value: \"totalEnergy\" }\n ),\n y: Inputs.radio(\n [\n \"totalEnergy\",\n \"energyEntropy\",\n \"dominantScale\",\n \"outcome\",\n \"timestampCount\"\n ],\n { label: \"Y-Axis\", value: \"outcome\" }\n ),\n c: Inputs.radio(\n [\n \"totalEnergy\",\n \"energyEntropy\",\n \"dominantScale\",\n \"outcome\",\n \"timestampCount\"\n ],\n { label: \"Color-Axis\", value: \"dominantScale\" }\n ),\n k: Inputs.range([0, 20], {\n label: \"How Many Standard Deviations?\",\n value: 2,\n step: 0.5\n }),\n clip: Inputs.toggle({ label: \"Clip to only students that finished?\" })\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1486,"value":"waveletFiltered = {\n const xMean = d3.mean(waveletAnalysis.map((d) => d[waveletAxes.x]));\n\n const xStd = d3.deviation(waveletAnalysis.map((d) => d[waveletAxes.x]));\n\n const yMean = d3.mean(waveletAnalysis.map((d) => d[waveletAxes.y]));\n const yStd = d3.deviation(waveletAnalysis.map((d) => d[waveletAxes.y]));\n\n const cMean = d3.mean(waveletAnalysis.map((d) => d[waveletAxes.c]));\n const cStd = d3.deviation(waveletAnalysis.map((d) => d[waveletAxes.c]));\n\n return waveletAnalysis\n .filter((d) => (waveletAxes.clip ? d.outcome : 1))\n .filter((x) => Math.abs(x[waveletAxes.x] - xMean) <= waveletAxes.k * xStd)\n .filter((y) => Math.abs(y[waveletAxes.y] - yMean) <= waveletAxes.k * yStd)\n .filter((c) => Math.abs(c[waveletAxes.c] - cMean) <= waveletAxes.k * cStd);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1490,"value":"Plot.plot({\n width: width,\n height: 500,\n inset: 8,\n grid: true,\n color: {\n legend: true,\n scheme: \"Cool\"\n },\n marks: [\n Plot.dot(waveletFiltered, {\n x: waveletAxes.x,\n y: waveletAxes.y,\n fill: waveletAxes.c,\n r: 1\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1656,"value":"{\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: waveletAxes.x,\n y: waveletAxes.y,\n type: r[0],\n r2: r[1].x((d) => d[waveletAxes.x]).y((d) => d[waveletAxes.y])(\n waveletFiltered\n ).rSquared\n }))\n .flat(Infinity)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1495,"value":"---\n#### Cumulative Signals Analysis","pinned":false,"hidden":false,"mode":"md"},{"id":1264,"value":"$3f","pinned":false,"hidden":false,"mode":"ojs"},{"id":1499,"value":"$40","pinned":false,"hidden":false,"mode":"ojs"},{"id":1511,"value":"finalFiltered = {\n const xMean = d3.mean(finalPacingMetrics.map((d) => d[finalAxes.x]));\n\n const xStd = d3.deviation(finalPacingMetrics.map((d) => d[finalAxes.x]));\n\n const yMean = d3.mean(finalPacingMetrics.map((d) => d[finalAxes.y]));\n const yStd = d3.deviation(finalPacingMetrics.map((d) => d[finalAxes.y]));\n\n const cMean = d3.mean(finalPacingMetrics.map((d) => d[finalAxes.c]));\n const cStd = d3.deviation(finalPacingMetrics.map((d) => d[finalAxes.c]));\n\n return finalPacingMetrics\n .filter((d) => (finalAxes.clip ? d.outcome : 1))\n .filter((x) => Math.abs(x[finalAxes.x] - xMean) <= finalAxes.k * xStd)\n .filter((y) => Math.abs(y[finalAxes.y] - yMean) <= finalAxes.k * yStd)\n .filter((c) => Math.abs(c[finalAxes.c] - cMean) <= finalAxes.k * cStd);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1514,"value":"Plot.plot({\n width: width,\n height: 500,\n inset: 8,\n grid: true,\n color: {\n legend: true,\n scheme: \"Cool\"\n },\n marks: [\n Plot.dot(finalFiltered, {\n x: finalAxes.x,\n y: finalAxes.y,\n fill: finalAxes.c,\n r: 1\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1742,"value":"{\n const regressions = [\n [\"Linear\", d3.regressionLinear()],\n [\"Quadratic\", d3.regressionQuad()],\n [\"Exponential\", d3.regressionExp()],\n [\"Logarithmic\", d3.regressionLog()],\n [\"Polynomial 3\", d3.regressionPoly()]\n ];\n\n return Inputs.table(\n regressions\n .map((r) => ({\n x: finalAxes.x,\n y: finalAxes.y,\n type: r[0],\n r2: r[1].x((d) => d[finalAxes.x]).y((d) => d[finalAxes.y])(\n finalFiltered\n ).rSquared\n }))\n .flat(Infinity)\n .sort((a, b) => d3.descending(a.r2, b.r2))\n ); //\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1529,"value":"\n---\n#### Summary Statistics ","pinned":false,"hidden":false,"mode":"md"},{"id":1274,"value":"$41","pinned":false,"hidden":false,"mode":"ojs"},{"id":1746,"value":"---\n#### Interventions","pinned":false,"hidden":false,"mode":"md"},{"id":1260,"value":"$42","pinned":false,"hidden":false,"mode":"ojs"},{"id":1756,"value":"viewof interventionAxes = Inputs.form({\n x: Inputs.radio([\"outcome\", \"riskScore\", \"primaryIndex\"], {\n label: \"X-Axis\",\n value: \"primaryIndex\"\n }),\n y: Inputs.radio([\"outcome\", \"riskScore\", \"primaryIndex\"], {\n label: \"Y-Axis\",\n value: \"outcome\"\n }),\n c: Inputs.radio([\"outcome\", \"riskScore\", \"primaryIndex\"], {\n label: \"Color-Axis\",\n value: \"riskScore\"\n }),\n k: Inputs.range([0, 20], {\n label: \"How Many Standard Deviations?\",\n value: 2,\n step: 0.5\n }),\n clip: Inputs.toggle({ label: \"Clip to only students that finished?\" })\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1778,"value":"interventionFiltered = {\n const xMean = d3.mean(interventionMapping.map((d) => d[interventionAxes.x]));\n\n const xStd = d3.deviation(\n interventionMapping.map((d) => d[interventionAxes.x])\n );\n\n const yMean = d3.mean(interventionMapping.map((d) => d[interventionAxes.y]));\n const yStd = d3.deviation(\n interventionMapping.map((d) => d[interventionAxes.y])\n );\n\n const cMean = d3.mean(interventionMapping.map((d) => d[interventionAxes.c]));\n const cStd = d3.deviation(\n interventionMapping.map((d) => d[interventionAxes.c])\n );\n\n return interventionMapping\n .filter((d) => (interventionAxes.clip ? d.outcome : 1))\n .filter(\n (x) =>\n Math.abs(x[interventionAxes.x] - xMean) <= interventionAxes.k * xStd\n )\n .filter(\n (y) =>\n Math.abs(y[interventionAxes.y] - yMean) <= interventionAxes.k * yStd\n )\n .filter(\n (c) =>\n Math.abs(c[interventionAxes.c] - cMean) <= interventionAxes.k * cStd\n );\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1754,"value":"Plot.plot({\n width: width,\n height: 500,\n inset: 8,\n grid: true,\n color: {\n legend: true,\n scheme: \"Cool\"\n },\n marks: [\n Plot.dot(interventionFiltered, {\n x: interventionAxes.x,\n y: interventionAxes.y,\n fill: interventionAxes.c,\n r: 1\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W43","stdlib":"1","titling":"auto"}