32:T699,getSsdBtwLineAndStd = function(lineDataEvol, stdLine, rgb = false,slice = false){ let start =0; let end = lineDataEvol[0].columnData.length; if(slice){ start = slice[0]; end = slice[1]; } let p_stdLine = stdLine || lineDataEvol[0].columnData.slice(start,end); // slice to select only a range of pixels if(slice){ p_stdLine = p_stdLine.slice(start,end); } let SSDArray = lineDataEvol.map(d => {return {'value':sumSquareDiff(d.columnData.slice(start,end),p_stdLine),'time':d.time}}); // normalization let max = d3.max(SSDArray, d=>d.value); max = max > 0 ? max : 1; SSDArray.map(d => { d.value = d.value/max; return d }); // if we want to have separate r,g,b SSD if(rgb){ let SSDArrayRed = lineDataEvol.map( d => {return {'value':sumSquareDiff(d3.filter(d.columnData.slice(start,end), (d,i) => (i+1)%3==1), d3.filter(p_stdLine, (d,i) => (i+1)%3==2)), 'time':d.time} } ); let SSDArrayGreen = lineDataEvol.map( d => {return {'value':sumSquareDiff(d3.filter(d.columnData.slice(start,end), (d,i) => (i+1)%3==2), d3.filter(p_stdLine, (d,i) => (i+1)%3==1)), 'time':d.time} } ); let SSDArrayBlue = lineDataEvol.map( d => {return {'value':sumSquareDiff(d3.filter(d.columnData.slice(start,end), (d,i) => (i+1)%3==0), d3.filter(p_stdLine, (d,i) => (i+1)%3==0)), 'time':d.time} } ); return [SSDArray,SSDArrayRed,SSDArrayGreen,SSDArrayBlue] } return SSDArray }33:T42b,getSaturationThresholds = function(stdLineSat){ let sumMaskSaturationStdLineEvol = getSumMaskSaturationStdLineEvol(stdLineSat); // find the threshold value were the variation is the most important let maskVariation = d3.pairs(sumMaskSaturationStdLineEvol).map(([a,b]) => Math.abs(b.value-a.value)); let thresholdIdx = d3.maxIndex(maskVariation); // here time corresponds to a saturation value btw 0 and 1 let thresholdInfIdx = thresholdIdx; while((maskVariation[thresholdInfIdx]>0 || maskVariation[thresholdInfIdx+1]>0 ) && thresholdInfIdx>0){thresholdInfIdx-=1}; let thresholdInf = sumMaskSaturationStdLineEvol[thresholdInfIdx].time; let thresholdSupIdx = thresholdIdx; while((maskVariation[thresholdSupIdx]>0 || maskVariation[thresholdInfIdx-1]>0 ) && thresholdInfIdx Math.abs(b.value-a.value)); let thresholdIdx = d3.maxIndex(maskVariation); // here time corresponds to a saturation value btw 0 and 1 let thresholdInfIdx = thresholdIdx; while((maskVariation[thresholdInfIdx]>0 || maskVariation[thresholdInfIdx+1]>0 ) && thresholdInfIdx>0) {thresholdInfIdx-=1;}; let thresholdInf = stdLineMaskValueEvol[thresholdInfIdx].time; let thresholdSupIdx = thresholdIdx; while((maskVariation[thresholdSupIdx]>0 || maskVariation[thresholdSupIdx-1]>0 ) && thresholdSupIdx { return {'value':sumSquareDiff(a.columnData,b.columnData), 'time':b.time} }); let cosConsecutiveFrames = d3.pairs(lineDataEvol) .map(([a,b]) => { return {'value':1-cosineSimilarity(a.columnData,b.columnData), 'time':b.time} }); let multipliedArray = ssdConsecutiveFrames.map((d,i) => { return {'value':d.value*cosConsecutiveFrames[i].value, 'time':d.time} }); // normalize array let max = d3.max(multipliedArray,d => d.value) > 0 ? d3.max(multipliedArray,d => d.value):1; let normMultipliedArray = multipliedArray.map(d => {return {'value':d.value/max,'time':d.time}}); return normMultipliedArray }36:T433,md`## Candidate Signals We finally have 3 (or even 5) candidate signals to allow us to detect the passage of a car, * The evolution of the difference between the RGB vector representing the line of pixels and this same vector representing the line of pixels at an instant for which we know that there are no cars (what we call *standard line*). Parameters of this signal are, * the *standard line*. We try to automatize its detection but in some cases, this detection may fail. * The evolution of the difference between the saturation mask for the line and the saturation for the *standard line*. The parameters are, * the threshold that defines the mask * The evolution of the difference between the value mask for the line and the value for the *standard line*. The parameters are, * the threshold that defines the mask The difference between two vectors is calculated by multiplying the Normalized Sum Squared Difference with the Cosine Difference (1 - cosine similarity). Other parameters are, * the line of pixels we chose to work on * the road lane * the video`37:T752,getSignalMaskSaturation = function( lineDataEvol, // RGB data evolution options = {} // options can be the stdLine index and the thresholds to apply the mask ) { // convert lineDataEvol to Hsv let lineDataEvolHsv = convertLineData2Hsv(lineDataEvol); // Extract only saturation evolution for each pixel let lineDataEvolSat = getSaturationEvol(lineDataEvolHsv); // autodetect stdLineIdx except if stdLineIdx is given let p_stdLineIdx = options.stdLineIdx || autoDetectStdLine(lineDataEvol,{'idx':true}); let stdLineSat = lineDataEvolSat[p_stdLineIdx].columnData; // thersholds to apply the mask let thresholds = options.thresholds || getSaturationThresholds(stdLineSat); // get mask evolution for both thresholds let saturationMaskEvolInf = getSaturationMaskEvol(lineDataEvolSat,thresholds.inf); let saturationMaskEvolSup = getSaturationMaskEvol(lineDataEvolSat,thresholds.sup); // compute the difference between line saturation mask evolution and standard line saturation mask evolution let saturationMaskEvolInfDiff = getCosSsdDiffBtwLineAndStd(saturationMaskEvolInf, saturationMaskEvolInf[p_stdLineIdx].columnData); let saturationMaskEvolSupDiff = getCosSsdDiffBtwLineAndStd(saturationMaskEvolSup, saturationMaskEvolSup[p_stdLineIdx].columnData); // in the case we want to debug and understand parameters set by the algorithm, we return these parameters return { 'signalInf':saturationMaskEvolInfDiff, 'signalSup':saturationMaskEvolSupDiff, 'signal':saturationMaskEvolSupDiff, 'thresholds':thresholds, 'stdLineIdx':p_stdLineIdx, 'stdLineTime':lineDataEvol[p_stdLineIdx].time, 'lineDataEvolInf':saturationMaskEvolInf, 'lineDataEvolSup':saturationMaskEvolSup, } }38:T6b3,getSignalMaskValue = function( lineDataEvol, // RGB data evolution options = {} ) { // convert lineDataEvol to Hsv let lineDataEvolHsv = convertLineData2Hsv(lineDataEvol); // Extract only value evolution for each pixel let lineDataEvolVal = getValueEvol(lineDataEvolHsv); // autodetect stdLineIdx except if stdLineIdx is given let p_stdLineIdx = options.stdLineIdx || autoDetectStdLine(lineDataEvol,{'idx':true}); let stdLineVal = lineDataEvolVal[p_stdLineIdx].columnData; // thersholds to apply the mask let thresholds = options.thresholds || getValueThresholds(stdLineVal); // get mask evolution for both thresholds let valueMaskEvolInf = getSaturationMaskEvol(lineDataEvolVal,thresholds.inf); let valueMaskEvolSup = getSaturationMaskEvol(lineDataEvolVal,thresholds.sup); // compute the difference between line saturation mask evolution and standard line saturation mask evolution let valueMaskEvolInfDiff = getCosSsdDiffBtwLineAndStd(valueMaskEvolInf, valueMaskEvolInf[p_stdLineIdx].columnData); let valueMaskEvolSupDiff = getCosSsdDiffBtwLineAndStd(valueMaskEvolSup, valueMaskEvolSup[p_stdLineIdx].columnData); // in the case we want to debug and understand parameters set by the algorithm, we return these parameters return { 'signalInf':valueMaskEvolInfDiff, 'signalSup':valueMaskEvolSupDiff, 'signal':valueMaskEvolSupDiff, 'thresholds':thresholds, 'stdLineIdx':p_stdLineIdx, 'stdLineTime':lineDataEvol[p_stdLineIdx].time, 'lineDataEvolInf':valueMaskEvolInf, 'lineDataEvolSup':valueMaskEvolSup } }39:T434,drawValueMask = function(lineDataEvol,valueThld){ const lineToDraw = lineDataEvol[0].columnData; const nbLines = lineDataEvol.length; const w = 3; const h = lineToDraw.length/3; const canvas = DOM.canvas(1000, h+10); const ctx = canvas.getContext('2d'); for (let k = 0; k < nbLines; k+=1){ const lineToDraw = lineDataEvol[k].columnData; const imageData = ctx.createImageData(w, h); // Iterate through every pixel for (let i = 0; i < imageData.data.length; i += 4) { let j=Math.ceil(i/(4*w)); // Modify pixel data let r = lineToDraw[j*3]; let g = lineToDraw[j*3+1]; let b = lineToDraw[j*3+2]; let hsv = rgb2Hsv(r,g,b) let value = 0; if((hsv.v > valueThld)){value=1} imageData.data[i + 0] = value*255; // R value imageData.data[i + 1] = value*255; // G value imageData.data[i + 2] = value*255; // B value imageData.data[i + 3] = 255; // A value } // Draw image data to the canvas ctx.putImageData(imageData, k*w, 0); } return canvas }3a:T435,drawSaturationMask = function(lineDataEvol,satThld){ const lineToDraw = lineDataEvol[0].columnData; const nbLines = lineDataEvol.length; const w = 3; const h = lineToDraw.length/3; const canvas = DOM.canvas(1000, h+10); const ctx = canvas.getContext('2d'); for (let k = 0; k < nbLines; k+=1){ const lineToDraw = lineDataEvol[k].columnData; const imageData = ctx.createImageData(w, h); // Iterate through every pixel for (let i = 0; i < imageData.data.length; i += 4) { let j=Math.ceil(i/(4*w)); // Modify pixel data let r = lineToDraw[j*3]; let g = lineToDraw[j*3+1]; let b = lineToDraw[j*3+2]; let hsv = rgb2Hsv(r,g,b) let value = 0; if((hsv.s > satThld)){value=1} imageData.data[i + 0] = value*255; // R value imageData.data[i + 1] = value*255; // G value imageData.data[i + 2] = value*255; // B value imageData.data[i + 3] = 255; // A value } // Draw image data to the canvas ctx.putImageData(imageData, k*w, 0); } return canvas }3b:[] 30:{"title":"Extract a signal from a line of pixels to detect cars passage","cells":[{"id":0,"value":"md`# Extract a signal from a line of pixels to detect cars passage\n\nThe purpose of this notebook is to explore approaches to detect the passage of a car thanks to local variation of pixels of a \"line\" perpendicular to the track.\n\n*A demonstration of the final result which consists in counting the number of cars in circulation is available [here](https://observablehq.com/d/b4945894fb574964?collection=@grolland/trackfic-public-version).*`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3707,"value":"md`## Example\n\nLet's imagine we want detect cars that cross the red line.\n\nThe example below shows a signal which corresponds to the evolution of the difference between the line of pixels drawn above the graph and a standard line on which we know that there is no cars. By the difference we mean a metric that accounts for the angle and the distance between the RGB vectors which represent the lines.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4539,"value":"drawLineOnImage(columnIdx,getFrameAt(t0,videoFrames,timeRes),selectedLane,{w:350})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4525,"value":"displayLineDataEvol(lineDataEvol,{hFactor:3.5,drawCursor:true,tCursor:t0,leftPad:56,w:4.2})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4518,"value":"graph([signalExample],{t:t0,ytitle:'Difference to std line',h:100,showLegend:true,legendDomain:['Signal'],t:t0})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4530,"value":"viewof t0 = Scrubber(times,{loop:false})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4552,"value":"md`By scrolling the time slider, we can see that the passage of cars is well highlighted by this signal. \nIn particular we can see that although the two cars between 7 and 8.5 seconds are close, the signal peaks down which may allow to separate them.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4566,"value":"md`\n---\n## Detailed approach\n---\n\nConsider that we want to count the number of cars that pass on the lane highlighted below. Our approach is to look at the evolution of the red line orthogonal to the lane. Thanks to [the previous notebook](https://observablehq.com/d/66467e3b94445c09?collection=@grolland/trackfic-public-version), we have the evolution of this line of pixels frame by frame which is represented by a sorted array that contains the R,G,B components of each pixel of the line. `","pinned":false,"hidden":false,"mode":"ojs"},{"id":62,"value":"drawLineOnImage(columnIdx,getFrameAt(t1,videoFrames,timeRes),selectedLane,{w:350})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4591,"value":"md`Evolution of the line of pixels frame by frame :`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4574,"value":"displayLineDataEvol(lineDataEvol,{hFactor:2,drawCursor:true,tCursor:t1,w:3})","pinned":false,"hidden":false,"mode":"ojs"},{"id":4580,"value":"viewof t1 = Scrubber(times,{loop:false,w:500})","pinned":false,"hidden":false,"mode":"ojs"},{"id":99,"value":"md`According to a manual count on the video, there are 7 cars that \"cross\" the line, so our goal is to detect these 7 variations on this line of pixels.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":59,"value":"md`\nWe aim to convert the R,G,B data of the line into a signal that we want to analyze afterwards to detect the passage of cars. This signal must be able to detect the pixel variations due to the passage of cars.\n\n## Average of the R,G,B components over all pixels\n\nLet's start with a simple signal, i.e the average of R,G,B components over all pixels. The results is shown below. Blue, green and red lines respectly represent the average of each color while the gray line represent the average of the 3 components which corresponds to the greyscale average.\n\nThe grey bands above the graph represents the manually detected cars.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1314,"value":"graph([averageBLine,averageRLine,averageGLine,averageRGBLine],{t:t3,xtitle:'Time [s]',ytitle:'R,G,B average values',groundTruth:groundTruthTimes,color:['steelblue','red','green','grey'],h:100,showLegend:true,legendDomain:['Line blue avg','Line green avg','Line red avg','Line RGB avg']})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1111,"value":"viewof t3 = Scrubber(times,{loop:false})","pinned":false,"hidden":false,"mode":"ojs"},{"id":107,"value":"md`We seem to be able to detect a number of variations. However, it is difficult to distinguish 7 of them distinctly. Moreover, thanks to this first approach, we understand that one of the difficulties will be to distinguish 2 cars that are close to each other like the 5th and 6th ones.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2124,"value":"md`## Compare the line of pixels to a *standard line*\nIn order to highlight the variations due to the passage of a car, we will compare the vector characterizing th line of pixels with a vector at a time when we know that there are no cars (what we call *standard line*).\n\nWe will use two methods to compare two vectors, the first is to calculate the Sum of Squared difference between these two vectors and the second is based on the angle between these two vectors.\n\n### Sum of Squared Differences (SSD).\nWe seek to measure the difference between two vectors. For this purpose, we can start by computing the sum of squared differences between them.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":151,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":164,"value":"graph([ssdBtwLineAndStd[0]],{t:t4,ytitle:'SSD btwn line & std line',groundTruth:groundTruthTimes,h:100,showLegend:true,legendDomain:['Signal']})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1116,"value":"viewof t4 = Scrubber(times,{loop:false})","pinned":false,"hidden":false,"mode":"ojs"},{"id":167,"value":"md`In this case, we can distinguish 7 different variations although the two cars between 7 and 8.5s are difficult to split. `","pinned":false,"hidden":false,"mode":"ojs"},{"id":2119,"value":"md`### Cosine similarity between the line and the *standard line*\n\nThe sum of the squared differences measures the distance between the pixel line and the standard line. However, to compare two vectors, it may make sense to use the angle between them. The cosine similiarity allows you to measure the similarity between two vectors taking into account the angle between them. Since we want to measure the difference, we will calculate *cosine difference* = 1 - cosine similiarty.\n\n${tex`Cosine Sim(x,y) = \\frac{\\sum x_iy_i}{\\sqrt{\\sum x_i^2}\\sqrt{\\sum y_i^2}}`}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":518,"value":"getCosBtwLineAndStd = function(lineDataEvol,stdLine){\n\n let p_stdLine = stdLine || lineDataEvol[0].columnData;\n \n let simArray = lineDataEvol.map(d => {return {'value':1-cosineSimilarity(d.columnData,p_stdLine),'time':d.time}});\n \n let max = d3.max(simArray, d=>d.value);\n max = max > 0 ? max : 1;\n \n simArray.map(d => {\n d.value = d.value/max;\n return d\n });\n \n return simArray\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":572,"value":"graph([cosBtwLineAndStd],{t:t5,ytitle:'1 - CosineSim(line,std line)',groundTruth:groundTruthTimes,h:100,showLegend:true,legendDomain:['Signal']})","pinned":false,"hidden":false,"mode":"ojs"},{"id":5484,"value":"viewof t5 = Scrubber(times,{loop:false})","pinned":false,"hidden":false,"mode":"ojs"},{"id":5498,"value":"md`We can see that the overall look of the signal remains the same as with SSD. It seems that the gap between the two cars between 7 and 8.5 seconds is more marked in this case.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2298,"value":"md`### Multiply *Cosine difference* and Sum of Squared Differences.\nSince we are looking to highlight the difference between the two lines and our two previous difference measures (cosine and SSD) are between 0 and 1, we can multiply them to get a difference measure between two vectors that takes into account both the angle and the distance between them.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3172,"value":"getCosSsdDiffBtwLineAndStd = function(pixelsLine,stdLine){\n \n let p_stdLine = stdLine || pixelsLine[0].columnData;\n \n let SSDLineToStd = getSsdBtwLineAndStd(pixelsLine,p_stdLine);\n let cosSimLineStdLine = getCosBtwLineAndStd(pixelsLine,p_stdLine);\n \n let multipliedArray = SSDLineToStd.map((d,i) => {\n return {'value':d.value*cosSimLineStdLine[i].value,'time':d.time}});\n \n // normalize array\n let max = d3.max(multipliedArray,d => d.value) > 0 ? d3.max(multipliedArray,d => d.value) : 1;\n let normMultipliedArray = multipliedArray.map(d => {return {'value':d.value/max,'time':d.time}});\n \n return normMultipliedArray\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":830,"value":"graph([multiplyCosineAndSSD],{t:t6,ytitle:'CosineDiff x SSD',groundTruth:groundTruthTimes,h:100,showLegend:true,legendDomain:['Signal']})","pinned":false,"hidden":false,"mode":"ojs"},{"id":5502,"value":"viewof t6 = Scrubber(times,{loop:false})","pinned":false,"hidden":false,"mode":"ojs"},{"id":5664,"value":"md`We notice that this multiplication allows us to increase the signal gap between the close cars (between 7 and 8.5s). From now on, when we want to compare two vectors (here the line and the standard line) **we will use this difference measure, i.e ${tex`CosineDiff*SSD`}**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2570,"value":"md`## From RGB to HSV`","pinned":false,"hidden":false,"mode":"ojs"},{"id":5891,"value":"md`Hue`","pinned":false,"hidden":false,"mode":"ojs"},{"id":5872,"value":"displayLineDataEvol(hueEvol,{dataType:'h',hFactor:0.6})","pinned":false,"hidden":false,"mode":"ojs"},{"id":5894,"value":"md`Saturation`","pinned":false,"hidden":false,"mode":"ojs"},{"id":5859,"value":"displayLineDataEvol(saturationEvol,{dataType:'s',hFactor:0.6})","pinned":false,"hidden":false,"mode":"ojs"},{"id":5900,"value":"md`Value`","pinned":false,"hidden":false,"mode":"ojs"},{"id":5897,"value":"displayLineDataEvol(valueEvol,{dataType:'v',hFactor:0.6})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3152,"value":"md`### Saturation evolution`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3210,"value":"// supposing lineDataEvolHSV converted to HSV\n// 0 <= h <= 360; 0<= s,v <=1\n// returns an array where each item contains a vector containing the saturation value of each pixel.\ngetSaturationEvol = function(lineDataEvolHsv){\n let saturationEvol = lineDataEvolHsv.map(d => {\n let newColumnData = d3.filter(d.columnData, (d,i) => (i+1)%3==2); // extract s component from each line data\n return {'time':d.time, 'columnData':newColumnData}\n })\n return saturationEvol\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3224,"value":"saturationEvol = getSaturationEvol(lineDataEvolHsv)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3227,"value":"saturationDiffEvol = getCosSsdDiffBtwLineAndStd(saturationEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3230,"value":"graph([saturationDiffEvol],{ytitle:'Difference',groundTruth:groundTruthTimes,h:150})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3713,"value":"md`**Use a mask**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3282,"value":"viewof saturationThreshold = slider({\n value: 0.38,\n min: 0,\n max: 1,\n step: 0.01,\n description: 'saturation limit'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3297,"value":"displayLineDataEvol(saturationMaskEvol,{dataType:'s',hFactor:0.6})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3216,"value":"// saturationEvol is an array where each item contains a vector containing the saturation value of each pixel.\n// returns an array where each item contains a vector containing the saturation masked saturation value of each pixel i.e, if sat > threshold ==> 1 else 0\ngetSaturationMaskEvol = function(saturationEvol,threshold = 0.2){\n return saturationEvol.map(d => {\n return {'time':d.time,\n 'columnData':d.columnData.map(v => v>threshold ? 1:0)\n }\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3269,"value":"saturationMaskEvol = getSaturationMaskEvol(saturationEvol,saturationThreshold)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3976,"value":"satMaskDiffEvol = getCosSsdDiffBtwLineAndStd(saturationMaskEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3279,"value":"graph([satMaskDiffEvol],{ytitle:'Difference to std line',groundTruth:groundTruthTimes,h:150})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3769,"value":"md`**Determine the saturation threshold (thld)**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3772,"value":"getSumMaskSaturationStdLineEvol = function(stdLineSat){\n // stdLineSat must be in saturation format\n \n let stdLineMaskEvol = [];\n for(let sat=0; sat<1; sat+=0.01){\n stdLineMaskEvol.push({'time':sat,\n 'value':d3.sum(stdLineSat.map(v => v>sat ? 1:0))\n })\n }\n return stdLineMaskEvol\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3775,"value":"sumMaskSaturationStdLineEvol = getSumMaskSaturationStdLineEvol(saturationEvol[0].columnData)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3778,"value":"graph([sumMaskSaturationStdLineEvol],{xtitle:'Value Threshold',ytitle:'Sum of the mask',h:100})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3784,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":3801,"value":"satThlds = getSaturationThresholds(saturationEvol[0].columnData)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3809,"value":"saturationMaskEvolInf = getSaturationMaskEvol(saturationEvol,satThlds.inf)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3814,"value":"saturationMaskEvolSup = getSaturationMaskEvol(saturationEvol,satThlds.sup)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3817,"value":"graph([getCosSsdDiffBtwLineAndStd(saturationMaskEvolInf),getCosSsdDiffBtwLineAndStd(saturationMaskEvolSup)],{ytitle:'Difference',groundTruth:groundTruthTimes,h:150})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3155,"value":"md`### Value evolution`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3242,"value":"// supposing lineDataEvolHSV converted to HSV\ngetValueEvol = function(lineDataEvolHsv){\n let saturationEvol = lineDataEvolHsv.map(d => {\n let newColumnData = d3.filter(d.columnData, (d,i) => (i+1)%3==0);\n return {'time':d.time, 'columnData':newColumnData}\n })\n return saturationEvol\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3245,"value":"valueEvol = getValueEvol(lineDataEvolHsv)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3248,"value":"valueDiffEvol = getCosSsdDiffBtwLineAndStd(valueEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3251,"value":"graph([valueDiffEvol],{ytitle:'Difference',groundTruth:groundTruthTimes,h:150})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3737,"value":"md`**Use a mask**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3335,"value":"viewof valueThreshold = slider({\n value: 0.26,\n min: 0,\n max: 1,\n step: 0.01,\n description: 'value limit'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3338,"value":"displayLineDataEvol(valueMaskEvol,{dataType:'s',hFactor:0.6})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3330,"value":"getValueMaskEvol = function(valueEvol,threshold = 0.2){\n return valueEvol.map(d => {\n return {'time':d.time,\n 'columnData':d.columnData.map(v => v>threshold ? 1:0)\n }\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3333,"value":"valueMaskEvol = getValueMaskEvol(valueEvol,valueThreshold)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3342,"value":"valueMaskDiffEvol = getCosSsdDiffBtwLineAndStd(valueMaskEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3345,"value":"graph([valueMaskDiffEvol],{ytitle:'Difference',groundTruth:groundTruthTimes,h:150})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3741,"value":"md`**Determine the mask threshold value**\n\nWhen you move the slider, you will notice that the graph changes completely for a certain threshold value. We would like to find this value because it corresponds to the value for which the mask of the standard line goes from a majority of 0 to a majority of 1. Thus, we could determine a threshold above this value to detect the parts of the image that are lighter than the standard; and a threshold below to detect the darker parts.\n\nLet's plot the evolution of the standard line mask sum as a function of Value limit.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3748,"value":"getSumStdLineMaskValueEvol = function(stdLineValue){\n // stdLineValue is an array where each item contains a vector with the Value component for each pixel.\n \n let stdLineMaskEvol = [];\n for(let value=0; value<1; value+=0.01){\n stdLineMaskEvol.push({'time':value,\n 'value':d3.sum(stdLineValue.map(v => v>value ? 1:0))\n })\n }\n return stdLineMaskEvol\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3756,"value":"sumStdLineMaskValueEvol = getSumStdLineMaskValueEvol(valueEvol[0].columnData)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3759,"value":"graph([sumStdLineMaskValueEvol],{xtitle:'Value Threshold',ytitle:'Sum of the mask',h:100})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3821,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":3825,"value":"valThlds = getValueThresholds(valueEvol[0].columnData)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3828,"value":"valMaskEvolInf = getValueMaskEvol(valueEvol,valThlds.inf)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3831,"value":"valMaskEvolSup = getValueMaskEvol(valueEvol,valThlds.sup)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3837,"value":"graph([getCosSsdDiffBtwLineAndStd(valMaskEvolInf),getCosSsdDiffBtwLineAndStd(valMaskEvolSup)],{ytitle:'Difference',groundTruth:groundTruthTimes,h:150})","pinned":false,"hidden":false,"mode":"ojs"},{"id":2567,"value":"md`### Autodetect *standard line*\nSo far, the standard line is determined by taking the pixel value of the selected line at t=0. However, as can be seen below, by varying the selected line, it is possible that at t=0 there is a car, so the standard line will not fulfill its role.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3876,"value":"viewof columnIdxBis = slider({\n value: 328,\n min: 0,\n max: wProj,\n step: 1,\n description: 'Select a line'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3873,"value":"drawProjection(videoSrc,selectedLane,{columnIdx:columnIdxBis,t:0})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3888,"value":"md`In order to automatically determine when to take the standard line, we will calculate the difference between the vector representing the pixel line at time t and the same vector at time t-1. Thus, we should detect a moment for which the pixel line represents the road without any car except maybe in case of traffic jams.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3893,"value":"$35","pinned":false,"hidden":false,"mode":"ojs"},{"id":3891,"value":"autoDetectStdLine = function(lineDataEvol,options={}){\n // lineDataEvol should be in RGB format\n \n \n let time = options.time || false; // set time to true if you want to get the time when the standard line is selected instead of the value\n let idx = options.idx || false; // set idx to true if you want to get the idx of the standard line is selected instead of the value\n \n // get the difference between two consecutive line of pixels for each time\n let cosSsdBtwConsecutiveFrames = getCosSsdBtwConsecutiveFrames(lineDataEvol);\n \n // Retrieve the line for which this difference is minimal\n let stdLineIndex = d3.minIndex(cosSsdBtwConsecutiveFrames, d => d.value);\n \n if(time){\n return lineDataEvol[stdLineIndex].time\n }\n \n if(idx){\n return stdLineIndex\n }\n \n return lineDataEvol[stdLineIndex].columnData\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3913,"value":"md`Let's test with the line selected above.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3910,"value":"lineDataEvolTest = realLineEvolProc(\n selectedLane, // the area on which you want to get a line of pixel\n videoFrames, // the frames of the video\n columnIdxBis, // choose the column you want on the rect\n hProj, // assuming you project your area on a rect. hProj is the height of this proj\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3909,"value":"timeStdLineTest = autoDetectStdLine(lineDataEvolTest,{'time':true})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3924,"value":"drawProjection(videoSrc,selectedLane,{columnIdx:columnIdxBis,t:timeStdLineTest})","pinned":false,"hidden":false,"mode":"ojs"},{"id":3927,"value":"md`As we can see above, the result as expected, since at the time detected, there's no car on the line.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":855,"value":"$36","pinned":false,"hidden":false,"mode":"ojs"},{"id":3952,"value":"md`**RGB difference**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3966,"value":"getSignalRGB = function(lineDataEvol,stdLineIdx){\n // autodetect stdLineIdx except if stdLine is given\n let p_stdLineIdx = stdLineIdx || autoDetectStdLine(lineDataEvol,{'idx':true});\n let p_stdLine = lineDataEvol[p_stdLineIdx].columnData;\n \n // compute the evolution of the difference between the line of pixels and the standard line\n let signal = getCosSsdDiffBtwLineAndStd(lineDataEvol,p_stdLine);\n \n // in the case we want to debug and understand parameters set by the algorithm, we return these parameters\n return {\n 'signal':signal,\n 'signalInf':signal,\n 'signalSup':signal,\n 'stdLineIdx':p_stdLineIdx,\n 'stdLineTime':lineDataEvol[p_stdLineIdx].time,\n 'lineDataEvol':lineDataEvol,\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4136,"value":"signalRGB = getSignalRGB(lineDataEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3956,"value":"md`**Saturation Mask Difference**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4109,"value":"$37","pinned":false,"hidden":false,"mode":"ojs"},{"id":4122,"value":"signalMaskSaturation = getSignalMaskSaturation(lineDataEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":3960,"value":"md`**Value Mask Difference**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4142,"value":"$38","pinned":false,"hidden":false,"mode":"ojs"},{"id":4157,"value":"signalMaskValue = getSignalMaskValue(lineDataEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":2132,"value":"md`\n---\n## Credits\n\n*Developed by Gabin Rolland, Bob-Dylan Bety, Lucas Heinry and Thomas Brisson as part of a student project at École Centrale de Lyon, supervised by Romain Vuillemot.*\n\n---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":861,"value":"md`\n## Appendix : code\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3661,"value":"md`### Functions`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3665,"value":"norm = (array) => Math.sqrt(array.reduce((a, b) => a + b**2)+array[0]**2-array[0]);","pinned":false,"hidden":false,"mode":"ojs"},{"id":3669,"value":"sumSquareDiff = function(array1, array2){\n var diff = [];\n \n const norm1 = norm(array1);\n const norm2 = norm(array2);\n \n if(norm1==0){return norm2};\n if(norm2==0){return norm1};\n \n for (let i=0; i < array1.length; i++){\n diff.push(array1[i] - array2[i]);\n }\n return norm(diff)**2/(norm1*norm2)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3682,"value":"dot = function(v1,v2){\n return d3.sum(v1.map((d,i) => d*v2[i]))\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":3687,"value":"cosineSimilarity = function(v1,v2){\n \n const normV1 = norm(v1);\n const normV2 = norm(v2);\n \n if(normV1*normV2==0){\n return 0\n }\n else{\n return dot(v1,v2)/(normV1*normV2)\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4424,"value":"$39","pinned":false,"hidden":false,"mode":"ojs"},{"id":4433,"value":"$3a","pinned":false,"hidden":false,"mode":"ojs"},{"id":3162,"value":"convertLineData2Hsv = function(lineDataEvol){\n let lineDataEvolHsv = lineDataEvol.map(\n d => {\n let newColumnData = [];\n for(let i=0; i {\n let newColumnData = d3.filter(d.columnData, (d,i) => (i+1)%3==1); // extract s component from each line data\n return {'time':d.time, 'columnData':newColumnData}\n })\n return hueEvol\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":4098,"value":"convertLineDataEvolToDifferentFormats = function(lineDataEvol)\n{\n let lineDataEvolHsv = convertLineData2Hsv(lineDataEvol);\n let lineDataEvolVal = valueEvol(lineDataEvolHsv);\n let lineDataEvolSat = saturationEvol(lineDataEvolHsv);\n \n return {'rgb':lineDataEvol,'hsv':lineDataEvolHsv,'s':lineDataEvolSat,'v':lineDataEvolVal}\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":851,"value":"getSignalEvol = getSignalRGB","pinned":false,"hidden":false,"mode":"ojs"},{"id":11,"value":"md`### Data`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4514,"value":"signalExample = getSignalRGB(lineDataEvol).signal","pinned":false,"hidden":false,"mode":"ojs"},{"id":18,"value":"videoSrc = \"https://raw.githubusercontent.com/Gabsrol/TrackFicVideos/main/sample_periph_lyon.mp4\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":1982,"value":"videoSrc2 = \"https://raw.githubusercontent.com/bbety/TrackFicVideos/main/dense_traffic_short.mp4\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":47,"value":"videoFrames = getVideoFrames(videoSrc,timeRes)","pinned":false,"hidden":false,"mode":"ojs"},{"id":795,"value":"timeRes = 0.0667","pinned":false,"hidden":false,"mode":"ojs"},{"id":939,"value":"duration = (await vidvid.info(videoSrc)).duration","pinned":false,"hidden":false,"mode":"ojs"},{"id":945,"value":"times = d3.range(0,Math.round(duration),timeRes)","pinned":false,"hidden":false,"mode":"ojs"},{"id":5,"value":"md`### State`","pinned":false,"hidden":false,"mode":"ojs"},{"id":43,"value":"selectedLane = lanesCoordinates[7]","pinned":false,"hidden":false,"mode":"ojs"},{"id":220,"value":"drawLineOnImage(columnIdx,getFrameAt(0,videoFrames,timeRes),selectedLane)","pinned":false,"hidden":false,"mode":"ojs"},{"id":25,"value":"viewof columnIdx = slider({\n value: 278,\n min: 0,\n max: wProj,\n step: 1,\n description: 'Select a column of pixels'\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":46,"value":"lineDataEvol = realLineEvolProc(\n selectedLane, // the area on which you want to get a line of pixel\n videoFrames, // the frames of the video\n columnIdx, // choose the column you want on the rect\n hProj, // assuming you project your area on a rect. hProj is the height of this proj\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":27,"value":"wProj = 600","pinned":false,"hidden":false,"mode":"ojs"},{"id":52,"value":"hProj = 100","pinned":false,"hidden":false,"mode":"ojs"},{"id":5160,"value":"md`**Average of R,G,B components**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":77,"value":"averageRGBLine = lineDataEvol.map(d => {return {'value':d3.mean(d.columnData),'time':d.time}})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1277,"value":"averageRLine = lineDataEvol.map(d => {return {'value':d3.mean(d3.filter(d.columnData,(d,i) => (i+1)%3==2)),'time':d.time}})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1267,"value":"averageBLine = lineDataEvol.map(d => {return {'value':d3.mean(d3.filter(d.columnData,(d,i) => (i+1)%3==0)),'time':d.time}})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1270,"value":"averageGLine = lineDataEvol.map(d => {return {'value':d3.mean(d3.filter(d.columnData,(d,i) => (i+1)%3==1)),'time':d.time}})","pinned":false,"hidden":false,"mode":"ojs"},{"id":5473,"value":"md`**SSD and Cosine Sim**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":2582,"value":"ssdBtwLineAndStd = getSsdBtwLineAndStd(lineDataEvol,false,true)","pinned":false,"hidden":false,"mode":"ojs"},{"id":524,"value":"cosBtwLineAndStd = getCosBtwLineAndStd(lineDataEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":827,"value":"multiplyCosineAndSSD = getCosSsdDiffBtwLineAndStd(lineDataEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":5974,"value":"md`**From RGB to HSV**`","pinned":false,"hidden":false,"mode":"ojs"},{"id":3167,"value":"lineDataEvolHsv = convertLineData2Hsv(lineDataEvol)","pinned":false,"hidden":false,"mode":"ojs"},{"id":5882,"value":"hueEvol = getHueEvol(lineDataEvolHsv)","pinned":false,"hidden":false,"mode":"ojs"},{"id":865,"value":"md`### Manual Detection of cars\nWe need to know when a car actually passes by to be able to evaluate our methods.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1140,"value":"groundTruthTimes = [\n {'beginTime':0.39,'endTime':1.02},\n {'beginTime':1.4,'endTime':2.11},\n {'beginTime':2.9,'endTime':3.85},\n {'beginTime':4.2,'endTime':4.83},\n {'beginTime':7,'endTime':7.72},\n {'beginTime':7.8,'endTime':8.45},\n {'beginTime':10,'endTime':11}\n]","pinned":false,"hidden":false,"mode":"ojs"},{"id":15,"value":"md`### Imports`","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"d3 = require.alias({\n \"d3-array\": \"d3@5\",\n \"d3-axis\": \"d3@5\",\n \"d3-dispatch\": \"d3@5\",\n \"d3-drag\": \"d3@5\",\n \"d3-ease\": \"d3@5\",\n \"d3-scale\": \"d3@5\",\n \"d3-selection\": \"d3@5\",\n \"d3-transition\": \"d3@5\"\n})(\"d3\", \"d3@5\",\"d3-simple-slider@1\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"import {drawLineOnImage, realLineEvolProc, drawProjection} from '66467e3b94445c09'","pinned":false,"hidden":false,"mode":"ojs"},{"id":33,"value":"import { slider } from '@jashkenas/inputs'","pinned":false,"hidden":false,"mode":"ojs"},{"id":207,"value":"import { vl } from \"@vega/vega-lite-api\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":689,"value":"import {Range} from \"@observablehq/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":786,"value":"import {graph, getVideoFrames, getFrameAt, displayLineDataEvol,Scrubber, hsv2Rgb, rgb2Hsv} from 'd0a0a267bb0e0626'","pinned":false,"hidden":false,"mode":"ojs"},{"id":925,"value":"import {lanesCoordinates} from '2ef01a803dc6ad51'","pinned":false,"hidden":false,"mode":"ojs"},{"id":2111,"value":"import {Cell} from '@mbostock/dashboard'","pinned":false,"hidden":false,"mode":"ojs"},{"id":940,"value":"vidvid = import('https://unpkg.com/vidvid-info?module')","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W3b","stdlib":"1","titling":"auto"}