2a:null 2b:["$","title",null,{"children":"Model of Measurement Volume | bayeslife | Observable"}] 2e:{"id":"91f879754f82359b","slug":"model-of-measurement-volume","title":"Model of Measurement Volume","thumbnail":"48180b34ce77929183bdea92d0e3e57749308ab7b3b4d847d41952a4da4eb029","access_level":"public","create_time":"$D2018-10-20T00:17:16.739Z","update_time":"$D2018-10-20T00:17:16.739Z","view_time":null,"first_public_time":"$D2018-10-20T00:17:32.768Z","fork":{"id":"ebeed5e3ee72c2d5","slug":"model-of-measurement-volume","title":"Model of Measurement Volume","owner":{"id":"7329d081429dfcaa","login":"ptomlinsongti","name":""}},"stars":0,"starred":false,"owner":{"type":"team","id":"925f3d86a51431ab","login":"bayeslife","name":"","avatar_url":"https://avatars-next.observableusercontent.com/4caa0b9d6c95984b952b87533c82b8fa2b745c9188ba732293a841dc5fbbd668","tier":"starter_2024"},"version":407,"publish_version":407,"role":null,"authored":false,"authors":[{"type":"individual","id":"3ec1aa52fcc5b5c4","login":"bayeslife","name":"","avatar_url":"https://avatars-next.observableusercontent.com/4caa0b9d6c95984b952b87533c82b8fa2b745c9188ba732293a841dc5fbbd668","tier":null}],"editors":[{"type":"individual","id":"3ec1aa52fcc5b5c4","login":"bayeslife","name":"","avatar_url":"https://avatars-next.observableusercontent.com/4caa0b9d6c95984b952b87533c82b8fa2b745c9188ba732293a841dc5fbbd668","tier":null}],"comments":[],"trashed":false,"subscribed":null,"forks":0,"collection":null,"type":"notebook"} 31:[] 2f:{"title":"Model of Measurement Volume","cells":[{"id":0,"value":"md`# Model of Measurement Volume\n## Production Rate\nMineQ measurements will flow into the platform based on a number of factors.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":255,"value":"md`Different Customers`","pinned":false,"hidden":false,"mode":"ojs"},{"id":256,"value":"viewof customer = html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":58,"value":"md`have different number of assets`","pinned":false,"hidden":false,"mode":"ojs"},{"id":33,"value":"viewof assets = {\n const inp = html``\n switch(customer) {\n case \"CPM\":\n inp.value=50\n break;\n case \"RoyHill\":\n inp.value=10\n break;\n case \"Emeco\":\n inp.value=500\n break;\n case \"Coates\":\n inp.value=200\n break;\n case \"Simplest\":\n inp.value=1\n break;\n case \"Test\":\n inp.value=2\n break;\n }\n return inp\n }","pinned":false,"hidden":false,"mode":"ojs"},{"id":40,"value":"md`Another factor is the number of device types (as an average for simplicity)`","pinned":false,"hidden":false,"mode":"ojs"},{"id":64,"value":"viewof devicetypes = {\n const inp = html``\n switch(customer) {\n case \"CPM\":\n inp.value=5\n break;\n case \"RoyHill\":\n inp.value=5\n break;\n case \"Emeco\":\n inp.value=1\n break;\n case \"Coates\":\n inp.value=1\n break;\n case \"Simplest\":\n inp.value=1\n break;\n case \"Test\":\n inp.value=2\n break;\n }\n return inp\n }","pinned":false,"hidden":false,"mode":"ojs"},{"id":82,"value":"md`Another factor is the number of measurements per device type ( as an average for simplicity)`","pinned":false,"hidden":false,"mode":"ojs"},{"id":270,"value":"viewof hardware = {\n const inp = html``\n switch(customer) {\n case \"CPM\":\n inp.value='MineQ'\n break;\n case \"RoyHill\":\n inp.value='MineQ'\n break;\n case \"Emeco\":\n inp.value='TrackQ'\n break;\n case \"Coates\":\n inp.value='TrackQ'\n break;\n case \"Simplest\":\n inp.value='TrackQ'\n break;\n case \"Test\":\n inp.value='MineQ'\n break;\n }\n return inp\n }","pinned":false,"hidden":false,"mode":"ojs"},{"id":88,"value":"viewof measurements = {\n const inp = html``\n switch(hardware) {\n case \"MineQ\":\n inp.value=10\n break;\n case \"TrackQ\":\n inp.value=1\n break;\n }\n return inp\n }","pinned":false,"hidden":false,"mode":"ojs"},{"id":106,"value":"md`Another factor to consider is the number of seconds between reported measurements. This is known as the sample rate`","pinned":false,"hidden":false,"mode":"ojs"},{"id":110,"value":"viewof samplerate = html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":68,"value":"html`The above configuration will produce the following number of data points into the MineQ platform\n

${dataPerPeriod(time_constants().second,assets,devicetypes,measurements,samplerate,scale_constants().one)} per second
${dataPerPeriod(time_constants().hour,assets,devicetypes,measurements,samplerate,scale_constants().million)} million per hour
${dataPerPeriod(time_constants().day,assets,devicetypes,measurements,samplerate,scale_constants().million)} millon\nper day
${dataPerPeriod(time_constants().month,assets,devicetypes,measurements,samplerate,scale_constants().billion)} billion per month

\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":126,"value":"md`## Data Transmission\nEach data point is represented in transit as`","pinned":false,"hidden":false,"mode":"ojs"},{"id":206,"value":"{\n const context = DOM.context2d(800, 22);\n context.strokeRect(0,0,400,20);\n context.strokeText(\"Timestamp(8 bytes)\",0,10);\n context.strokeRect(400,0,200,20);\n context.strokeText(\"Value(4 bytes)\",400,10);\n context.strokeRect(600,0,100,20);\n context.strokeText(\"Measurement Id(2 bytes)\",600,10);\n return context.canvas;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":133,"value":"md`Each device will package up its measurements into a batch that additionally includes`","pinned":false,"hidden":false,"mode":"ojs"},{"id":225,"value":"{\n const context = DOM.context2d(800, 22);\n context.strokeRect(0,0,200,20);\n context.strokeText(\"Asset Id(2 bytes)\",0,10);\n context.strokeRect(200,0,200,20);\n context.strokeText(\"Device Id(2 bytes)\",200,10);\n context.strokeRect(400,0,100,20);\n context.strokeText(\"Count(1 byte)\",400,10);\n context.strokeRect(500,0,100,20);\n context.strokeText(\"Type of Message(1 byte)\",500,10);\n return context.canvas;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":137,"value":"html`The result of this encoding is the need to transfer\n

${byteTransferPerPeriod(time_constants().second,assets,devicetypes,measurements,samplerate,transmission_constants().byte)} bytes per second\n

${byteTransferPerPeriod(time_constants().hour,assets,devicetypes,measurements,samplerate,transmission_constants().bytesPerMegaByte)} megabyte per hour\n

${byteTransferPerPeriod(time_constants().day,assets,devicetypes,measurements,samplerate,transmission_constants().bytesPerMegaByte)} megabyte per day\n

${byteTransferPerPeriod(time_constants().month,assets,devicetypes,measurements,samplerate,transmission_constants().bytesPerGigaByte)} gigabyte per month\n

\nNote this does not consider the DDS or OpenVPN overheads that will be additional\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":275,"value":"md`## Real Time Storage\nA measurement is represented in the MineQ Operational Storate (mongo) as a json document with a number of denormalized properties`","pinned":false,"hidden":false,"mode":"ojs"},{"id":291,"value":"html`The storage required for measurements alone\n

${storagePerPeriod(time_constants().second,assets,devicetypes,measurements,samplerate,byte_constants().byte,storage_constants().measurementDocumentBytes)} bytes per second\n

${storagePerPeriod(time_constants().hour,assets,devicetypes,measurements,samplerate,byte_constants().bytesPerMegaByte,storage_constants().measurementDocumentBytes)} megabyte per hour\n

${storagePerPeriod(time_constants().day,assets,devicetypes,measurements,samplerate,byte_constants().bytesPerMegaByte,storage_constants().measurementDocumentBytes)} megabyte per day\n

${storagePerPeriod(time_constants().month,assets,devicetypes,measurements,samplerate,byte_constants().bytesPerGigaByte,storage_constants().measurementDocumentBytes)} gigabyte per month\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":397,"value":"md`## Real Time Document Storage\nIn the real time storage each measurement is represented as a separate document.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":399,"value":"html`The real time documents created\n

${storagePerPeriod(time_constants().second,assets,devicetypes,measurements,samplerate,1,1)} documents per second\n

${storagePerPeriod(time_constants().hour,assets,devicetypes,measurements,samplerate,1000,1)} thousand documents per hour\n

${storagePerPeriod(time_constants().day,assets,devicetypes,measurements,samplerate,1000000,1)} million documents per day\n

${storagePerPeriod(time_constants().month,assets,devicetypes,measurements,samplerate,1000000000,1)} billion documents per month\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":362,"value":"md`## Reporting Document Storage\nAs measurement are collapsed into reporting documents we can expect the following volume of reporting DAY documents`","pinned":false,"hidden":false,"mode":"ojs"},{"id":365,"value":"html`\n

${dataPerPeriod(1/*documents per day*/,assets,devicetypes,measurements,samplerate,1)} DAY documents per day\n

${dataPerPeriod(1/*documents per day*/*30,assets,devicetypes,measurements,samplerate,1)} DAY documents per month\n

${dataPerPeriod(1/*one day*/*365,assets,devicetypes,measurements,samplerate,1)} DAY documents per year\n\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":118,"value":"md`#### Utility functions and constants`","pinned":false,"hidden":false,"mode":"ojs"},{"id":71,"value":"function dataPerPeriod(period,assets,devicetypes,measurements,samplerate,unit){\n return (period*assets*devicetypes*measurements/samplerate)/unit;\n}\n ","pinned":false,"hidden":false,"mode":"ojs"},{"id":124,"value":"function byteTransferPerPeriod(period,assets,devicetypes,measurements,samplerate,unit){\n let c = transmission_constants();\n let bytes = period*assets*devicetypes*c.transferBytesPerDeviceType+\n period*assets*devicetypes*c.transferBytesPerMeasurement*measurements/samplerate;\n \n return Math.round(bytes/unit)\n \n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":294,"value":"function storagePerPeriod(period,assets,devicetypes,measurements,samplerate,unit,documentsize){\n let data = dataPerPeriod(period,assets,devicetypes,measurements,samplerate,scale_constants().one);\n return Math.round(data*documentsize/unit)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":369,"value":"function reportingStoragePerPeriod(period,assets,devicetypes,measurements,samplerate,unit,documentsize){\n let compression_ratio = 86400\n let data = dataPerPeriod(period,assets,devicetypes,measurements,samplerate,scale_constants().one);\n return Math.round(data/unit)/86400\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":297,"value":"function time_constants(){\n return {\n second: 1,\n hour: 3600,\n day: 3600*24,\n month: 3600*24*30,\n seconds_per_day: 3600*60*60\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":155,"value":"function transmission_constants(){\n return {\n transferBytesPerMeasurement: 8+4+2,\n transferBytesPerDeviceType: 2+2+1+1,\n byte: 1,\n bytesPerMegaByte: 1024*1024,\n bytesPerGigaByte: 1024*1024*1024\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":286,"value":"function storage_constants(){\n return {\n //Taken from Dev DB\n geomeasurementDocumentBytes: 200,\n measurementDocumentBytes: 370 \n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":311,"value":"function byte_constants(){\n return {\n byte: 1,\n bytesPerMegaByte: 1024*1024,\n bytesPerGigaByte: 1024*1024*1024\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":319,"value":"function scale_constants(){\n return {\n one: 1,\n thousand: 1000,\n million: 1000000,\n billion: 1000000000\n }\n}","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}