31:T4e9,function describeData(data, options) { let toStr = options || false; let header = Object.keys(data[0]); let acceptable = []; header.forEach(head => { if (typeof data[0][head] == "number") { acceptable.push(head); } }); let names = ["count", "mean", "std", "min", "25%", "50%", "75%", "max"]; let stats = {}; names.forEach(elt => stats[elt]={}); acceptable.forEach(head => { let count = d3.count(data, d => d[head]); let mean = d3.mean(data, d => d[head]); let std = d3.deviation(data, d => d[head]); let [min, max] = d3.extent(data, d => d[head]); let x25 = d3.quantile(data, 0.25, d => d[head]); let x50 = d3.quantile(data, 0.5, d => d[head]); let x75 = d3.quantile(data, 0.75, d => d[head]); stats["count"][head]=count; stats["mean"][head]=mean; stats["std"][head]=std; stats["min"][head]=min; stats["25%"][head]=x25; stats["50%"][head]=x50; stats["75%"][head]=x75; stats["max"][head]=max; }); if (typeof toStr == "boolean" && toStr) { let df = []; names.forEach(name => { let obj = {}; obj[""] = name; df.push({...obj, ...stats[name]}); }); return z.print(df); } else { return stats; } }32:T1081,class plot4 { constructor(data, selector) { this.data = data; this.elt = selector; this.colors = { r: "#D62728", b: "#3366CC", g: "#109618", y: "#F2B601", o: "#F58518", pl: "#990099", lb: "#0099C6", pk: "#FF9DA6", lv: "#636EFA", lo: "#EF553B", default: "#1F77B4" }; } type(kind) { this.type = kind; } title(t) { this.Title = t; } xlabel(xl) { this.xLabel = xl; } ylabel(yl) { this. yLabel = yl; } ticks(bool) { this.showTicks = bool; } bins(Bins) { this.nBins = Bins; } show() { if (this.type == "hist") { let hist; if (!this.nBins) { hist = d3.histogram(); } else { let scale = d3.scaleLinear() .domain(d3.extent(this.data)); hist = d3.histogram() .thresholds(scale.ticks(this.nBins)); } let bins = hist(this.data); let x = Array.from(bins, elt => elt = `${elt["x0"]}-${elt["x1"]}`); let y = Array.from(bins, elt => elt = elt.length); let trace0 = { type: "bar", x: x, y: y, width: Array(bins.length).fill(1) }; let data = [trace0]; let layout = { title: {text: this.Title || ""}, xaxis: {title: {text: this.xLabel || ""}, showticklabels: this.showTicks && true}, yaxis: {title: {text: this.yLabel || ""}} }; Plotly.newPlot(this.elt, data, layout); } else if (this.type == "hist-m") { let hist; if (!this.nBins) { hist = d3.histogram(); } else { let scale = d3.scaleLinear() .domain(d3.extent(this.data)); hist = d3.histogram() .thresholds(scale.ticks(this.nBins)); } let data = []; this.data.forEach(part => { let bins = hist(part[0]); let x = Array.from(bins, elt => elt = `${elt["x0"]}-${elt["x1"]}`); let y = Array.from(bins, elt => elt = elt.length); let trace = { type: "bar", x: x, y: y, width: Array(bins.length).fill(1) }; if(part[1]) { trace.name = part[1]; } if(part[2]) { let color = this.colors[part[2]]; trace.marker = {color}; } data.push(trace); }); let layout = { title: {text: this.Title || ""}, xaxis: {title: {text: this.xLabel || ""}, showticklabels: this.showTicks && true}, yaxis: {title: {text: this.yLabel || ""}} }; Plotly.newPlot(this.elt, data, layout); } else if (this.type == "scatter") { let x = this.data[0]; let y = this.data[1]; let trace0 = { type: "scatter", x: x, y: y, mode: "markers" }; let data = [trace0]; let layout = { title: {text: this.Title || ""}, xaxis: {title: {text: this.xLabel || ""}, showticklabels: this.showTicks && true}, yaxis: {title: {text: this.yLabel || ""}} }; Plotly.newPlot(this.elt, data, layout); } else if (this.type == "scatter-m") { let data = []; this.data.forEach(part => { let x = part[0]; let y = part[1]; let trace = { type: "scatter", x: x, y: y, mode: "markers" }; if(part[2]) { trace.name = part[2]; } if(part[3]) { let color = this.colors[part[3]]; trace.marker = {color}; } data.push(trace); }); let layout = { title: {text: this.Title || ""}, xaxis: {title: {text: this.xLabel || ""}, showticklabels: this.showTicks && true}, yaxis: {title: {text: this.yLabel || ""}} }; Plotly.newPlot(this.elt, data, layout); } } }33:[] 2f:{"title":"Medical Cost Prediction: JS Edition","cells":[{"id":0,"value":"md`# Medical Cost Prediction: JS Edition`","pinned":false,"hidden":false,"mode":"ojs"},{"id":4,"value":"md`## Medical Cost\nIn this workshop, we will explore a dataset of insurance data with the below info. We will try to predict the cost of treatment for individuals based on their age, sex, bmi and other info.\n\\nColumns\n- **age**: Age of primary beneficiary.\n- **sex**: Insurance contractor gender, female, male.\n- **bmi**: Body mass index, providing an understanding of body, weights that are relatively high or low relative to height, objective index of body weight (kg / m ^ 2) using the ratio of height to weight, ideally 18.5 to 24.9.\n- **children**: Number of children covered by health insurance / Number of dependents.\n- **smoker**: Smoking\n- **region**: The beneficiary's residential area in the US, northeast, southeast, southwest, northwest.\n- **charges**: Individual medical costs billed by health insurance.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":23,"value":"md`## 1. Import Necessary Javascript Libraries\nWe will need the following libraries:\n- D3 — for manipulating documents based on data.\n- Zebras — data manipulation and analysis library.\n- lodash — a modern JavaScript utility library delivering modularity, performance & extras.\n- Plotly — is a high-level declarative charting library.\n- mljs — a compilation of machine learning tools.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":30,"value":"d3 = require(\"d3\", \"d3-array@^2.1\");","pinned":true,"hidden":false,"mode":"ojs"},{"id":32,"value":"z = require(\"zebras\");","pinned":true,"hidden":false,"mode":"ojs"},{"id":305,"value":"_ = require(\"lodash\");","pinned":true,"hidden":false,"mode":"ojs"},{"id":34,"value":"Plotly = require(\"https://cdn.plot.ly/plotly-latest.min.js\");","pinned":true,"hidden":false,"mode":"ojs"},{"id":36,"value":"ml = require(\"https://www.lactame.com/lib/ml/4.0.0/ml.min.js\");","pinned":true,"hidden":false,"mode":"ojs"},{"id":40,"value":"md`## 2. Finding The Data\nDataset can be found on Kaggle. For simplicity, it has been uploaded to Github. Link is provided for you in the cell below.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":43,"value":"url = \"https://raw.githubusercontent.com/BeirutAI/medical-cost-prediction/master/data/insurance.csv\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":48,"value":"md`## 3. Read & Visualize Data\nData is stored on github in a csv (Comma Separated Values) file. To load the data to our code, we use **d3**, more specifically, the **csv** function.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":51,"value":"// read CSV file in d3.\ndata = d3.csv(url).then((data) => {\n // parsing numbers.\n return z.parseNums([\"age\", \"bmi\", \"children\", \"charges\"], data);\n});","pinned":true,"hidden":false,"mode":"ojs"},{"id":77,"value":"// display first 5 rows as an ASCII table.\nz.printHead(5, data);","pinned":true,"hidden":false,"mode":"ojs"},{"id":85,"value":"md`## 4. Exploratory Data Analysis\nLet's dig deeper & understand our data\n\\n**Task**: how many rows & columns in our dataset\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":89,"value":"{\n // get the number of rows and columns\n let rows = z.shape(data)[0];\n let columns = z.shape(data)[1];\n \n return `There are ${rows} rows and ${columns} columns.`;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":102,"value":"md`Using the function **describeData()**, we can check the mean, standard deviation, maximum, and minimum of each numerical feature (column)`","pinned":false,"hidden":false,"mode":"ojs"},{"id":104,"value":"describeData(data, true);","pinned":true,"hidden":false,"mode":"ojs"},{"id":122,"value":"md`### Data visualization\n#### Distribution of charges\n\\nFirst, let's look at the distribution of charges. This will help us to know how much patients spend on treatment on average.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":171,"value":"charges = z.getCol(\"charges\", data);","pinned":true,"hidden":false,"mode":"ojs"},{"id":128,"value":"// plot the histogram of the charges\n{\n // create a div to for the chart.\n let div = html`

`;\n yield div;\n \n // plot the charges.\n let chart = new plot4(charges, \"charges_chart\");\n chart.type(\"hist\");\n chart.title(\"Distribution of charges\");\n chart.xlabel(\"Charges\");\n chart.ylabel(\"Frequency\");\n chart.show();\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":131,"value":"md`#### Correlation between smoking and cost of treatment\n\\nLet's see if smokers spend more or less on treatment than non-smokers!\n\\nFirst, let's see how many smokers vs non-smokers we have.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":137,"value":"// select smokers \nsmokers = data.filter(data => data.smoker == \"yes\");","pinned":true,"hidden":false,"mode":"ojs"},{"id":139,"value":"// select non smokers\nnon_smokers = data.filter(data => data.smoker == \"no\");","pinned":true,"hidden":false,"mode":"ojs"},{"id":142,"value":"`There are ${smokers.length} smokers and ${non_smokers.length} non-smokers.`","pinned":true,"hidden":false,"mode":"ojs"},{"id":146,"value":"md`Now let's plot the charges for both.`","pinned":true,"hidden":false,"mode":"ojs"},{"id":148,"value":"{\n // get charges for both.\n let smokersCharges = z.getCol(\"charges\", smokers);\n let nonSmokersCharges = z.getCol(\"charges\", non_smokers);\n \n // create a div for containing both div charts.\n let div =\n html`

`;\n yield div;\n \n // plot first one.\n let chart1 = new plot4(smokersCharges, \"smoker-chart\");\n chart1.type(\"hist\");\n chart1.title(\"Distribution of charges for smokers\");\n chart1.show();\n \n // plot second one.\n let chart2 = new plot4(nonSmokersCharges, \"non-smoker-chart\");\n chart2.type(\"hist\");\n chart2.title(\"Distribution of charges for non-smokers\");\n chart2.show();\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":154,"value":"md`Smoking patients spend more on treatment.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":156,"value":"md`#### Correlation between age and cost of treatment\nFirst, let's look at the distribution of age in our dataset, and then look at how age affects the cost of treatment.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":160,"value":"// get age data\nage = z.getCol(\"age\", data);","pinned":true,"hidden":false,"mode":"ojs"},{"id":158,"value":"{\n // plot histogram for age distribution\n let div = html`

`;\n yield div;\n \n let chart = new plot4(age, \"age-chart\");\n chart.type(\"hist\");\n chart.title(\"Age distribution\");\n chart.xlabel(\"Age\");\n chart.ylabel(\"Frequency\");\n chart.bins(20);\n chart.show();\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":167,"value":"md`Let's plot the correlation between age and cost of treatment`","pinned":false,"hidden":false,"mode":"ojs"},{"id":169,"value":"{\n // draw a scatter plot to show correlation between age and charges\n let div = html`

`;\n yield div;\n \n let chart = new plot4([age, charges], \"scatter-age-chart\");\n chart.type(\"scatter\");\n chart.title(\"Cost of treatment for different ages\");\n chart.xlabel(\"Age\");\n chart.ylabel(\"Charges\");\n chart.show();\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":175,"value":"md`Let's check if smoking also affects this curve.\n\\n**Task**: show a scatter plot with the correlation between age and cost of treatment while showing smokers in red and non-smokers in blue\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":181,"value":"{\n let smokersCharges = z.getCol(\"charges\", smokers);\n let nonSmokersCharges = z.getCol(\"charges\", non_smokers);\n \n let smokersAge = z.getCol(\"age\", smokers);\n let nonSmokersAge = z.getCol(\"age\", non_smokers);\n \n let div = html`

`;\n yield div;\n \n let chart = new plot4([\n [smokersAge, smokersCharges, \"smokers\", \"r\"],\n [nonSmokersAge, nonSmokersCharges, \"non_smokers\", \"b\"]\n ], \"scatter-age-smoke-chart\");\n chart.type(\"scatter-m\");\n chart.title(\"Cost of treatment for different ages\");\n chart.xlabel(\"Age\");\n chart.ylabel(\"Charges\");\n chart.show();\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":188,"value":"md`#### Correlation between BMI and cost of treatment\n# Body Mass Index\n![alt text](https://4.bp.blogspot.com/-nBF9Z1tFGhI/W3MqbdD0j7I/AAAAAAAAAIs/UdyXTIxsBT8Pl8usABxEK_Fusj5S0SnBQCLcBGAs/s1600/HOW%2BTO%2BCALCULATE%2BBODY%2BMASS%2BINDEX%2BBMI.jpg)\n# BMI Chart\n![BMI char](https://images.squarespace-cdn.com/content/v1/56fae4be1d07c0c393d8faa5/1551103826935-HCXS8U78500C06GQ1PLJ/ke17ZwdGBToddI8pDm48kNMeyc_nGAbaGjp3EBJ2o08UqsxRUqqbr1mOJYKfIPR7LoDQ9mXPOjoJoqy81S2I8N_N4V1vUb5AoIIIbLZhVYxCRW4BPu10St3TBAUQYVKckzCNDuUMr1wTvf7-fqd2hrX5O2-_PoO3UJ2jNU1VzJbe6G9-F0r9BTATNUu-NBMy/BMI+Chart.jpg)\nFirst, let's look at the distribution of BMI in our dataset, and then look at how it affects the cost of treatment.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":193,"value":"// get bmi.\nbmi = z.getCol(\"bmi\", data);","pinned":true,"hidden":false,"mode":"ojs"},{"id":195,"value":"{\n // draw a histogram to show the distribution of BMI\n let div = html`

`;\n yield div;\n \n let chart = new plot4(bmi, \"bmi_chart\");\n chart.type(\"hist\");\n chart.title(\"BMI distribution\");\n chart.xlabel(\"BMI\");\n chart.ylabel(\"Frequency\");\n chart.show();\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":201,"value":"// select obese people\nobese = data.filter(d => d.bmi >= 30);","pinned":true,"hidden":false,"mode":"ojs"},{"id":202,"value":"// select overweight people\noverweight = data.filter(d => (d.bmi >= 25 && d.bmi < 30));","pinned":true,"hidden":false,"mode":"ojs"},{"id":205,"value":"// select healthy people \nhealthy = data.filter(d => d.bmi < 25);","pinned":true,"hidden":false,"mode":"ojs"},{"id":207,"value":"`There are ${obese.length} obese, ${overweight.length} overweight and ${healthy.length} healthy individuals.`","pinned":true,"hidden":false,"mode":"ojs"},{"id":209,"value":"md`To compare, let's plot the distribution of charges for all 3 groups (obese, overweight and healthy) in 1 plot, while showing \\`obese\\` data in red, \\`overweight\\` data in yellow and \\`healthy\\` data in green.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":212,"value":"{\n let div = html`

`;\n yield div;\n \n let obeseCharges = z.getCol(\"charges\", obese);\n let overweightCharges = z.getCol(\"charges\", overweight);\n let healthyCharges = z.getCol(\"charges\", healthy);\n \n let chart = new plot4([\n [obeseCharges, \"obese\", \"r\"],\n [overweightCharges, \"overweight\", \"y\"],\n [healthyCharges, \"healthy\", \"g\"]\n ], \"chart8\");\n chart.type(\"hist-m\");\n chart.title(\"Charges distribution\");\n chart.xlabel(\"Charges\");\n chart.ylabel(\"Frequency\");\n chart.ticks(false);\n chart.show();\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":214,"value":"md`Patients with BMI above 30 spend more on treatment!`","pinned":true,"hidden":false,"mode":"ojs"},{"id":217,"value":"md`## 5. Preprocessing\n\"Garbage in, garbage out\".\n\\nData should be preprocessed and cleaned to get rid of noisy data.\nPreprocessing includes:\n- dealing with missing data\n - remove whole rows (if they are not a lot)\n - infer (e.g., date of birth & age)\n - fill with mean, median, or even 0\n- removing unsued column(s)\n- convert categorical (non numerical) data into numerical\n- normalization: standarize data ranges for all features (e.g., between 0 and 1)\n---\nLet's start by removing missing data.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":236,"value":"// print how many missing value in each column.\ndescribeData(data)[\"count\"];","pinned":true,"hidden":false,"mode":"ojs"},{"id":219,"value":"// find rows with missing values.\nmissing = {\n let indexs = [];\n data.forEach((row, i) => {\n if (isNaN(row[\"bmi\"])) {\n indexs.push(i);\n };\n });\n \n let rows = [];\n indexs.forEach(i => {\n let row = data[i];\n row.index = i;\n rows.push(row);\n });\n return rows;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":239,"value":"// removing rows with missing values.\nmissing.forEach((row, i) => data.splice(row.index-i,1));","pinned":true,"hidden":false,"mode":"ojs"},{"id":241,"value":"describeData(data, true);","pinned":true,"hidden":false,"mode":"ojs"},{"id":251,"value":"md`#### Remove unused columns\nLet's remove the \\`region\\` column since we don't really care about it\n`","pinned":true,"hidden":false,"mode":"ojs"},{"id":255,"value":"// dropping the region feature\ncopyOfdata = z.dropCol(\"region\", data);","pinned":true,"hidden":false,"mode":"ojs"},{"id":258,"value":"z.printHead(2, copyOfdata);","pinned":true,"hidden":false,"mode":"ojs"},{"id":270,"value":"md`#### Convert Categorical columns to numerical\n* We need to convert the sex column from male/female to 0/1.\n* We need to convert the smoker column from no/yes to 0/1.\n\\nLet's start with the sex column\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":273,"value":"{\n // define dictionary\n let gender = {\"male\": 0, \"female\": 1};\n \n // replace sex column with 0/1\n copyOfdata.forEach(row => row.sex = gender[row.sex]);\n \n // print head to verify\n return z.printHead(5, copyOfdata)\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":277,"value":"{\n // define dictionary\n let smokers = {\"no\": 0, \"yes\": 1};\n \n // replace smokers column with 0/1\n copyOfdata.forEach(row => row.smoker = smokers[row.smoker]);\n \n // print head to verify\n return z.printHead(5, copyOfdata)\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":282,"value":"md`#### Normalization\nLet's scale all the columns by dividing by the maximum\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":284,"value":"// get the max of each column\ndata_max = describeData(copyOfdata)[\"max\"];","pinned":true,"hidden":false,"mode":"ojs"},{"id":287,"value":"{\n // divide each column by its maximum value\n copyOfdata.forEach(row => {\n let columns = Object.keys(copyOfdata[0]);\n columns.forEach(column => row[column] = row[column] / data_max[column]);\n });\n \n return describeData(copyOfdata, true);\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":301,"value":"md`### Training\n#### Data split\nBefore training, we need to split data into training (80%) & testing (20%)\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":316,"value":"md`#### Make model\n##### LinearRegression\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":303,"value":"model_data = {\n // store smoker column as input x.\n let x = z.getCol(\"smoker\",copyOfdata);\n // store the charges column as the output (label) in y \n let y = z.getCol(\"charges\",copyOfdata);\n \n // split dataset in a 80/20 split\n let test_size = 0.2;\n let [x_train, x_test] = _.chunk(x, data.length*(1-test_size));\n let [y_train, y_test] = _.chunk(y, data.length*(1-test_size));\n \n let train_data = [x_train, y_train];\n let test_data = [x_test, y_test];\n \n return {train_data, test_data};\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":310,"value":"// create our regression model\nmodel = new ml.SimpleLinearRegression(model_data[\"train_data\"][0], model_data[\"train_data\"][1]);","pinned":true,"hidden":false,"mode":"ojs"},{"id":312,"value":"// test our model\nmodel.score(model_data[\"test_data\"][0], model_data[\"test_data\"][1]);","pinned":true,"hidden":false,"mode":"ojs"},{"id":100,"value":"md`

\n\n--- \n\n## Appendix\n\nThe cells below power the cells above.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":96,"value":"$31","pinned":true,"hidden":false,"mode":"ojs"},{"id":97,"value":"$32","pinned":true,"hidden":false,"mode":"ojs"},{"id":90,"value":"z.shape = (data) => {\n let rows = data.length;\n let columns = Object.keys(data[0]).length;\n \n return [rows, columns];\n}","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"}