31:T11f5,chart2 = { const h = 250; const w = 500; const svg = d3.select(DOM.svg(width, h)); const data1 = lossData( 1, 1, 0.001); const data0 = lossData( 0, 1, 0.001); const adomain = [0, 1]; const ydomain1 = d3.extent(data1, d => d.y); const ydomain0 = d3.extent(data1, d => d.y); const z = d3.scaleLinear().domain([-7, 7]).range([50, 50 + w]); const a = d3.scaleLinear().domain(adomain).range([50, 50 + w]); const y1 = d3.scaleLinear().domain(ydomain1).range([h -70,20]); const y0 = d3.scaleLinear().domain(ydomain0).range([h -70,20]); const g = svg.append("g").classed("series", true); const line = y => d3.line() .curve(d3.curveMonotoneX) .x(d => a(d.x)) .y(d => y(d.y)); // // LOSS CURVES (for y=0 and y=1) // g.append("path") .datum(data1) .attr("fill", "transparent") .attr("stroke", "#A93226") .attr("stroke-width", 1.5) .attr("d", line(y1)); g.append("text") .attr("x", a(0)) .attr("y", 12) .attr("fill", "#A93226") .text("Loss(a | y=1)"); g.append("path") .datum(data0) .attr("fill", "transparent") .attr("stroke", "#2980B9") .attr("stroke-width", 1.5) .attr("d", line(y0)); g.append("text") .attr("x", a(1)) .attr("y", 12) .attr("fill", "#2980B9") .attr("text-anchor", "end") .text("Loss(a | y=0)"); // // MOUSEOVER EXAMPLE // let z0 = -1.5; let a0 = sigmoid(z0); const gexample = g.append("g") .classed("example", true) .attr("transform", `translate(${a(a0)}, 0)`); gexample.append("line") .attr("x1", 0) .attr("y1", 20) .attr("x2", 0) .attr("y2", y1(0) ) .style("stroke", "black") .style("stroke-dasharray", "4,2"); gexample.append("line") .attr("id", "linez") .attr("x1", 0) .attr("y1", y1(0) ) .attr("x2", z(z0) - a(a0)) .attr("y2", y1(0) + 40) .style("stroke", "black") .style("stroke-dasharray", "4,2"); gexample.append("circle") .attr("id", "y1") .attr("fill", "#A93226") .attr("cx", 0) .attr("cy", y1(loss(1, a0))) .attr("r", 4); gexample.append("circle") .attr("id", "y0") .attr("fill", "#2980B9") .attr("cx", 0) .attr("cy", y0(loss(0, a0))) .attr("r", 4); // // AXIS // const aAxis = g => g.attr("transform", `translate(0,${y1(0) + 2} )`) .call(d3.axisBottom(a).tickValues(d3.ticks(0, 1, 10))); svg.append("g") .call(aAxis); const zAxis = g => g .attr("transform", `translate(0,${y1(0) + 40} )`) .call(d3.axisBottom(z).tickValues(d3.ticks(-7, 7, 14))); svg.append("g").call(zAxis); g.append("text") .attr("x", a(0.5)) .attr("y", y1(0) + 30) .attr("text-anchor", "middle") .text("a"); g.append("text") .attr("x", a(0.5)) .attr("y", y1(0) + 70) .attr("text-anchor", "middle") .text("z"); // // SIGMOID // const data = sigmoidData( 10, 0.5); const xdomain = d3.extent(data, d => d.x); const ydomain = [0, 1]; var x = d3.scaleLinear().domain(xdomain).range([150 + w, 150 + w + 200]); var y = d3.scaleLinear().domain(ydomain).range([h -80,60]); var gs = svg.append("g").classed("sigmoid", true); var lines = d3.line() .curve(d3.curveMonotoneX) .x(d => x(d.x)) .y(d => y(d.y)); gs.append("path") .datum(sigmoidData()) .attr("fill", "transparent") .attr("stroke", "grey") .attr("stroke-width", 1.5) .attr("d", lines); const values = [[xdomain[0], "0", "0"], [0, "0.5", "-1.2"], [xdomain[1], "+1", "-1"]]; gs.selectAll("text.value") .data(values) .enter() .append("text") .classed("value", true) .attr("x", d => x(d[0])) .attr("y", d => y(sigmoid(d[0]))) .attr("dy", "-0.2em") .attr("dx", d => d[2] + "em") .text(d => d[1]); gs.append("text").attr("x", x(0)).attr("y", h-40).attr("text-anchor", "middle") .text("a(z) = σ(z): sigmoid function"); gs.append("circle") .attr("fill", "grey") .attr("cx", x(z0)) .attr("cy", y(sigmoid(z0))) .attr("r", 4); // // HANDLE MOUSEOVER // svg.on("touchmove mousemove", function() { let xpos = d3.mouse(this)[0]; if (xpos <= 50 || xpos >= (w + 100)) return; let zpos = z.invert(xpos); let apos = sigmoid(zpos); gexample.attr("transform", `translate(${a(apos)}, 0)`); gexample.select("#y1").attr("cy", y1(loss(1, apos))); gexample.select("#y0").attr("cy", y0(loss(0, apos))); gexample.select("#linez").attr("x2", z(zpos) - a(apos)); gs.select("circle").attr("cy", y(apos)).attr("cx", x(zpos)); }); return svg.node(); }32:[] 2f:{"title":"Logistic Regression","cells":[{"id":4,"value":"md `# Logistic Regression\nSummary sheet of mathematical formulas used during a course in \"Machine Learning\"\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":491,"value":"md `## Notations\n| | |\n|---|---|\n|n| number of features|\n|m| number of training examples|\n|i| refers to the i-th training example|\n|${tex` \\large \\{x^{(i)},y^{(i)}\\}`}| i-th traning example, x: input features, y output value (labeled value)|\n|${tex`\\large \\hat y`} or a| prediction, for the same x, we want ${tex `\\large \\hat y \\approx y`}, also noted as a (activation)|\n|{w, b}| training model (learning parameters)|\n|${tex `\\large \\alpha`}| learning rate|\n|${tex `\\large \\lambda`}| regularization parameter|\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":501,"value":"md `#### For one training example: ${tex` \\large \\{x^{(i)},y^{(i)}\\}`}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":0,"value":"md `\n${tex `\n\\large\n\\textbf{features}: x^{(i)} \\in \\R^{n \\times 1} = \\begin{bmatrix}\nx_1^{(i)}\\\\\nx_2^{(i)}\\\\\n\\vdots\\\\\nx_n^{(i)}\n\\end{bmatrix} , \n\\;\n\\textbf{label}:\\; y^{(i)} \\in \\R,\n\\;\n\\textbf{parameters}:\nw \\in \\R^{n \\times 1} = \\begin{bmatrix}\nw_1 \\\\\nw_2 \\\\\n\\vdots\\\\\nw_n\n\\end{bmatrix}\n,\\; b \\in \\R\n`\n}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":41,"value":"md `## Prediction ${tex `\\hat y^{(i)}`}\ny {0,1}, for binary classification\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":508,"value":"md `\n

\n${tex `\n\\LARGE\n z^{(i)} = w^T \\cdot x^{(i)}\\ + b \\;\n\\small\n\\;\\;\\bigg( \\displaystyle\\sum_{j=1}^{n} w_j \\cdot x_j^{(i)} + b\\bigg)\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":177,"value":"md `\n

\n${tex `\n\\LARGE\n\n a^{(i)} = \\hat y^{(i)} = \\sigma (z^{(i)}) \\;\n= \\frac{1}{1 + e^{-z^{\\small(i)}}} \\; \\small (\\text{sigmoid function} \\; )\n\\\\\n\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":237,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":202,"value":"md `## Loss Function ${tex `\\mathcal{L} ( \\hat y^{(i)}, y^{(i)} )`}\nMeasure error for one training example. ${tex `\\hat y^{(i)} \\approx y^{(i)}`}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":243,"value":"md `\n

\n${tex `\n\\Large\n \\mathcal{L}( a^{(i)},y^{(i)} ) = - y^{(i)} \\cdot \\log (a^{(i)}) - (1 - y^{(i)}) \\cdot \\log (1 - a^{(i)}) \n\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":706,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":238,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":218,"value":"md `## Cost Function ${tex `J ( w, b )`} and Vectorization across multiple examples\nMean of error across the entire training set (m training examples)\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":276,"value":"md `\n

\n${tex `\n\\Large\n J(w, b) =\\frac {1} {m} \\displaystyle\\sum_{i=1}^m \\mathcal{L}(\\hat y^{(i)},y^{(i)} ) \n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":399,"value":"md `\n${tex `\n\\large\nw \\in \\R^{n \\times 1} \\;\\;: \nw = \\begin{bmatrix}\nw_1\\\\\n\\vdots \\\\\nw_n\\\\\n\\end{bmatrix}\n,\\;\\; X \\in \\R^{n \\times m}\\;\\;: X = \n\\begin{bmatrix}\n\\mid & \\mid & & \\mid \\\\\nx^{(1)} & x^{(2)} & \\ldots & x^{(m)} \\\\\n\\mid & \\mid & & \\mid \\\\\n\\end{bmatrix}\\\\\nY \\in \\R^{1 \\times m} \\;\\;: Y = \n\\begin{bmatrix}\ny^{(1)} & \\cdots & y^{(m)}\n\\end{bmatrix}\n\\\\\nA \\in \\R^{1 \\times m} \\;\\;: A = \n\\begin{bmatrix}\na^{(1)} & \\cdots & a^{(m)}\n\\end{bmatrix}\n\\\\\nZ \\in \\R^{1 \\times m}\\;\\;: Z = \n\\begin{bmatrix}\nz^{(1)} & \\ldots & z^{(m)} \n\\end{bmatrix}\n\\\\\nB = \n\\begin{bmatrix}\nb & b & \\ldots & b\n\\end{bmatrix}\n\n`}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":480,"value":"md `\n

\n${tex `\n\\Large\n Z = w^T \\cdot X + B\n\\;\\; \\to \\large (\\forall i,\\; z^{(i)} = w^T \\cdot x^{(i)} + b)\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":321,"value":"md `\n

\n${tex `\n\\Large\n A = \\sigma(Z)\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":536,"value":"md `\n

\n${tex `\n\\Large\n J(w, b) = -\\frac{1}{m} \\cdot sum\\bigg(\\; Y * log(A) + (1 - Y) * log(1 - A)\\;\\bigg)\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":417,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":420,"value":"md `## Gradient Descent \nminimize cost function J(w,b)\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":650,"value":"md `\n#### repeat until convergence \\{\n

\n${tex `\n\\Large\n\\;\\;\\;\\; \\normalsize \\textit{// compute activation \\;\\; \\small (prediction)} \\\\\n\\Large\n\\;\\;\\;\\; A = \\cdots \\\\\n\\;\\;\\;\\; \\normalsize \\textit{// compute cost function J \\small (mainly for tracking that J is decreasing across iterations)} \\\\\n\\Large\n \\;\\;\\;\\; J = \\cdots \\\\\n\\Large\n \\;\\;\\;\\; \\textit{\\normalsize // compute gradient descent:}\\\\ \\Large\n \\;\\;\\;\\; w = w - \\alpha \\cdot \\frac{\\partial }{\\partial w} J \\small(w, b) \\\\ \\Large\n \\;\\;\\;\\; | \\\\\n \\;\\;\\;\\; b = b - \\alpha \\cdot \\frac{\\partial }{\\partial b} J\\small(w, b)\n`}\n

\n#### \\}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":963,"value":"md `#### Partial Derivatives:\n#### relative to w:\n(vectorization form on all parameters ${tex `w_j`})\n

\n${tex `\n\\large\n \\frac{\\partial } {\\partial w} J(w, b) =\\frac {1} {m} \\displaystyle\\sum_{i=1}^m \\frac{\\partial }{\\partial w} \\mathcal{L}(\\hat a^{(i)},y^{(i)} )\n\n `}\n

\n`\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":1007,"value":"md `\n

\n${tex `\n\\Large\n\\frac{\\partial }{\\partial w} \\mathcal{L}(a, y) = \\frac{\\partial \\mathcal{L}}{\\partial a} \\cdot \\frac{\\partial a}{\\partial z} \\cdot \\frac{\\partial z}{\\partial w} \\small\\textit{ (chain rule)}\n= \\large (1) \\cdot (2) \\cdot (3)\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1009,"value":"md `\n

\n${tex `\n\\large\n(1) = \\frac{\\partial \\mathcal{L}}{\\partial a} = \\frac{\\partial }{\\partial a} ( - y \\cdot \\log (a) - (1 - y) \\cdot \\log (1 - a) ) = -\\frac{y}{a} + \\frac{1-y}{1-a}\n\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1016,"value":"md `\n

\n${tex `\n\\large\n \n(2) = \\frac{\\partial a}{\\partial z} = \\sigma^{'}(z) = \\sigma(z) (1 - \\sigma(z)) = a (1 - a)\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1027,"value":"md `\n

\n${tex `\n\\large\n \n(3) = \\frac{\\partial z}{\\partial w} = \\frac{\\partial}{\\partial w} (w^{T} \\cdot x + b) = x\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1034,"value":"md `\n

\n${tex `\n\\Large\n \n\\implies \\frac{\\partial }{\\partial w} \\mathcal{L}(a, y) = x \\cdot a(1-a) \\cdot (-\\frac{y}{a} + \\frac{1-y}{1-a})\n= x \\cdot (a - y) \\;\\; \\in \\R^{n \\times 1}\n\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1094,"value":"md `#### relative to b:\n

\n${tex `\n\\normalsize\n \\frac{\\partial } {\\partial b} J(w, b) =\\frac {1} {m} \\displaystyle\\sum_{i=1}^m \\frac{\\partial }{\\partial b} \\mathcal{L}(\\hat a^{(i)},y^{(i)} )\n\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1043,"value":"md `\n

\n${tex `\n\\Large\n\\frac{\\partial }{\\partial b} \\mathcal{L}(a, y) = \\frac{\\partial \\mathcal{L}}{\\partial a} \\cdot \\frac{\\partial a}{\\partial z} \\cdot \\frac{\\partial z}{\\partial b} \\small\\textit{ (chain rule)}\n= \\large (1) \\cdot (2) \\cdot (4)\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1052,"value":"md `\n

\n${tex `\n\\Large\n \n\\implies \\frac{\\partial }{\\partial b} \\mathcal{L}(a, y) = (a - y)\\;\\;\\; \\in \\R^{1 \\times 1} \\; \\;\\;\\small ( (4) = \\frac{\\partial z}{\\partial b} = 1) \n\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":918,"value":"md `#### Vectorization form (on all training examples) of partial derivatives:\n\n

\n${tex `\n\\Large\n\\left\\{ \n\\begin{array}{ll}\n \\frac{\\partial }{\\partial w} J \\small(w, b) \\Large = \\frac{1}{m} X \\cdot (A - Y)^{T} \\;\\; \\in \\R^{n \\times 1}\\\\\n| \\\\\n \\frac{\\partial }{\\partial b} J\\small(w, b) \\Large = \\frac{1}{m} sum(A - Y) \\;\\; \\in \\R^{1 \\times 1}\n\\end{array}\n\\right.\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":378,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1107,"value":"md `## Regularization \nAdding regularization will often help to prevent overfitting\n#### L2 Regularization\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1115,"value":"md `\n

\n${tex `\n\\large\n J(w, b) =\\frac {1} {m} \\displaystyle\\sum_{i=1}^m \\mathcal{L}(\\hat y^{(i)},y^{(i)} ) + \\frac{\\lambda}{2m} \\parallel w \\parallel^{2}\n\\\\\n\\parallel w \\parallel^{2} = \\displaystyle\\sum_{j=1}^n w_j^2 \\;\\; \\small \\textit{(euclidian normal)}\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1136,"value":"md `\n

\n${tex `\n\\large\n \n\\frac{\\partial }{\\partial w_j} (\\displaystyle\\sum_{j=1}^n w_j^2) = 2\\cdot w_j\n\\\\ \\implies \\frac{\\partial \\mathcal L}{\\partial w} = x \\cdot (a - y) + \\lambda \\cdot w \\;\\; \\small (\\textit{vectorization form on all parameters} \\; w_j)\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1181,"value":"md `\n

\n${tex `\n\\Large\n\n\\implies \\frac{\\partial }{\\partial w} J \\small(w, b) \\Large = \\frac{1}{m} X \\cdot (A - Y)^{T} + \\frac{\\lambda}{m} \\cdot w \\;\\; \\in \\R^{n \\times 1} \\;\\;\\small (\\textit{vectorization form on all training examples} )\n\n\n\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1110,"value":"md `---`","pinned":false,"hidden":false,"mode":"ojs"},{"id":550,"value":"sigmoid = (z) => (1 / (1 + Math.exp(-z)))","pinned":false,"hidden":false,"mode":"ojs"},{"id":874,"value":"sigmoidInvert = (a) => Math.log(a / (1 - a))","pinned":false,"hidden":false,"mode":"ojs"},{"id":555,"value":"sigmoidData = (limit = 10, step=1) => {\n var data = [];\n for (var z = -limit; z < limit; z += step) {\n data.push({\n x: z,\n y: sigmoid(z)\n })\n }\n return data;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":703,"value":"loss = (y, z) => -(y === 1 ? Math.log(z) : Math.log(1- z))","pinned":false,"hidden":false,"mode":"ojs"},{"id":709,"value":"lossData = (y, limit = 10, step=1) => {\n var data = [];\n for (var z = step; z < limit; z += step) {\n data.push({\n x: z,\n y: loss(y, z)\n })\n }\n return data;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":381,"value":"formulaBgColor = \"#F4F6F6\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":441,"value":"formulaTxtColor = \"#34495E\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":387,"value":"titleColor = \"#1F618D\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":547,"value":"d3 = require(\"d3@5\")\n","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}