31:T47e,md `

${tex ` \normalsize \textit{ // 0. initialize parameters W and B for each layer l. (W matrices randomly initialized) }\\ \large \text{initialize\_parameters() } \\ \Large \text{repeat until convergence \{} \;\;\;\; \normalsize \textit{// gradient descent iterations} \\ \;\;\;\; \textit{\normalsize // 1. forward propagation}\\ \large \;\;\;\; \text{forward\_propagation (X, params)} \\ \;\;\;\; \normalsize \textit{// 2. compute cost function J \small (mainly for tracking that J is decreasing across iterations)} \\ \large \;\;\;\; J = \cdots \\ \;\;\;\; \normalsize \textit{// 3. initialize backward propagation} \\ \large \;\;\;\; dA^{[L]} = \frac{\partial \mathcal{L}}{\partial A^{[L]}}(A^{[L]}, Y) \\ \;\;\;\; \textit{\normalsize // 4. backward propagation}\\ \large \;\;\;\; \text{forward\_propagation(} dA^{[L]},params) \\ \large \;\;\;\; \textit{\normalsize // 5. update parameters for each layer l:}\\ \large \;\;\;\; W^{[l]} = W^{[l]} - \alpha \cdot dW^{[l]} \\ \large \;\;\;\; B^{[l]} = B^{[l]} - \alpha \cdot dB^{[l]} \\ \text{ \}} `}

` 32:T660,drawFunction = (svg, func, w, name, xdomain) => { const dataFunc = (step=1) => { var data = []; for (var z = xdomain[0]; z <= xdomain[1]; z += step) { data.push({ x: z, y: func(z) }) } return data; }; const height = 130; const data = dataFunc( 0.5); const ydomain = [func(xdomain[0]), func(xdomain[1])]; const yoffset = 25; var x = d3.scaleLinear().domain(xdomain).range([ 0, w ]); var y = d3.scaleLinear().domain(ydomain).range([height, yoffset]); var gs = svg.append("g"); var lines = d3.line() .curve(d3.curveMonotoneX) .x(d => x(d.x)) .y(d => y(d.y)); gs.append("path") .datum(data) .attr("fill", "transparent") .attr("stroke", "steelblue") .attr("stroke-width", 2) .attr("d", lines); const values = [[xdomain[0], "0", "0"], [0, "0.5", "-1.2"], [xdomain[1], "+1", "-1"]]; gs.selectAll("text.value") .data(values) .enter() .append("text") .classed("value", true) .attr("x", d => x(d[0])) .attr("y", d => y(func(d[0]))) .attr("dy", "-0.2em") .attr("dx", d => d[2] + "em") .style("font-size", "0.7em") .text(d => func(d[0]).toFixed(2)); gs.append("text").attr("x", x(0)).attr("y", 0).attr("text-anchor", "middle") .text(`g(z): ${name} function`); /* gs.append("circle") .attr("fill", "steelblue") .attr("cx", x(-1)) .attr("cy", y(sigmoid(-1))) .attr("r", 4);*/ const aAxis = g => g.attr("transform", `translate(0,${y(0) + 2} )`) .call(d3.axisBottom(x).tickValues([xdomain[0], 0, xdomain[1]])); gs.append("g").call(aAxis); return gs; }33:[] 2f:{"title":"Supervised Learning - Deep Learning - Part1","cells":[{"id":0,"value":"md `# Supervised Learning - Deep Learning - Part1\nSummary sheet of mathematical formulas used during a course in \"Deep Learning\"\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":5,"value":"md `## Notations\n| | |\n|---|---|\n|n| number of input features|\n|m| number of training examples|\n|(i)| refers to the i-th training example|\n|[l]| refers to the l-th layer in the network|\n|L| refers to the output layer in the network, this is also the number of hidden layers plus the output layer|\n|${tex` \\large \\{x^{(i)},y^{(i)}\\}`}| i-th traning example, x: input features, y output value (labeled value)|\n|${tex`\\large \\hat y`} or a| prediction, for the same x, we want ${tex `\\large \\hat y \\approx y`}, also noted as a (activation)|\n|g| activation function|\n|{W, b}| training model (learning parameters)|\n|${tex `\\large \\alpha`}| learning rate|\n|${tex `\\large \\lambda`}| regularization parameter|\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":11,"value":"md `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":36,"value":"md `svg image realized with Nodebox3, ©gilbertperrin `","pinned":false,"hidden":false,"mode":"ojs"},{"id":44,"value":"md `\n${tex `\n\n\\textbf{features}: x^{(i)} \\in \\R^{n \\times 1} = \\begin{bmatrix}\nx_1^{(i)}\\\\\nx_2^{(i)}\\\\\n\\vdots\\\\\nx_n^{(i)}\n\\end{bmatrix} , \n\\;\n\\textbf{labels}:\\; y^{(i)} \\in \\R^{n^{[L]} \\times 1} = \\begin{bmatrix}\ny_1^{(i)}\\\\\ny_2^{(i)}\\\\\n\\vdots\\\\\ny_{n^{[L]}}^{(i)}\n\\end{bmatrix} \n\\;\n`\n}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":58,"value":"md `\n${tex `\n\n\n\\textbf{parameters for layer l}:\nW^{[l]} \\in \\R^{n^{[l]} \\times n^{[l-1]}} = \\begin{bmatrix}\nW_{1,1} & \\dots & W_{1,n^{[l-1]}} \\\\\nW_{2,1} & \\dots & W_{2,n^{[l-1]}} \\\\\n\\vdots\\\\\nW_{n^{[l]},1} & \\dots & W_{n^{[l]},n^{[l-1]}}\n\\end{bmatrix}\n,\\; b^{[l]} \\in \\R^{n^{[l]} \\times 1 }\n= \\begin{bmatrix}\nb_1\\\\\nb_2\\\\\n\\vdots\\\\\nb_{n^{[l]}}\n\\end{bmatrix}\n`\n}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":55,"value":"md`

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":82,"value":"md `## Neuron, Activation Unit ${tex ` a^{[l](i)}`}\nfor one training example (i-th example), vectorized form on all units, indice \"(i) omitted.\"\n### Linear input function:\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":93,"value":"md `\n

\n${tex `\n\\LARGE\n z^{[l]} = W^{[l]} \\cdot a^{[l-1]}\\ + b^{[l]} \\;\\;\\;\\;\n \\in \\R^{n^{[l]} \\times 1 }\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":112,"value":"md `### Activation (non linear) output function:\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":118,"value":"md `\n

\n${tex `\n\\LARGE\n a^{[l]} = g^{[l]} (z^{[l]}) \\;\\;\\;\\;\n \\in \\R^{n^{[l]} \\times 1 } \n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":140,"value":"chart2 = { \n const h = 200;\n const svg = d3.select(DOM.svg(width, h));\n \n const functions = [\n {func: sigmoid,\n name: \"sigmoid\",\n range: [-10,10]\n },\n {func: tanh,\n name: \"tanh\",\n range: [-10,10]\n },\n {func: relu,\n name: \"ReLU\",\n range: [-2,2]\n },\n {func: lrelu,\n name: \"Leaky ReLU\",\n range: [-10,10]\n },\n ];\n \n const wpadding = 30;\n const w = (width - (functions.length - 1 ) * wpadding - 2 * 15) / functions.length;\n \n functions.forEach((d, i) => {\n const g = drawFunction(svg, d.func, w, d.name, d.range);\n g.attr(\"transform\", `translate(${10 + i * (w + 30)}, 50)`);\n });\n\n\n \n return svg.node();\n}\n ","pinned":false,"hidden":false,"mode":"ojs"},{"id":84,"value":"md `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":78,"value":"md `## Prediction ${tex `\\hat y^{(i)}`}\nThe prediction is given by the output layer and is computed by the Forward Propagation step.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":240,"value":"md `\n

\n${tex `\n\\LARGE\n a^{[L](i)} = \\hat y^{(i)} \\;\\;\\;\\;\n \\in \\R^{n^{[L]} \\times 1 } \n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":88,"value":"md `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":246,"value":"md `## Vectorization on all Training Samples\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":252,"value":"md `\n${tex `\n\\large\nX \\in \\R^{n \\times m}\\;\\;: X = \n\\begin{bmatrix}\n\\mid & \\mid & & \\mid \\\\\nx^{(1)} & x^{(2)} & \\ldots & x^{(m)} \\\\\n\\mid & \\mid & & \\mid \\\\\n\\end{bmatrix} \\;\\;\\;\\; ,\nY \\in \\R^{n^{[L]} \\times m}\\;\\;: Y = \n\\begin{bmatrix}\n\\mid & \\mid & & \\mid \\\\\ny^{(1)} & y^{(2)} & \\ldots & y^{(m)} \\\\\n\\mid & \\mid & & \\mid \\\\\n\\end{bmatrix}\\\\\nZ^{[l]} \\in \\R^{n^{[l]} \\times m}\\;\\;: Z^{[l]} = \n\\begin{bmatrix}\n\\mid & \\mid & & \\mid \\\\\nz^{[l](1)} & z^{[l](2)} & \\ldots & z^{[l](m)} \\\\\n\\mid & \\mid & & \\mid \\\\\n\\end{bmatrix}\\;\\;\\;\\; ,\nA^{[l]} \\in \\R^{n^{[l]} \\times m}\\;\\;: A^{[l]} = \n\\begin{bmatrix}\n\\mid & \\mid & & \\mid \\\\\na^{[l](1)} & a^{[l](2)} & \\ldots & a^{[l](m)} \\\\\n\\mid & \\mid & & \\mid \\\\\n\\end{bmatrix}\\\\\nB^{[l]} \\in \\R^{n^{[l]} \\times m}\\;\\;: B^{[l]} = \n\\begin{bmatrix}\n\\mid & \\mid & & \\mid \\\\\nb^{[l]} & b^{[l]} & \\ldots & b^{[l]} \\\\\n\\mid & \\mid & & \\mid \\\\\n\\end{bmatrix}\\;\\;\\;\\; ,\n`}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":267,"value":"md ` For each layer l:\n

\n${tex `\n\\LARGE\n Z^{[l]} = W^{[l]} \\cdot A^{[l-1]}\\ + B^{[l]} \\;\\;\\;\\;\n \\in \\R^{n^{[l]} \\times m }\\\\\nA^{[l]} = g^{[l]}(Z^{[l]}) \\;\\;\\;\\;\n \\in \\R^{n^{[l]} \\times m }\\\\\n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":276,"value":"md `

\n## Forward Propagation\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":284,"value":"md `\n\n

\n${tex `\n\\large\nA^{[0]} = X \\\\\n\\text{ for layer l, in range from 1 to L \\{} \\\\\n\\;\\;\\;\\; Z^{[l]} = W^{[l]} \\cdot A^{[l-1]}\\ + B^{[l]} \\\\\n\\;\\;\\;\\; A^{[l]} = g^{[l]} (Z^{[l]}) \\\\\n\\}\n`}\n

\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":302,"value":"md `The last activation layer (output layer), ${tex `A^{[L]}`} is the prediction for all training samples.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":310,"value":"md `

\n## Cost Function\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":313,"value":"md `\n

\n${tex `\n\\Large\n J(\\large W^{[L]}, b^{[L]}, W^{[L-1]}, \\cdots \\Large) =\\frac {1} {m} \\displaystyle\\sum_{i=1}^m \\mathcal{L}( a^{[L](i)},y^{(i)} ) \n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":321,"value":"md ` Example with the cross-entropy loss:\n

\n${tex `\n\n J(\\small W^{[L]}, b^{[L]}, W^{[L-1]}, \\cdots) =- \\frac {1} {m} \\displaystyle\\sum_{i=1}^m y^{(i)} \\cdot log(a^{[L](i)}) + (1 - y^{(i)}) \\cdot log(1 - a^{[L](i)}) \n`}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":328,"value":"md `

\n## Backward Propagation\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":334,"value":"md `\n#### Notation:\n${tex `\n\\large\n dv = \\frac{\\partial } {\\partial v} \\mathcal{L}(\\cdots) \\;\\; \\small (\\text {for any variable 'v' of }\\mathcal{L})\\\\\n\\text{example with v } = z^{[l]}: \\\\ \\Large\ndz^{[l]} = \\frac{\\partial } {\\partial z^{[l]}} \\mathcal{L}(\\cdots)\n\n\n`}\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":352,"value":"md `\n\n

\n${tex `\n\\large\ndA^{[L]} = \\cdots \\textit{// initiation} \\\\\n\\text{ for layer l, in range from L to 1 \\{} \\\\\n\\;\\;\\;\\; dZ^{[l]} = dA^{[l]} * g^{[l]'}(Z^{[l]}) \\\\\n\\;\\;\\;\\; dA^{[l-1]} = W^{[l]T} \\cdot dZ^{[l]}\\\\\n\\;\\;\\;\\; dW^{[l]} = \\frac{1}{m} \\cdot dZ^{[l]} \\cdot A^{[l-1]T} \\\\\n\\;\\;\\;\\; dB^{[l]} = \\frac{1}{m} sum(dZ^{[l]}) \\\\\n\n\\}\n`}\n

\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":369,"value":"md `#### Justification:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":373,"value":"md `\n

\n${tex `\n\\Large\ndz^{[l]} = \\frac{\\partial }{\\partial z^{[l]}} \\mathcal{L}(\\cdots) = \\frac{\\partial \\mathcal{L}}{\\partial a^{[l]}} \\cdot \\frac{\\partial a^{[l]}}{\\partial z^{[l]}} \\small\\textit{ (chain rule)}\n\\\\ \\;\n\\\\\n\\large\n\\;\\;\\;\\; \\implies dz^{[l]} = da^{[l]} * g^{[l]'}(z^{[l]}) \\;\\;\\;\\; \\small ( a^{[l]} = g(z^{[l]}) )\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":425,"value":"md `\n

\n${tex `\n\\Large\nda^{[l-1]} = \\frac{\\partial }{\\partial a^{[l-1]}} \\mathcal{L}(\\cdots) = \\frac{\\partial \\mathcal{L}}{\\partial z^{[l]}} \\cdot \\frac{\\partial z^{[l]}}{\\partial a^{[l-1]}} \\small\\textit{ (chain rule)} \n\\Large\n= dz^{[l]} \\cdot \\frac{\\partial}{\\partial a^{[l-1]}} (W^{[l]}\\cdot a^{[l-1]} + b^{[l]}) \n\\\\ \\;\n\\\\\n\\;\\;\\;\\; \\large \\implies da^{[l-1]} = W^{[l]T} \\cdot dz^{[l]}\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":392,"value":"md `\n

\n${tex `\n\\Large\ndW^{[l]} = \\frac{\\partial }{\\partial W^{[l]}} \\mathcal{L}(\\cdots) = \\frac{\\partial \\mathcal{L}}{\\partial z^{[l]}} \\cdot \\frac{\\partial z^{[l]}}{\\partial W^{[l]}} \\small\\textit{ (chain rule)} \n\\Large\n= dz^{[l]} \\cdot \\frac{\\partial}{\\partial W^{[l]}} (W^{[l]}\\cdot a^{[l-1]} + b^{[l]}) \n\\\\ \\;\n\\\\\n\\;\\;\\;\\; \\large \\implies dW^{[l]} = dz^{[l]} \\cdot a^{[l-1]T}\n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":410,"value":"md `\n

\n${tex `\n\\Large\ndb^{[l]} = \\frac{\\partial }{\\partial b^{[l]}} \\mathcal{L}(\\cdots) = \\frac{\\partial \\mathcal{L}}{\\partial z^{[l]}} \\cdot \\frac{\\partial z^{[l]}}{\\partial b^{[l]}} \\small\\textit{ (chain rule)} \n\\Large\n= dz^{[l]} \\cdot \\frac{\\partial }{\\partial b^{[l]}} (W^{[l]}\\cdot a^{[l-1]} + b^{[l]}) \n\\\\ \\;\n\\\\\n\\;\\;\\;\\; \\large \\implies db^{[l]} = dz^{[l]} \n `}\n

\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":361,"value":"md `

\n## Gradient Descent\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":433,"value":"md `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":445,"value":"md `svg image realized with Nodebox3, ©gilbertperrin `","pinned":false,"hidden":false,"mode":"ojs"},{"id":442,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":364,"value":"","pinned":false,"hidden":false,"mode":"ojs"},{"id":249,"value":"md `

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":123,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":129,"value":"sigmoid = (z) => (1 / (1 + Math.exp(-z)))","pinned":false,"hidden":false,"mode":"ojs"},{"id":177,"value":"relu = (z) => {\n return Math.max(0.0, z);\n} ","pinned":false,"hidden":false,"mode":"ojs"},{"id":216,"value":"lrelu = (z) => {\n return z <= 0 ? 0.1 * z : z;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":203,"value":"tanh = (z) => (( Math.exp(z) - Math.exp(-z)) / (Math.exp(z) + Math.exp(-z)))","pinned":false,"hidden":false,"mode":"ojs"},{"id":126,"value":"d3 = require(\"d3@5\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":7,"value":"titleColor = \"#1F618D\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":95,"value":"formulaTxtColor = \"#34495E\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":98,"value":"formulaBgColor = \"#F4F6F6\"\n","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"}