31:T6ca,

## One-dimensional Gaussian distribution estimation
> ### Likelihood of parameter ${tex`\theta`}
> ${tex.block` \begin{aligned} & \log P(X \mid \theta)=\log \prod_{i=1}^{N} p\left(x_{i} \mid \theta\right) \\ =& \sum_{i=1}^{N} \log \frac{1}{\sqrt{2 \pi} \sigma} \exp \left(-\frac{\left(x_{i}-\mu\right)^{2}}{2 \sigma^{2}}\right) \\ =& \sum_{i=1}^{N}\left[\log \frac{1}{\sqrt{2 \pi}}+\log \frac{1}{\sigma}-\frac{\left(x_{i}-\mu\right)^{2}}{2 \sigma^{2}}\right] \end{aligned} `} > ### ${tex`\mu_{M L E}`} solved by MLE >
> ${tex.block` \begin{gathered} \frac{\partial \sum_{i=1}^{N}\left(x_{i}-\mu\right)^{2}}{\partial \mu}=\sum_{i=1}^{N} 2\left(x_{i}-\mu\right)(-1)=0 \\ \Leftrightarrow \sum_{i=1}^{N}\left(x_{i}-\mu\right)=0\\ \Leftrightarrow \sum_{i=1}^{N} x_{i}-\underbrace{\sum_{i=1}^{N} \mu}_{N \mu}=0 \\~\\ \mu_{M L E}=\frac{1}{N} \sum_{i=1}^{N} x_{i} \end{gathered} `} > ### ${tex`\sigma_{M L E}`} solved by MLE >
> ${tex.block` \begin{gathered} \sigma_{M L E}^{2}=\underset{\sigma}{\operatorname{argmax}} P(X \mid \theta) \\ =\underset{\sigma}{\operatorname{argmax}} \underbrace{\sum_{i=1}^{N}\left(-\log \sigma-\frac{\left(x_{i}-\mu\right)^{2}}{2 \sigma^{2}}\right)}_{L} \\ \frac{\partial L}{\partial \sigma}=\sum_{i=1}^{N}\left[-\frac{1}{\sigma}+\left(x_{i}-\mu\right)^{2} \sigma^{-3}\right] \\ \Leftrightarrow \sum_{i=1}^{N}\left[-\sigma^{2}+\left(x_{i}-\mu\right)^{2}\right]=0 \\ \Leftrightarrow-\sum_{i=1}^{N} \sigma^{2}+\sum_{i=1}^{N}\left(x_{i}-\mu\right)^{2}=0 \end{gathered} \\ \sigma_{M L E}^{2}=\frac{1}{N} \sum_{i=1}^{N}\left(x_{i}-\mu\right)^{2} \\ `} >>>> When ${tex`\mu = \mu_{M L E}`}, >
> ${tex.block` \sigma_{M L E}^{2}=\frac{1}{N} \sum_{i=1}^{N}\left(x_{i}-\mu_{M L E}\right)^{2} `}32:T73b,plot1_2 = { let xmin = -5.2; let xmax = 5.2; let w = 800; let alpha; if (pars1_2.side == 'both'){ alpha = pars1_2.sig / 2; }else{ alpha = pars1_2.sig; } let t1 = jstat.normal.inv(alpha, pars1_2.mean, pars1_2.sd ); let t2 = jstat.normal.inv(1-alpha, pars1_2.mean, pars1_2.sd ); let p = plotter({ xDomain: [xmin, xmax], yDomain: [0.5, -0.05], grid: false, width: w, height: w / 2, xLabel: '', yLabel: '', yTickCount: 5, xTickCount: 5, }); let dx = 0.05; let left = [{ x: xmin, y: 0 }] .concat( d3.range(xmin, t1, dx).map(x => ({ x: x, y: jstat.normal.pdf(x, pars1_2.mean, pars1_2.sd ) })) ) .concat([{ x: t1, y: jstat.normal.pdf(t1, pars1_2.mean, pars1_2.sd) },{x: t1, y: 0}]) let right = [{x: t2, y: 0}] .concat( d3.range(t2, xmax, dx).map(x => ({x: x, y:jstat.normal.pdf(x, pars1_2.mean, pars1_2.sd ) })) ) if (pars1_2.side == 'both'){ p.polygon(left, { fill: 'lightblue', width: 0.2 }); p.polygon(right, { fill: 'lightblue', width: 0.2 }); p.text((alpha*100).toFixed(1)+'%', t1*1.5, 0.05, { stroke: 'purple', weight: 600 }); p.text((alpha*100).toFixed(1)+'%', t2*1.5, 0.05, { stroke: 'purple', weight: 600 }); }else if (pars1_2.side == 'left') { p.polygon(left, { fill: 'lightblue', width: 0.2 }); p.text((alpha*100).toFixed(1)+'%', t1*1.5, 0.05, { stroke: 'purple', weight: 600 }); }else if (pars1_2.side == 'right') { p.polygon(right, { fill: 'lightblue', width: 0.2 }); p.text((alpha*100).toFixed(1)+'%', t2*1.5, 0.05, { stroke: 'purple', weight: 600 }); } p.func(x => jstat.normal.pdf(x, pars1_2.mean, pars1_2.sd ), { stroke: 'black', width: 2 }); p.text(((1-pars1_2.sig)*100).toFixed(0)+'%', pars1_2.mean, 0.42, { stroke: 'purple', weight: 600 }); return p.node; }33:T606,

## Hypothesis Test >

${tex` H_0 : \mu = \mu_0`}

${tex` H_A : \mu \neq \mu_0 \Leftarrow H_A`} is what we want to claim

Null and alternative hypotheses should be pre-specified with both verbal statement and mathematical expression
Only parameters and constants can be shown in the hypotheses.

The test statistic

${tex`\large z = \frac{\bar{x} - \mu_0}{\sigma / \sqrt{n}}`}

The test statistic compares the data with what is expected under the null hypothesis.
Why pre-specify the test statistic?
Test statistics based on different statistical models could make different statistical inferences.
Significance level ${tex`\alpha`} is the tolerable risk level that we are willing to take in rejecting a true null hypothesis (i.e., Type 1 error rate).
Why pre-specify α?
Because ${tex`\alpha`} establishes a criterion, or "cut-off", for making a decision about the null hypothesis.

34:T924,plot1_3 = { let xmin0 = pars1_3.mean0 - 5.2; let xmax0 = pars1_3.mean0 + 5.2; let xmin1 = pars1_3.mean1 - 5.2; let xmax1 = pars1_3.mean1 + 5.2; let xmin = d3.min([xmin0, xmin1]); let xmax = d3.max([xmax0, xmax1]); let w = 800; let labels = [tex`H_0`, tex`H_1`, 'H0'] let alpha = pars1_3.sig; let t0 = jstat.normal.inv(1-alpha, pars1_3.mean0, pars1_3.sd ); let t1 = jstat.normal.inv(1-alpha, pars1_3.mean1, pars1_3.sd ); let t2 = jstat.normal.inv(alpha, pars1_3.mean1, pars1_3.sd ); let p = plotter({ xDomain: [xmin, xmax], yDomain: [0.5, -0.05], grid: false, width: w, height: w / 2, xLabel: '', yLabel: '', yTickCount: 5, xTickCount: 5, y: { axis: null } }); let dx = 0.05; let right0 = [{x: t0, y: 0}] .concat( d3.range(t0, xmax0, dx).map(x => ({x: x, y:jstat.normal.pdf(x, pars1_3.mean0, pars1_3.sd ) })) ); let right1 = [{x: t0, y: 0}] .concat( d3.range(t0, xmax1, dx).map(x => ({x: x, y:jstat.normal.pdf(x, pars1_3.mean1, pars1_3.sd ) })) ); let left1 = [{ x: xmin1, y: 0 }] .concat( d3.range(xmin1, t0, dx).map(x => ({ x: x, y: jstat.normal.pdf(x, pars1_3.mean1, pars1_3.sd ) })) ) .concat([{ x: t0, y: jstat.normal.pdf(t0, pars1_3.mean1, pars1_3.sd) },{x: t0, y: 0}]) p.func(x => jstat.normal.pdf(x, pars1_3.mean0, pars1_3.sd ), { stroke: 'grey', width: 2, opacity: 0.7, dash: '2,2' }); p.func(x => jstat.normal.pdf(x, pars1_3.mean1, pars1_3.sd ), { stroke: 'grey', width: 2, opacity: 0.7, }) p.polygon(right0, { fill: 'red', width: 0.2, opacity: 0.3 }); p.text('α', t0+0.2, 0.03, { stroke: 'red', weight: 600 }); p.polygon(right1, { fill: 'lightblue', width: 0.2, opacity: 0.5 }); p.text('1-β', t1, 0.03, { stroke: 'blue', weight: 600 }); p.polygon(left1, { fill: 'orange', width: 0.2, opacity: 0.5 }); p.text('β', t0-0.2, 0.03, { stroke: 'orange', weight: 600 }); p.text('H0', pars1_3.mean0, 0.4, { stroke: 'grey', weight: 600 }); p.text('H1', pars1_3.mean1, 0.4, { stroke: 'grey', weight: 600 }); p.line_segment(t0, -0.05, t0, 0.4, { stroke: 'grey', width: 1}); p.arrow(t0, -0.05, t0+1, -0.05, { stroke: 'grey'}); p.text(`Reject Null Hypothesis`, t0+2.5, -0.06, { stroke: 'grey' }); return p.node; }35:[] 2f:{"title":"ML | 1 Introduction, Math, and Statistics","cells":[{"id":0,"value":"# ML | 1 Introduction, Math, and Statistics","pinned":false,"hidden":false,"mode":"md"},{"id":3,"value":"

\n# Introduction\n## Frequentist vs Bayesian","pinned":false,"hidden":false,"mode":"md"},{"id":15,"value":"> Through machine learning algorithms, we can understand the underlying rules from a large amount of data, so as to build a model to solve the actual problem. Therefore, the main use of data in machine learning is to learn parameters. Suppose we have data ${tex`X`}:","pinned":false,"hidden":false,"mode":"md"},{"id":24,"value":"> ${tex`\n\nX=\\left[\\begin{array}{llll}\nx_{1} & x_{2} & \\cdots & x_{N}\n\\end{array}\\right]_{N \\times p}^{T}=\\left[\\begin{array}{cccc}\nx_{11} & x_{12} & \\cdots & x_{1 p} \\\\\nx_{21} & x_{22} & \\cdots & x_{2 p} \\\\\n\\vdots & \\vdots & \\ddots & \\vdots \\\\\nx_{N 1} & x_{N 2} & \\cdots & x_{N p}\n\\end{array}\\right]_{N \\times p}\n\\\\\n\n`}","pinned":false,"hidden":false,"mode":"md"},{"id":55,"value":"> The Frequentist believes that the parameter ${tex`\\theta`} is a **constant** and the data is a random variable; while the Bayesian believes that the parameter is a **random variable** that follows a probability distribution (the prior).","pinned":false,"hidden":false,"mode":"md"},{"id":80,"value":"

\n### Frequentist","pinned":false,"hidden":false,"mode":"md"},{"id":64,"value":"> The common method is the maximum likelihood estimation:","pinned":false,"hidden":false,"mode":"md"},{"id":69,"value":"> ${tex`\n\\theta_{M L E}=\\underset{\\theta}{\\operatorname{argmax} \\log P}(X \\mid \\theta), \\text { where } L(\\theta)=\\log P(X \\mid \\theta)\n`}","pinned":false,"hidden":false,"mode":"md"},{"id":75,"value":"> The steps for the solution are:\n1. build the model.\n2. Defining the loss function.\n3. Optimize the loss function.","pinned":false,"hidden":false,"mode":"md"},{"id":85,"value":"

\n### Bayesian","pinned":false,"hidden":false,"mode":"md"},{"id":89,"value":"> The parameter ${tex`\\theta`} is a random variable that has a probability distribution ${tex`P(X)`}, called the prior distribution. When we have sample data ${tex`X`}, its posterior probability is:","pinned":false,"hidden":false,"mode":"md"},{"id":94,"value":"> ${tex.block`\n\\underbrace{P(\\theta \\mid X)}_{\\text {posterior }}=\\frac{\\overbrace{P(X \\mid \\theta)}^{\\text {likelihood}} \\overbrace{P(\\theta)}^{\\text {prior}}}{P(X)} \n\n\\\\~\\\\\n`}\n\n> ${tex`\nwhere P(X)=\\int_{\\theta} P(X \\mid \\theta) P(\\theta) \\mathrm{d} \\theta ，therefore:\n\n\\\\~\\\\\n`}\n\n> ${tex.block`\nP(\\theta \\mid X) \\propto P(X \\mid \\theta) P(\\theta)\n\n\\\\~\\\\\n`}\n\n> ${tex`\n\\text{Maximum a posteriori probability (MAP)} :\n\n\\\\~\\\\\n`}\n\n> ${tex.block`\n\\theta_{M A P}=\\underset{\\theta}{\\operatorname{argmax}} P(\\theta \\mid X)=\\underset{\\theta}{\\operatorname{argmax}} P(X \\mid \\theta) P(\\theta)\n`}\n","pinned":false,"hidden":false,"mode":"md"},{"id":137,"value":"

\n# Math","pinned":false,"hidden":false,"mode":"md"},{"id":142,"value":"> Given data ${tex`X`}:\n>\n> \n> ${tex`\n\nX=\\left[\\begin{array}{llll}\nx_{1} & x_{2} & \\cdots & x_{N}\n\\end{array}\\right]_{N \\times p}^{T}=\\left[\\begin{array}{cccc}\nx_{11} & x_{12} & \\cdots & x_{1 p} \\\\\nx_{21} & x_{22} & \\cdots & x_{2 p} \\\\\n\\vdots & \\vdots & \\ddots & \\vdots \\\\\nx_{N 1} & x_{N 2} & \\cdots & x_{N p}\n\\end{array}\\right]_{N \\times p}\n\\\\~\\\\\n\n\\text{where } x_{i} \\in \\mathbb{R}^{p} \\text{ and } x_{i} \\stackrel{i i d}{\\sim} N(\\mu, \\Sigma) \\\\~\\\\\n\n\\text{Parameter } \\theta=(\\mu, \\Sigma)\n\n`}","pinned":false,"hidden":false,"mode":"md"},{"id":166,"value":"

\n## Mean and variance of Gaussian distribution by maximum likelihood estimation (MLE)","pinned":false,"hidden":false,"mode":"md"},{"id":171,"value":"

\n### MLE\n\n> ${tex`\n\\theta_{M L E}=\\underset{\\theta}{\\operatorname{argmax}} P(X \\mid \\theta)\n`}","pinned":false,"hidden":false,"mode":"md"},{"id":177,"value":"

\n## Gaussian distribution\n\n> One-dimensional Gaussian distribution:\n> ${tex.block`\np(x)=\\frac{1}{\\sqrt{2 \\pi} \\sigma} \\exp \\left(-\\frac{(x-\\mu)^{2}}{2 \\sigma^{2}}\\right)\n`}\n\n
\n\n> Multidimensional Gaussian distribution:\n> ${tex.block`\np(x)=\\frac{1}{(2 \\pi)^{D / 2}|\\Sigma|^{1 / 2}} \\exp \\left(-\\frac{1}{2}(x-\\mu)^{T} \\Sigma^{-1}(x-\\mu)\\right)\n`}","pinned":false,"hidden":false,"mode":"md"},{"id":187,"value":"$31","pinned":false,"hidden":false,"mode":"md"},{"id":242,"value":"

\n# Statistics: Basic Concepts","pinned":false,"hidden":false,"mode":"md"},{"id":246,"value":"## Bias and Variability\n>

Sample statistic is a point estimate of population parameter

Two measures for a point estimate:

Bias: systematic error from the truth: ${tex`\\bar{x} - \\mu`}.
Variability (${tex`s^2`}): measures the dispersion of the sampling data, i.e., it describes the spread of the sampling distribution\n

Bias is the difference between the sample mean ${tex`\\bar{x}`} and the population mean ${tex`\\mu`}

Variability describes the spread of the sampling distribution\n

","pinned":false,"hidden":false,"mode":"md"},{"id":250,"value":"## Confidence Interval\n>

Standard deviation (SD) ${tex`\\sigma`} measures the amount of variability, or data dispersion, from the individual data values to the population mean ${tex`\\mu`}.

Standard error of mean (SEM) ${tex`\\bar{x}`} , which is ${tex`\\frac{\\sigma}{\\sqrt{n}}`}, measures how far the sample mean ${tex`\\bar{x}`} is likely to be from the population mean ${tex`\\mu`}.

SEM (${tex`\\frac{\\sigma}{\\sqrt{n}}`}) is always smaller than SD (${tex`\\sigma`} )

Sample size increases ${tex`\\Rightarrow`} SEM decreases (i.e., more precision for the mean).

\n\n
","pinned":false,"hidden":false,"mode":"md"},{"id":254,"value":"viewof pars1_1 = Inputs.form({\n n: Inputs.range([50, 200], {step: 5, value: 100, label: tex`\\text{Sample Size: }(n)`}),\n mean: Inputs.range([-2, 2], {step: 0.01, value: 0, label: tex`\\text{Mean: }(\\mu)`}),\n sd: Inputs.range([0.3, 1], {step: 0.1, value: 0.5, label: tex`\\text{SD: }(\\sigma)`})\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":257,"value":"legend1_1 = html`

${['Normal Distribution of Sample Means', 'Distribution of Individuals'].map((d,i) => htl.html`

${d.toUpperCase()}

`)}

`","pinned":false,"hidden":false,"mode":"ojs"},{"id":260,"value":"plot1_1 = Plot.plot({\n inset: 10,\n grid: true,\n style: {\n background: '#fafafa',\n },\n x: {\n domain: [-2, 2]\n },\n y: {\n domain: [0, 10]\n },\n color: {\n legend: true\n },\n marks: [\n Plot.line(data1_1, {x: \"x\", y: \"sd\", stroke: 'blue'}),\n Plot.line(data1_1, {x: \"x\", y: \"sem\", stroke: 'green'}),\n // Plot.text(data1_1, {filter: (d,i) => i === Math.round(pars.n * 0.5), x: \"x\", y: \"sem\", text: d => `Normal Distribution \\n of Smaple Means`, dx: 18, dy:-10, fill: 'red'}),\n // Plot.text(data1_1, {filter: (d,i) => i === Math.round(pars.n * 0.5), x: \"x\", y: \"sd\", text: d => `Distribution of \\n Individuals`, dx: 18, dy:-10, fill: 'blue'}),\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":276,"value":">

A two-sided (1 – ${tex`\\alpha`}) x 100% confidence interval, (lower limit, upper limit), for a population mean ${tex`\\mu`} is

${tex`\\large \\bar{x}\\pm z_{\\alpha/2} \\frac{\\sigma}{\\sqrt{n}} `}

What does (1 – ${tex`\\alpha`}) x 100% C.I. mean?
\n We have (1 – ${tex`\\alpha`}) x 100% confidence that the population mean ${tex`\\mu`} falls in this interval, by assuming no bias introduced in the study.\n

${tex`\\large P\\left(\\bar{x} - z_{\\alpha/2} \\frac{\\sigma}{\\sqrt{n}} < \\mu < \\bar{x} + z_{\\alpha/2} \\frac{\\sigma}{\\sqrt{n}} \\right) = 1 - \\alpha`}

For a fixed confidence level, e.g., 95%,
\n Larger sample size ${tex`\\Rightarrow`} narrower confidence interval ${tex`\\Rightarrow`} more precise information on the value of ${tex`\\mu`}\n

","pinned":false,"hidden":false,"mode":"md"},{"id":280,"value":"viewof pars1_2 = Inputs.form({\n sig: Inputs.range([0.01, 1], {step: 0.01, value: 0.05, label: tex`\\alpha`}),\n mean: Inputs.range([-2, 2], {step: 0.01, value: 0, label: tex`\\text{Mean: }(\\mu)`}),\n sd: Inputs.range([0.1, 2], {step: 0.1, value: 1, label: tex`\\text{SD: }(\\sigma)`}),\n side: Inputs.radio([\"left\", \"right\", \"both\"], {label: \"1-side/2-sides\", value: 'both'})\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":283,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":286,"value":"$33","pinned":false,"hidden":false,"mode":"md"},{"id":290,"value":"

Type I error and Type II error

\n>

${tex`H_0: \\mu \\leq \\mu_0`}

${tex`H_1: \\mu > \\mu_0`}

${tex`\\alpha = P(z > z_a | \\mu = \\mu_0)`}

${tex`\\beta = P(z < z_a | \\mu > \\mu_0)`}

\n
","pinned":false,"hidden":false,"mode":"md"},{"id":294,"value":"viewof pars1_3 = Inputs.form({\n sig: Inputs.range([0.01, 1], {step: 0.01, value: 0.05, label: tex`\\alpha`}),\n mean0: Inputs.range([3, 7], {step: 0.1, value: 5, label: tex`H_0 \\text{ Mean: }(\\mu_0)`}),\n mean1: Inputs.range([6, 10], {step: 0.1, value: 8, label: tex`H_1 \\text{ Mean: }(\\mu)`}),\n sd: Inputs.range([0.1, 2], {step: 0.1, value: 1.2, label: tex`\\text{SD: }(\\sigma)`})\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":297,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":115,"value":"

\n## References\n\n> * https://youtu.be/UJFZOz-4T8w\n> * https://youtu.be/QyUy-wav-DM\n> * https://blog.csdn.net/qq_41485273/article/details/111756029","pinned":false,"hidden":false,"mode":"md"},{"id":271,"value":"

\n## Appendix","pinned":false,"hidden":false,"mode":"md"},{"id":262,"value":"data1_1 = {\n let x = d3.range(-2, 2, 4/pars1_1.n);\n let y = x.map(d => jstat.normal.pdf(d, pars1_1.mean, pars1_1.sd));\n let z = x.map(d => jstat.normal.pdf(d, pars1_1.mean, pars1_1.sd / Math.sqrt(pars1_1.n) ));\n return x.map((d,i) => ({'x': d, 'sd': y[i], 'sem': z[i] }))\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":264,"value":"jstat = require('jstat')","pinned":false,"hidden":false,"mode":"ojs"},{"id":265,"value":"import { plotter } from '50dadfdec01c15a8'","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}