diff --git a/statistical-and-mathematical-methods-for-ai/img/_orthogonal_complement.pdf b/statistical-and-mathematical-methods-for-ai/img/_orthogonal_complement.pdf new file mode 100644 index 0000000..b67b05e Binary files /dev/null and b/statistical-and-mathematical-methods-for-ai/img/_orthogonal_complement.pdf differ diff --git a/statistical-and-mathematical-methods-for-ai/main.tex b/statistical-and-mathematical-methods-for-ai/main.tex index 28ccab3..1f6ea18 100644 --- a/statistical-and-mathematical-methods-for-ai/main.tex +++ b/statistical-and-mathematical-methods-for-ai/main.tex @@ -1,7 +1,7 @@ \documentclass[11pt]{article} -\usepackage[margin=3cm, lmargin=2cm, rmargin=4cm]{geometry} +\usepackage[margin=3cm, lmargin=2cm, rmargin=4cm, marginparwidth=3cm]{geometry} \usepackage{graphicx, xcolor} -\usepackage{amsmath, amsfonts, amssymb, amsthm, mathtools} +\usepackage{amsmath, amsfonts, amssymb, amsthm, mathtools, bm} \usepackage{hyperref} \usepackage[nameinlink]{cleveref} \usepackage[all]{hypcap} % Links hyperref to object top and not caption @@ -23,14 +23,16 @@ \setlist[description]{labelindent=\parindent} % Indents `description` \renewcommand*{\marginfont}{\color{gray}\footnotesize} +\theoremstyle{definition} +\newtheorem{theorem}{Theorem}[section] \newtheorem{example}{Example}[section] -\newtheoremstyle{definition}{}{}{}{}{\bfseries}{.}{ }{\thmname{#1}\thmnumber{ #2}\thmnote{ (#3)}} \theoremstyle{definition} \newtheorem*{definition}{Def} \newcommand{\ubar}[1]{\text{\b{$#1$}}} -\renewcommand{\vec}[1]{\text{\textbf{#1}}} +\renewcommand{\vec}[1]{{\bm{#1}}} \newcommand{\nullvec}[0]{\bar{\vec{0}}} +\newcommand{\matr}[1]{{\bm{#1}}} \begin{document} diff --git a/statistical-and-mathematical-methods-for-ai/sections/_linear_algebra.tex b/statistical-and-mathematical-methods-for-ai/sections/_linear_algebra.tex new file mode 100644 index 0000000..ebd9b71 --- /dev/null +++ b/statistical-and-mathematical-methods-for-ai/sections/_linear_algebra.tex @@ -0,0 +1,262 @@ +\section{Linear algebra} + + +\subsection{Vector space} + +A \textbf{vector space} over $\mathbb{R}$ is a nonempty set $V$, whose elements are called vectors, with two operations: +\marginnote{Vector space} +\begin{center} + \begin{tabular}{l c} + Addition & $+ : V \times V \rightarrow V$ \\ + Scalar multiplication & $\cdot : \mathbb{R} \times V \rightarrow V$ + \end{tabular} +\end{center} +A vector space has the following properties: +\begin{enumerate} + \item Addition is commutative and associative + \item A null vector exists: $\exists \nullvec \in V$ s.t. $\forall \vec{u} \in V: \nullvec + \vec{u} = \vec{u} + \nullvec = \vec{u}$ + \item An identity element for scalar multiplication exists: $\forall \vec{u} \in V: 1\vec{u} = \vec{u}$ + \item Each vector has its opposite: $\forall \vec{u} \in V, \exists \vec{a} \in V: \vec{a} + \vec{u} = \vec{u} + \vec{a} = \nullvec$ + \item Distributive properties: + \[ \forall \alpha \in \mathbb{R}, \forall \vec{u}, \vec{w} \in V: \alpha(\vec{u} + \vec{w}) = \alpha \vec{u} + \alpha \vec{w} \] + \[ \forall \alpha, \beta \in \mathbb{R}, \forall \vec{u} \in V: (\alpha + \beta)\vec{u} = \alpha \vec{u} + \beta \vec{u} \] + \item Associative property: + \[ \forall \alpha, \beta \in \mathbb{R}, \forall \vec{u} \in V: (\alpha \beta)\vec{u} = \alpha (\beta \vec{u}) \] +\end{enumerate} +% +A subset $U \subseteq V$ of a vector space $V$, is a \textbf{subspace} iff $U$ is a vector space. +\marginnote{Subspace} + + +\subsubsection{Basis} +\marginnote{Basis} +Let $V$ be a vector space of dimension $n$. +A basis $\beta = \{ \vec{v}_1, \dots, \vec{v}_n \}$ of $V$ is a set of $n$ linear independent vectors of $V$.\\ +Each element of $V$ can be represented as a linear combination of the vectors in the basis $\beta$: +\[ \forall \vec{w} \in V: \vec{w} = \lambda_1\vec{v}_1 + \dots + \lambda_n\vec{v}_n \text{ where } \lambda_i \in \mathbb{R} \] +% +The canonical basis of a vector space is a basis where each vector represents a dimension $i$ \marginnote{Canonical basis} +(i.e. 1 in position $i$ and 0 in all other positions). +\begin{example} + The canonical basis $\beta$ of $\mathbb{R}^3$ is $\beta = \{ (1, 0, 0), (0, 1, 0), (0, 0, 1) \}$ +\end{example} + +\subsubsection{Dot product} +The dot product of two vectors in $\vec{x}, \vec{y} \in \mathbb{R}^n$ is defined as: \marginnote{Dot product} +\begin{equation*} + \left\langle \vec{x}, \vec{y} \right\rangle = + \vec{x}^T \vec{y} = \sum_{i=1}^{n} x_i \cdot y_i +\end{equation*} + + +\subsection{Matrix} + +This is a {\tiny(very formal definition of)} matrix: \marginnote{Matrix} +\begin{equation*} + \matr{A} = + \begin{pmatrix} + a_{11} & a_{12} & \dots & a_{1n} \\ + a_{21} & a_{22} & \dots & a_{2n} \\ + \vdots & \vdots & \ddots & \vdots \\ + a_{m1} & a_{m2} & \dots & a_{mn} + \end{pmatrix} +\end{equation*} + +\subsubsection{Invertible matrix} +A matrix $\matr{A} \in \mathbb{R}^{n \times n}$ is invertible (non-singular) if: \marginnote{Non-singular matrix} +\begin{equation*} + \exists \matr{B} \in \mathbb{R}^{n \times n}: \matr{AB} = \matr{BA} = \matr{I} +\end{equation*} +where $\matr{I}$ is the identity matrix. $\matr{B}$ is denoted as $\matr{A}^{-1}$. + +\subsubsection{Kernel} +The null space (kernel) of a matrix $\matr{A} \in \mathbb{R}^{m \times n}$ is a subspace such that: \marginnote{Kernel} +\begin{equation*} + \text{Ker}(\matr{A}) = \{ \vec{x} \in \mathbb{R}^n : \matr{A}\vec{x} = \nullvec \} +\end{equation*} +% +\begin{theorem} \label{th:kernel_invertible} + A square matrix $\matr{A}$ with $\text{\normalfont Ker}(\matr{A}) = \{\nullvec\}$ is non singular. +\end{theorem} + + + +\subsection{Norms} + +\subsubsection{Vector norms} +The norm of a vector is a function: \marginnote{Vector norm} +\begin{equation*} + \Vert \cdot \Vert: \mathbb{R}^n \rightarrow \mathbb{R} +\end{equation*} +such that for each $\lambda \in \mathbb{R}$ and $\vec{x}, \vec{y} \in \mathbb{R}^n$: +\begin{itemize} + \item $\Vert \vec{x} \Vert \geq 0$ + \item $\Vert \vec{x} \Vert = 0 \iff \vec{x} = 0$ + \item $\Vert \lambda \vec{x} \Vert = \vert \lambda \vert \cdot \Vert \vec{x} \Vert$ + \item $\Vert \vec{x} + \vec{y} \Vert \leq \Vert \vec{x} \Vert + \Vert \vec{y} \Vert$ +\end{itemize} +% +Common norms are: +\begin{description} + \item[2-norm] $\Vert \vec{x} \Vert_2 = \sqrt{ \sum_{i=1}^{n} x_i^2 }$ + + \item[1-norm] $\Vert \vec{x} \Vert_1 = \sum_{i=1}^{n} \vert x_i \vert$ + + \item[$\infty$-norm] $\Vert \vec{x} \Vert_{\infty} = \max_{1 \leq i \leq n} \vert x_i \vert$ +\end{description} +% +In general, different norms tend to maintain the same proportion. +In some cases, unbalanced results may be given when comparing different norms. +\begin{example} + Let $\vec{x} = (1, 1000)$ and $\vec{y} = (999, 1000)$. Their norms are: + \begin{center} + \begin{tabular}{l l} + $\Vert \vec{x} \Vert_{2} = \sqrt{1000001}$ & $\Vert \vec{y} \Vert_{2} = \sqrt{1998001}$ \\ + $\Vert \vec{x} \Vert_{\infty} = 1000$ & $\Vert \vec{y} \Vert_{\infty} = 1000$ \\ + \end{tabular} + \end{center} +\end{example} + + +\subsubsection{Matrix norms} +The norm of a matrix is a function: \marginnote{Matrix norm} +\begin{equation*} + \Vert \cdot \Vert: \mathbb{R}^{m \times n} \rightarrow \mathbb{R} +\end{equation*} +such that for each $\lambda \in \mathbb{R}$ and $\matr{A}, \matr{B} \in \mathbb{R}^{m \times n}$: +\begin{itemize} + \item $\Vert \matr{A} \Vert \geq 0$ + \item $\Vert \matr{A} \Vert = 0 \iff \matr{A} = \bar{0}$ + \item $\Vert \lambda \matr{A} \Vert = \vert \lambda \vert \cdot \Vert \matr{A} \Vert$ + \item $\Vert \matr{A} + \matr{B} \Vert \leq \Vert \matr{A} \Vert + \Vert \matr{B} \Vert$ +\end{itemize} +% +Common norms are: +\begin{description} + \item[2-norm] + $\Vert \matr{A} \Vert_2 = \sqrt{ \rho(\matr{A}^T\matr{A}) }$,\\ + where $\rho(\matr{X})$ is the largest absolute value of the eigenvalues of $\matr{X}$ (spectral radius). + + \item[1-norm] $\Vert \matr{A} \Vert_1 = \max_{1 \leq j \leq n} \sum_{i=1}^{m} \vert a_{i,j} \vert$ + + \item[Frobenius norm] $\Vert \matr{A} \Vert_F = \sqrt{ \sum_{i=1}^{m} \sum_{j=1}^{n} a_{i,j}^2 }$ +\end{description} + + + +\subsection{Symmetric, positive definite matrices} + +\begin{description} + \item[Symmetric matrix] \marginnote{Symmetric matrix} + A square matrix $\matr{A} \in \mathbb{R}^{n \times n}$ is symmetric $\iff \matr{A} = \matr{A}^T$ + + \item[Positive semidefinite matrix] \marginnote{Positive semidefinite matrix} + A symmetric matrix $\matr{A} \in \mathbb{R}^{n \times n}$ is positive semidefinite iff + \begin{equation*} + \forall \vec{x} \in \mathbb{R}^n \smallsetminus \{0\}: \vec{x}^T \matr{A} \vec{x} \geq 0 + \end{equation*} + + \item[Positive definite matrix] \marginnote{Positive definite matrix} + A symmetric matrix $\matr{A} \in \mathbb{R}^{n \times n}$ is positive definite iff + \begin{equation*} + \forall \vec{x} \in \mathbb{R}^n \smallsetminus \{0\}: \vec{x}^T \matr{A} \vec{x} > 0 + \end{equation*} + % + It has the following properties: + \begin{enumerate} + \item The null space of $\matr{A}$ has the null vector only: $\text{Ker}(\matr{A}) = \{ \nullvec \}$. \\ + Which implies that $\matr{A}$ is non-singular (\Cref{th:kernel_invertible}). + \item The diagonal elements of $\matr{A}$ are all positive. + \end{enumerate} + \begin{theorem} + If the eigenvalues of a symmetric matrix $\matr{B} \in \mathbb{R}^{n \times n}$ are all positive. + Then $\matr{B}$ is positive definite. + \end{theorem} +\end{description} + + + +\subsection{Orthogonality} +\begin{description} + \item[Angle between vectors] \marginnote{Angle between vectors} + The angle $\omega$ between two vectors $\vec{x}$ and $\vec{y}$ can be obtained from: + \begin{equation*} + \cos\omega = \frac{\left\langle \vec{x}, \vec{y} \right\rangle }{\Vert \vec{x} \Vert_2 \cdot \Vert \vec{y} \Vert_2} + \end{equation*} + + \item[Orthogonal vectors] \marginnote{Orthogonal vectors} + Two vectors $\vec{x}$ and $\vec{y}$ are orthogonal ($\vec{x} \perp \vec{y}$) when: + \[ \left\langle \vec{x}, \vec{y} \right\rangle = 0 \] + + \item[Orthonormal vectors] \marginnote{Orthonormal vectors} + Two vectors $\vec{x}$ and $\vec{y}$ are orthonormal when: + \[ \vec{x} \perp \vec{y} \text{ and } \Vert \vec{x} \Vert = \Vert \vec{y} \Vert=1 \] + \begin{theorem} + The canonical basis of a vector space is orthonormal. + \end{theorem} + + \item[Orthogonal matrix] \marginnote{Orthogonal matrix} + A matrix $\matr{A} \in \mathbb{R}^{n \times n}$ is orthogonal if its columns are \underline{orthonormal} vectors. + It has the following properties: + \begin{enumerate} + \item $\matr{A}\matr{A}^T = \matr{I} = \matr{A}^T\matr{A}$, which implies $\matr{A}^{-1} = \matr{A}^T$. + \item The length of a vector is unchanged when mapped through an orthogonal matrix: + \[ \Vert \matr{A}\vec{x} \Vert^2 = \Vert \vec{x} \Vert^2 \] + \item The angle between two vectors is unchanged when both are mapped through an orthogonal matrix: + \[ + \cos\omega = \frac{(\matr{A}\vec{x})^T(\matr{A}\vec{y})}{\Vert \matr{A}\vec{x} \Vert \cdot \Vert \matr{A}\vec{y} \Vert} = + \frac{\vec{x}^T\vec{y}}{\Vert \vec{x} \Vert \cdot \Vert \vec{y} \Vert} + \] + \end{enumerate} + + \item[Orthogonal basis] \marginnote{Orthogonal basis} + Given an $n$-dimensional vector space $V$ and a basis $\beta = \{ \vec{b}_1, \dots, \vec{b}_n \}$ of $V$. + $\beta$ is an orthogonal basis if: + \[ \vec{b}_i \perp \vec{b}_j \text{ for } i \neq j \text{ (i.e.} \left\langle \vec{b}_i, \vec{b}_j \right\rangle = 0 \text{)} \] + + \item[Orthonormal basis] \marginnote{Orthonormal basis} + Given an $n$-dimensional vector space $V$ and an orthogonal basis $\beta = \{ \vec{b}_1, \dots, \vec{b}_n \}$ of $V$. + $\beta$ is an orthonormal basis if: + \[ \Vert \vec{b}_i \Vert_2 = 1 \text{ (or} \left\langle \vec{b}_i, \vec{b}_i \right\rangle = 1 \text{)} \] + + \item[Orthogonal complement] \marginnote{Orthogonal complement} + Given a $n$-dimensional vector space $V$ and a $m$-dimensional subspace $U \subseteq V$. + The orthogonal complement $U^\perp$ of $U$ is a $(n-m)$-dimensional subspace of $V$ such that it + contains all the vectors orthogonal to every vector in $U$: + \[ \forall \vec{w} \in V: \vec{w} \in U^\perp \iff (\forall \vec{u} \in U: \vec{w} \perp \vec{u}) \] + % + Note that $U \cap U^\perp = \{ \nullvec \}$ and + it is possible to represent all vectors in $V$ as a linear combination of both the basis of $U$ and $U^\perp$. + + The vector $\vec{w} \in U^\perp$ s.t. $\Vert \vec{w} \Vert = 1$ is the \textbf{normal vector} of $U$. \marginnote{Normal vector} + % + \begin{figure}[h] + \centering + \includegraphics[width=0.4\textwidth]{img/_orthogonal_complement.pdf} + \caption{Orthogonal complement of a subspace $U \subseteq \mathbb{R}^3$} + \end{figure} +\end{description} + + + +\subsection{Projections} +Projections are methods to map high-dimensional data into a lower-dimensional space +while minimizing the compression loss.\\ +\marginnote{Orthogonal projection} +Let $V$ be a vector space and $U \subseteq V$ a subspace of $V$. +A linear mapping $\pi: V \rightarrow U$ is a (orthogonal) projection if: +\[ \pi^2 = \pi \circ \pi = \pi \] +In other words, applying $\pi$ multiple times gives the same result (i.e. idempotency).\\ +$\pi$ can be expressed as a transformation matrix $\matr{P}_\pi$ such that: +\[ \matr{P}_\pi^2 = \matr{P}_\pi \] + +\subsubsection{Projection onto general subspaces} +To project a vector $\vec{x} \in \mathbb{R}^n$ into a lower-dimensional subspace $U \subseteq \mathbb{R}^n$, +it is possible to use the basis of $U$.\\ +% +Let $m = \text{dim}(U)$ be the dimension of $U$ and +$\matr{B} = (\vec{b}_1, \dots, \vec{b}_m) \in \mathbb{R}^{n \times m}$ an ordered basis of $U$. +A projection $\pi_U(\vec{x})$ represents $\vec{x}$ as a linear combination of the basis: +\[ \pi_U(\vec{x}) = \sum_{i=1}^{m} \lambda_i \vec{b}_i = \matr{B}\vec{\lambda} \] +where $\vec{\lambda} = (\lambda_1, \dots, \lambda_m)^T \in \mathbb{R}^{m}$ are the new coordinates of $\vec{x}$ +and is found by minimizing the distance between $\pi_U(\vec{x})$ and $\vec{x}$. \ No newline at end of file