From aee383d8bc2afa4d6ee9b854c93488f1b3903c37 Mon Sep 17 00:00:00 2001 From: NotXia <35894453+NotXia@users.noreply.github.com> Date: Sat, 23 Sep 2023 16:16:54 +0200 Subject: [PATCH] Add SMM linear algebra --- .../img/_orthogonal_complement.pdf | Bin 0 -> 5050 bytes .../main.tex | 10 +- .../sections/_linear_algebra.tex | 262 ++++++++++++++++++ 3 files changed, 268 insertions(+), 4 deletions(-) create mode 100644 statistical-and-mathematical-methods-for-ai/img/_orthogonal_complement.pdf create mode 100644 statistical-and-mathematical-methods-for-ai/sections/_linear_algebra.tex diff --git a/statistical-and-mathematical-methods-for-ai/img/_orthogonal_complement.pdf b/statistical-and-mathematical-methods-for-ai/img/_orthogonal_complement.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b67b05ea87863ad86c80d1ecf6dbd595645f623a GIT binary patch literal 5050 zcmb_gcUY6z(vNgQm5zW!1wjEJ2~~RN9TKX521uxpgd`9;8j&s?L7Jjiz*UhdA|0vH zt4foms7niofWQ~ryL;DtxV!iJp6{RcIVbbZWZvJ*Z_dmSv(V9#2A`D&ianTFSph;o zvLLLB7f@9dC}S0fLxW_ror%sEtUFM~!r2{70Lf8bfCFW;T=7^vEdCNm{>NTj9ffE>wL8USDs z5W>1|=(nGLH>v#pZxSr~-`WH#{OqS+f*RXpS*B|wvj+NRSokm1d^o&s;wehG-9 zG4YGPl|eImXXUJF_-v!g1^<`=9{3(& z!E}1MpJXWAX=28TC&3NQG61h2eIXgO+4xVaP}-SRn7 zI`17RD&$C^2RF+{Fax+^IHeA9?9ci0ci~t1OZflSjx|^`^gA}R^}>&fW*DEeLL3)m zc8T_8;sbC6a7rHJ**|8{Z|QDl?&5`ZB?4s-elElx1lUh=pp2+7WCuk z1eDpA{m=sgWneHYRlYA(l$Qs|D=1Lqa5Mqyhj*p&2C}b0B@Tj!_j4s$IOEYiL;^^f zs`oEK$p1Cdu3M>LxxpO2?np}YX})g{*))c6Getz`9KuAwL5*|VbFmTRx}E^6;=~kB zYp&GO$?;$nidzeNLddrI#EVBuAF<)eYaa>m>i1_w!e2D80otE_?qn3_NQZaqR(gh( z`F*H$68z9|WXpW9A?ehS?sti)GggO8rJ_Zn?ROYTY9QOe8$+SH;^n}kS!wH1P3!Z| znRp6Mxw5g}@J@hMKWg@nshG5xowx_nG*n~OnvLWa(5W<_obcBD8oo5otmSoUzB99p zQxO>h(Nf;uQQw%gx6*6>1cGr z-($VYH3&@dS|{^F^xu=PX_xhiw3H(b%^X|rr;BfLMLf@3f4Ms>QkpTWV&dEMkulQNr7w{k?mJ=e zR=2H~ZC~QAy%rPK$Q>JcPhZzm@x>*F?JUYd&+_Zn+x?Nm7ru_a1e~(!F4<|b;rIRu zoAzdyj)3Q(3Llq#@zyC&40}P~e4OIw8B;7`Uow99d>4HmjX{CJ_9Na6j*HWePOXKN zjO_uCDJP#Dkj9XIOyhs069gjvGxeR4)O?yj9M^))z;pNQ8lI$CwZY3g9cT4@@c@>q ze5a4~^7GY(UM^RF12v&bq09V%3m1cc0<(NtT%OmCQiR2_hB>McOdNV#> zFb#r2$(ZG5BwhAE72YpBE9GJlgh%F;1`6jKQ=YX&p5i`V$X2$ItBKz$%jK`BIm)}W zQmB-EMt^;A+-PWNd}cJgO4JkTapcp?m1Xt&+=zQ`x9B@1d_Eq474*NYk_TYb;MF9{ z61fxx6t33xutekv>%Y9_q-FpFU=BbqSJ!-hS}_U#TpGrU-v zO|J2M*rLeYFvA5#@d6g{p^B;AP6A)*6E|2dy}G~ITGGf}WO%~6ERIk8wuw}za#E>K z=~;Cd7F}yGE(f;n43NB*zrMry+D7S_u!2&{)9(lW-xY2M1oGFsdgjvv;)o>WO}C(;K?|``)(g}5rq-40m*3-( zKhfr{OjXKoNT#0yyx87j4>P;4oskD)*zE3O%WPo8#~VAoIT5c!&-sD*Ig)are7yVA zVf!g(2S~TtrbHRTre(7jg(1qFPhS-XNK*7~yK+h?#3?G++YYhSAaMFBrxPPvXw~9S zfg{uZb|4vzhU0j&K; zfrl#o1vys~(%A)xLU9vVef{xx8-ICghLi-e?%LYCNIjN#DHax%cumZfcGgN1%K*Fm z{NiN3hC_H{aR-eqJLPH{Ee&Wj6BM0B%NhR+Hm>@5TvdwYb$q;!dSCY@eY)<=1pzVO z(e5l@-EbJ_KZ99#`FdH} zdjXrZx;Lxn?}-)a0&6!<7wQIrTCag+Goi9zNQwDY&qu+I`i)!dpCH;@TIu`B8EvH}Ml{OZ@uUsMgx&5K)AYl+>a2VGhHYFICAlD>7IJ1N~!Hk-@B)mBm8JCuZODp-88 zy_FTaHH+Qbd}lj64lZ@T2W>>u7M3c1yZog1;&Sl9py$A7n79f0Vw+#2LqgoE5Owws zjfSeK08P3Wj)UT<_0DAp z`r|cFnPF`ZlHzx}EyPW0UWtHc5uHUoK(ao)Rslm2QjY7>yz}C`voq2SMWn>|cgsAK z>M-e%un7u|Un@Y7CbeT#W!sEHglvF6f_{=)D5P`bUJQvGD zhQ}e|I^YPN zpQ&!!h+R7ITH{7ooz&L-&%u~?V3+DV@nm2)j9*!SNVpW zJl{_4RBc;q%q>DwH_XHFzMrNSS;rmD%)ja}{A}~3;tSiNg2&;fLzKdqbc`2K!IoaYX%OUO#-A(VBniv7azlI%1!+)GJ{-B|@ zp^hHbhq%v1**~!n{GZt9=eNcjjYIr9daMa$e+2KdNWTW^Mc^zodb>%kTgQ9G#&-y- z<}$+hRktPZr}w-XR}^NoD6Ul_5P3RgqGDMk?E zGr1JzGjWgss1dhZjyw;Gv~(Ab5CIVfN#&w&hj1A~-~=w?%Q-^VroPX<9!xP(qi>~d zKuHNM*KQ9^58f#m zLlJKcX%T^|+zb!+ZpD@Rrlz={pt{Uvu_tk|`i-}7wXqSee^1)p>3Ri%^Tr+Bd*kJ+ zb)?dj<1|+oXQhuPg-b`o^6iyftpDoZ0ogWtB0i?#zOoDe^jcp0bO59NeCqz_jb*4U zHG)X(q0K-LIcm@Bi6;<2UX{(3BHLs^X2KJfFL6o6DI8g0vN}~%vpN4)_NQAp(y|3wACFXqk!l^!L(|@#9$zTT4-iSx=b4YuoE#zQ~dApJ< z=~SJrXBf@U#KmLo3`nubF31R>=Oj>u$vurACgDIj+PW&>#Dq?8>ZR`kMPt7dVrQ}U zKjRT%FYdD~-X$SCdNFJm%U8NJ_Z?HD1+Nw=WLK7Cm4{FlIJ5{;}Zj zgd{2s7hV(%%G&iilG}!S&U6)|D)3f3K$z}53c}0z;9-$zfS~Pj#nGsMjW5dAM+`;` z(;s`Tp|#h1Pgt0BT^eT_SvrTYm3njMr3A!;rhSRH$1ndK2Jw}Jegi{!Y4udn9<7|{YqVt?xz#_(AOtbFFHD!ssz$y zkVQO`=hQAGr_?>vt?<@3qaXS-M|(<7HF0UJvnjIPG9+Wp;bVD`y5QVO5yPWr^wlE+ zB8|34-$|Mb2V(^V-#oFaF6rJ=7|rj3C0~M^kE$}ytl#XmseJ6|vow#fz#2o%hfFP0`+7fBxNm~S#nC{BS7KipJj3~9X_jw$Y4tp_X(FV)FfZ=3hk<&=x3@!(zN^JM1bqEMTUFw*CAhPm* zUvR(>@XysotgD^1QIkWbujmkVbQD=3m&8OCgcTS7ayS~207r20J2yK&>kfGt4$SVJ zvcH;k=Q~(5vtjH7XH}EpGSnbL=RddJMAw>){_`V)zw$Sb1Ed2 y5$h1|gut<;hQ}@=HRKgK_XWn!BEatlAWk4UL 0 + \end{equation*} + % + It has the following properties: + \begin{enumerate} + \item The null space of $\matr{A}$ has the null vector only: $\text{Ker}(\matr{A}) = \{ \nullvec \}$. \\ + Which implies that $\matr{A}$ is non-singular (\Cref{th:kernel_invertible}). + \item The diagonal elements of $\matr{A}$ are all positive. + \end{enumerate} + \begin{theorem} + If the eigenvalues of a symmetric matrix $\matr{B} \in \mathbb{R}^{n \times n}$ are all positive. + Then $\matr{B}$ is positive definite. + \end{theorem} +\end{description} + + + +\subsection{Orthogonality} +\begin{description} + \item[Angle between vectors] \marginnote{Angle between vectors} + The angle $\omega$ between two vectors $\vec{x}$ and $\vec{y}$ can be obtained from: + \begin{equation*} + \cos\omega = \frac{\left\langle \vec{x}, \vec{y} \right\rangle }{\Vert \vec{x} \Vert_2 \cdot \Vert \vec{y} \Vert_2} + \end{equation*} + + \item[Orthogonal vectors] \marginnote{Orthogonal vectors} + Two vectors $\vec{x}$ and $\vec{y}$ are orthogonal ($\vec{x} \perp \vec{y}$) when: + \[ \left\langle \vec{x}, \vec{y} \right\rangle = 0 \] + + \item[Orthonormal vectors] \marginnote{Orthonormal vectors} + Two vectors $\vec{x}$ and $\vec{y}$ are orthonormal when: + \[ \vec{x} \perp \vec{y} \text{ and } \Vert \vec{x} \Vert = \Vert \vec{y} \Vert=1 \] + \begin{theorem} + The canonical basis of a vector space is orthonormal. + \end{theorem} + + \item[Orthogonal matrix] \marginnote{Orthogonal matrix} + A matrix $\matr{A} \in \mathbb{R}^{n \times n}$ is orthogonal if its columns are \underline{orthonormal} vectors. + It has the following properties: + \begin{enumerate} + \item $\matr{A}\matr{A}^T = \matr{I} = \matr{A}^T\matr{A}$, which implies $\matr{A}^{-1} = \matr{A}^T$. + \item The length of a vector is unchanged when mapped through an orthogonal matrix: + \[ \Vert \matr{A}\vec{x} \Vert^2 = \Vert \vec{x} \Vert^2 \] + \item The angle between two vectors is unchanged when both are mapped through an orthogonal matrix: + \[ + \cos\omega = \frac{(\matr{A}\vec{x})^T(\matr{A}\vec{y})}{\Vert \matr{A}\vec{x} \Vert \cdot \Vert \matr{A}\vec{y} \Vert} = + \frac{\vec{x}^T\vec{y}}{\Vert \vec{x} \Vert \cdot \Vert \vec{y} \Vert} + \] + \end{enumerate} + + \item[Orthogonal basis] \marginnote{Orthogonal basis} + Given an $n$-dimensional vector space $V$ and a basis $\beta = \{ \vec{b}_1, \dots, \vec{b}_n \}$ of $V$. + $\beta$ is an orthogonal basis if: + \[ \vec{b}_i \perp \vec{b}_j \text{ for } i \neq j \text{ (i.e.} \left\langle \vec{b}_i, \vec{b}_j \right\rangle = 0 \text{)} \] + + \item[Orthonormal basis] \marginnote{Orthonormal basis} + Given an $n$-dimensional vector space $V$ and an orthogonal basis $\beta = \{ \vec{b}_1, \dots, \vec{b}_n \}$ of $V$. + $\beta$ is an orthonormal basis if: + \[ \Vert \vec{b}_i \Vert_2 = 1 \text{ (or} \left\langle \vec{b}_i, \vec{b}_i \right\rangle = 1 \text{)} \] + + \item[Orthogonal complement] \marginnote{Orthogonal complement} + Given a $n$-dimensional vector space $V$ and a $m$-dimensional subspace $U \subseteq V$. + The orthogonal complement $U^\perp$ of $U$ is a $(n-m)$-dimensional subspace of $V$ such that it + contains all the vectors orthogonal to every vector in $U$: + \[ \forall \vec{w} \in V: \vec{w} \in U^\perp \iff (\forall \vec{u} \in U: \vec{w} \perp \vec{u}) \] + % + Note that $U \cap U^\perp = \{ \nullvec \}$ and + it is possible to represent all vectors in $V$ as a linear combination of both the basis of $U$ and $U^\perp$. + + The vector $\vec{w} \in U^\perp$ s.t. $\Vert \vec{w} \Vert = 1$ is the \textbf{normal vector} of $U$. \marginnote{Normal vector} + % + \begin{figure}[h] + \centering + \includegraphics[width=0.4\textwidth]{img/_orthogonal_complement.pdf} + \caption{Orthogonal complement of a subspace $U \subseteq \mathbb{R}^3$} + \end{figure} +\end{description} + + + +\subsection{Projections} +Projections are methods to map high-dimensional data into a lower-dimensional space +while minimizing the compression loss.\\ +\marginnote{Orthogonal projection} +Let $V$ be a vector space and $U \subseteq V$ a subspace of $V$. +A linear mapping $\pi: V \rightarrow U$ is a (orthogonal) projection if: +\[ \pi^2 = \pi \circ \pi = \pi \] +In other words, applying $\pi$ multiple times gives the same result (i.e. idempotency).\\ +$\pi$ can be expressed as a transformation matrix $\matr{P}_\pi$ such that: +\[ \matr{P}_\pi^2 = \matr{P}_\pi \] + +\subsubsection{Projection onto general subspaces} +To project a vector $\vec{x} \in \mathbb{R}^n$ into a lower-dimensional subspace $U \subseteq \mathbb{R}^n$, +it is possible to use the basis of $U$.\\ +% +Let $m = \text{dim}(U)$ be the dimension of $U$ and +$\matr{B} = (\vec{b}_1, \dots, \vec{b}_m) \in \mathbb{R}^{n \times m}$ an ordered basis of $U$. +A projection $\pi_U(\vec{x})$ represents $\vec{x}$ as a linear combination of the basis: +\[ \pi_U(\vec{x}) = \sum_{i=1}^{m} \lambda_i \vec{b}_i = \matr{B}\vec{\lambda} \] +where $\vec{\lambda} = (\lambda_1, \dots, \lambda_m)^T \in \mathbb{R}^{m}$ are the new coordinates of $\vec{x}$ +and is found by minimizing the distance between $\pi_U(\vec{x})$ and $\vec{x}$. \ No newline at end of file