\chapter{Camera calibration}

\begin{description}
    \item[World reference frame (WRF)] \marginnote{World reference frame (WRF)}
        Coordinate system $(X_W, Y_W, Z_W)$ of the real world relative to a reference point (e.g. a corner).

    \item[Camera reference frame (CRF)] \marginnote{Camera reference frame (CRF)}
        Coordinate system $(X_C, Y_C, Z_C)$ that characterizes a camera.

    \item[Image reference frame (IRF)] \marginnote{Image reference frame}
        Coordinate system $(U, V)$ of the image.
        They are obtained as a perspective projection of CRF coordinates as:
        \[ 
            u = \frac{f}{z}x_C 
            \hspace{3em} 
            v = \frac{f}{z}y_C 
        \]

    \begin{figure}[H]
        \centering
        \includegraphics[width=0.8\linewidth]{./img/_formation_system.pdf}
        \caption{Example of WRF, CRF and IRF}
    \end{figure}
\end{description}



\section{Forward imaging model}


\subsection{Image pixelization (CRF to IRF)}
\marginnote{Image pixelization}

The conversion from the camera reference frame to the image reference frame
is done in two steps:
\begin{descriptionlist}
    \item[Discetization] \marginnote{Discetization}
        Given the sizes (in mm) $\Delta u$ and $\Delta v$ of the pixels,
        it is sufficient to modify the perspective projection to map CRF coordinates into a discrete grid:
        \[ 
            u = \frac{1}{\Delta u}\frac{f}{z_C}x_C 
            \hspace{3em} 
            v = \frac{1}{\Delta v}\frac{f}{z_C}y_C 
        \]

    \item[Origin translation] \marginnote{Origin translation}
        To avoid negative pixels, the origin of the image has to be translated from the piercing point $c$ to the top-left corner.
        This is done by adding an offset $(u_0, v_0)$ to the projection (in the new system, $c = (u_0, v_0)$):
        \[ 
            u = \frac{1}{\Delta u}\frac{f}{z_C}x_C + u_0
            \hspace{3em} 
            v = \frac{1}{\Delta v}\frac{f}{z_C}y_C +v_0
        \]

    \begin{figure}[H]
        \centering
        \includegraphics[width=0.9\linewidth]{./img/_pixelization.pdf}
        \caption{Pixelization process}
    \end{figure}

    \item[Intrinsic parameters] \marginnote{Intrinsic parameters}
        Parameters needed to convert from CRF to IRF.

        By fixing $f_u = \frac{f}{\Delta u}$ and $f_v = \frac{f}{\Delta v}$, the projection can be rewritten as:
        \[ 
            u = f_u\frac{x_C}{z_C} + u_0
            \hspace{3em} 
            v = f_v\frac{y_C}{z_C} +v_0
        \]
        Therefore, there is a total of 4 parameters: $f_u$, $f_v$, $u_0$ and $v_0$.
\end{descriptionlist}


\subsection{Roto-translation (WRF to CRF)}
\marginnote{Roto-translation}

The conversion from the world reference system to the camera reference system
is done through a roto-translation wrt the optical center.

Given: 
\begin{itemize}
    \item A WRF point $\vec{M}_W = (x_W, y_W, z_W)$,
    \item A rotation matrix $\matr{R}$,
    \item A translation vector $\vec{t}$,
\end{itemize}
the coordinates $\vec{M}_C$ in CRF corresponding to $\vec{M}_W$ are given by:
\[  
    \vec{M}_C = \begin{pmatrix} x_C \\ y_C \\ z_C \end{pmatrix} =
    \matr{R}\vec{M}_W + \vec{t} =
    \begin{pmatrix}
        r_{11} & r_{12} & r_{13} \\
        r_{21} & r_{22} & r_{23} \\
        r_{31} & r_{32} & r_{33} \\
    \end{pmatrix}
    \begin{pmatrix}
        x_W \\ y_W \\ z_W
    \end{pmatrix}
    +
    \begin{pmatrix}
        t_1 \\ t_2 \\ t_3
    \end{pmatrix}
\]

\begin{remark}
    The coordinates $\vec{C}_W$ of the optical center $\vec{C}$ are obtained as:
    \[ 
        \nullvec = \matr{R}\vec{C}_W + \vec{t} 
            \iff (\nullvec - \vec{t}) = \matr{R}\vec{C}_W 
            \iff \vec{C}_W = \matr{R}^T (\nullvec - \vec{t})
            \iff \vec{C}_W = -\matr{R}^T\vec{t}
    \]
\end{remark}

\begin{description}
    \item[Extrinsic parameters] \marginnote{Extrinsic parameters}
        \phantom{}
        \begin{itemize}
            \item The rotation matrix $\matr{R}$ has 9 elements of which 3 are independent (i.e. the rotation angles around the axes). 
            \item The translation matrix $\vec{t}$ has 3 elements.
        \end{itemize}

        Therefore, there is a total of 6 parameters.
\end{description}


\begin{remark}
    It is not possible to combine the intrinsic camera model and the extrinsic roto-translation to
    create a linear model for the forward imaging model.
    \[
        u = f_u \frac{r_{11}x_W + r_{12}y_W + r_{13}z_W + t_1}{r_{31}x_W + r_{32}y_W + r_{33}z_W + t_3} + u_0
        \hspace{1.5em}
        v = f_v \frac{r_{21}x_W + r_{22}y_W + r_{23}z_W + t_2}{r_{31}x_W + r_{32}y_W + r_{33}z_W + t_3} + v_0
    \]
\end{remark}



\section{Projective space}

\begin{remark}
    In the 2D Euclidean plane $\mathbb{R}^2$, parallel lines never intersect and points at infinity cannot be represented.
    \begin{figure}[H]
        \centering
        \begin{subfigure}{0.45\linewidth}
            \centering
            \includegraphics[width=0.45\linewidth]{./img/point_infinity_example1.png}
        \end{subfigure}
        \begin{subfigure}{0.45\linewidth}
            \centering
            \includegraphics[width=0.8\linewidth]{./img/point_infinity_example2.png}
        \end{subfigure}
        \caption{Example of point at infinity}
    \end{figure}
\end{remark}

\begin{remark}
    Point at infinity is a point in space while the vanishing point is in the image plane.
\end{remark}

\begin{description}
    \item[Homogeneous coordinates] \marginnote{Homogeneous coordinates}
        Without loss of generality, consider the 2D Euclidean space $\mathbb{R}^2$.

        Given a coordinate $(u, v)$ in Euclidean space, its homogeneous coordinates have an additional dimension
        such that:
        \[ (u, v) \equiv (ku, kv, k) \,\forall k \neq 0 \]
        In other words, a 2D Euclidean point is represented by an equivalence class of 3D points.

    \item[Projective space] \marginnote{Projective space}
        Space $\mathbb{P}^n$ associated with the homogeneous coordinates of an Euclidean space $\mathbb{R}^n$.

        \begin{figure}[H]
            \centering
            \includegraphics[width=0.6\linewidth]{./img/_projective_space.pdf}
            \caption{Example of projective space $\mathbb{P}^2$}
        \end{figure}

        \begin{remark}
            $\nullvec$ is not a valid point in $\mathbb{P}^n$.
        \end{remark}

        \begin{remark}
            A projective space allows to homogeneously handle both ordinary (image) and ideal (scene) points without introducing additional complexity.
        \end{remark}

    \item[Point at infinity] \marginnote{Point at infinity}
        Given the parametric equation of a 2D line defined as:
        \[ 
            \vec{m} = \vec{m}_0 + \lambda \vec{d} = 
            \begin{pmatrix} u_0 \\ v_0 \end{pmatrix} + \lambda \begin{pmatrix} a \\ b \end{pmatrix} =
            \begin{pmatrix} u_0 + \lambda a \\ v_0 + \lambda b \end{pmatrix}
        \]
        It is possible to define a generic point in the projective space along the line $m$ as:
        \[ 
            \tilde{\vec{m}} \equiv 
            \begin{pmatrix} \vec{m} \\ 1 \end{pmatrix} \equiv
            \begin{pmatrix} u_0 + \lambda a \\ v_0 + \lambda b \\ 1 \end{pmatrix} \equiv
            \begin{pmatrix} \frac{u_0}{\lambda} + a \\ \frac{v_0}{\lambda} + b \\ \frac{1}{\lambda} \end{pmatrix}
        \]

        The projective coordinates $\tilde{\vec{m}}_\infty$ of the point at infinity of a line $m$ is given by:
        \[ \tilde{\vec{m}}_\infty = \lim_{\lambda \rightarrow \infty} \tilde{\vec{m}} \equiv \begin{pmatrix} a \\ b \\ 0 \end{pmatrix} \]

        \begin{figure}[H]
            \centering
            \includegraphics[width=0.6\linewidth]{./img/_projective_point_inifinity.pdf}
            \caption{Example of infinity point in $\mathbb{P}^2$}
        \end{figure}

        In 3D, the definition is trivially extended as:
        \[ 
            \tilde{\vec{M}}_\infty = 
            \lim_{\lambda \rightarrow \infty} \begin{pmatrix} \frac{x_0}{\lambda} + a \\ \frac{y_0}{\lambda} + b \\ \frac{z_0}{\lambda} + c \\ \frac{1}{\lambda} \end{pmatrix} \equiv
            \begin{pmatrix} a \\ b \\ c \\ 0 \end{pmatrix}
        \]

    \item[Perspective projection] \marginnote{Perspective projection in projective space}
        Given a point $\vec{M}_C = (x_C, y_C, z_C)$ in the CRF and its corresponding point $\vec{m} = (u, v)$ in the image,
        the non-linear perspective projection in Euclidean space can be done linearly in the projective space as:
        \[  
            \begin{split}
                \tilde{\vec{m}} &\equiv 
                \begin{pmatrix} u \\ v \\ 1 \end{pmatrix} \equiv
                \begin{pmatrix} f_u\frac{x_C}{z_C} + u_0 \\ f_v\frac{y_C}{z_C} +v_0 \\ 1 \end{pmatrix} \equiv
                z_C \begin{pmatrix} f_u\frac{x_C}{z_C} + u_0 \\ f_v\frac{y_C}{z_C} +v_0 \\ 1 \end{pmatrix} \\
                &\equiv \begin{pmatrix} f_u x_C + z_C u_0 \\ f_v y_C + z_C v_0 \\ z_C \end{pmatrix} \equiv
                \begin{pmatrix} f_u & 0 & u_0 & 0 \\ 0 & f_v & v_0 & 0 \\ 0 & 0 & 1 & 0 \end{pmatrix} \begin{pmatrix} x_C \\ y_C \\ z_C \\ 1 \end{pmatrix} \equiv
                \matr{P}_\text{int} \tilde{\vec{M}}_C
            \end{split}
        \]

        \begin{remark}
            The equation can be written to take account of the arbitrary scale factor $k$ as:
            \[ k\tilde{\vec{m}} = \matr{P}_\text{int} \tilde{\vec{M}}_C \]
            or, if $k$ is omitted, as:
            \[ \tilde{\vec{m}} \approx \matr{P}_\text{int} \tilde{\vec{M}}_C \]
        \end{remark}

        \begin{remark}
            In projective space, we can also project in Euclidean space the point at infinity of parallel 3D lines in CRF with direction $(a, b, c)$:
            \[ 
                \tilde{\vec{m}}_\infty \equiv 
                    \matr{P}_\text{int} \begin{pmatrix} a \\ b \\ c \\ 0 \end{pmatrix} \equiv
                    \begin{pmatrix} f_u & 0 & u_0 & 0 \\ 0 & f_v & v_0 & 0 \\ 0 & 0 & 1 & 0 \end{pmatrix} \begin{pmatrix} a \\ b \\ c \\ 0 \end{pmatrix} \equiv
                    \begin{pmatrix} f_u a + c u_0 \\ f_v b + c v_0 \\ c \end{pmatrix} \equiv
                    c\begin{pmatrix} f_u \frac{a}{c} + u_0 \\ f_v \frac{b}{c} + v_0 \\ 1 \end{pmatrix}
            \]
            Therefore, the Euclidean coordinates are:
            \[ \vec{m}_\infty = \begin{pmatrix} f_u \frac{a}{c} + u_0 \\ f_v \frac{b}{c} + v_0 \end{pmatrix} \]
            
            Note that this is not possible when $c = 0$.
        \end{remark}
\end{description}