LINMA2380: Update exercise solutions for 2020--2021

src/q7/matrix-INMA2380/exercises/ch3.tex

@@ -1,187 +1,229 @@
-\section{Eigenvalues, eigenvectors and similitude}
-\exo{1}
-\begin{solution}
-  If $A \Ima(x) \subseteq \Ima(x)$, since $\Ima(x)$ and $A\Ima(x)$
-  are vector subspaces and $\Ima(x)$ is of dimension 1,
-  that means that either $A\Ima(x) = \Ima(x)$ or $A\Ima(x) = \{0\}$.
-  In the first case, it is an eigenvector of a nonzero eigenvalue.
-  In the second case, it is an eigenvector of 0.
-\end{solution}
+\section{Unitary transformations and the Singular Value Decomposition}
 
 \exo{2}
-\begin{solution}
-  Let's first show that if $A$ is normal,
-  it is still normal after an unitary transformation.
-
-  \begin{align*}
-    (U^TAU)^T(U^TAU)
-    & = U^TA^TAU\\
-    & = U^TAA^TU\\
-    & = U^TAUU^TA^TU\\
-    & = U^TAU(U^TAU)^T.
-  \end{align*}
-  That means that the Schur form of $A$ is normal too.
+Assuming that all the diagonal elements of \(\Lambda\) are distinct, show that the matrix \(U_{up}\) is necessarily diagonal and consists only of phases:
+\[
+U_{up} = \diag\{e^{i\psi_1}, \dots, e^{i \psi_n}\}.
+\]
 
-  We can now use the same argument as in theorem~3.3 since $A_s^* = A_s^T$
-  and apply it for all $n_1+n_2 = n$ such that $n_1$ is at the end of a block.
+\begin{solution}
+	Let $U_{up} = [u_{ij}]_{i,j=1}^{n}$.
+	We must have
+	\[ U_{up}\Lambda = \Lambda U_{up}. \]
+	At the position $i,j$, we have
+	\[ u_{ij} \lambda_j = \lambda_i u_{ij} \]
+	or
+	\[ u_{ij} (\lambda_j - \lambda_i) = 0. \]
+	If $i \neq j$, this means that $u_{ij} = 0$ since $\lambda_j - \lambda_i \neq 0$.
+
+	$U_{up}$ must therefore be diagonal.
+	However it also needs to be hermitian for $UU_{up}$ to be Hermitian.
+	Indeed, we need to have
+	\begin{align*}
+	UU_{up} U_{up}^*U^* & = I\\
+	U_{up} U_{up}^* & = U^*U\\
+	U_{up} U_{up}^* & = I.
+	\end{align*}
+	This means that we need to have $1 = u_{ii}\overline{u_{ii}} = \abs{u_{ii}}^2$
+	so $u_{ii}$ is on the unit circle and is a phase.
 \end{solution}
 
 \exo{2}
+Show that if \(X\) satisfies
+\begin{itemize}
+	\item (1), then \(AX\) is a projection;
+	\item (1) and (3), then \(AX\) is an orthogonal projection;
+	\item (2), then \(XA\) is a projection;
+	\item (2) and (4), then \(XA\) is an orthogonal projection.
+\end{itemize}
+
 \begin{solution}
-  Let's first show that if $A$ is anti-symmetric,
-  it is still anti-symmetric after an unitary transformation.
+	\begin{itemize}
+		\item
+		\begin{align*}
+		AXAX
+		& = (AXA)X\\
+		& = AX.
+		\end{align*}
+		\item
+		$AX$ is a projection since (1) is satisfied.
+
+		From Theorem~3.6, $\Ker(AX) = \Ima((AX)^*)^\perp$.
+		Therefore, we can just show that $\Ima((AX)^*) = \Ima(AX)$ which is obvious
+		since (3) is satisfied.
+		\item
+		\begin{align*}
+		XAXA
+		& = (XAX)A\\
+		& = XA.
+		\end{align*}
+		\item
+		$XA$ is a projection since (2) is satisfied.
+
+		From Theorem~3.6, $\Ker(XA) = \Ima((XA)^*)^\perp$.
+		Therefore we can just show that $\Ima((XA)^*) = \Ima(XA)$ which is obvious
+		since (4) is satisfied.
+	\end{itemize}
+\end{solution}
 
-  \begin{align*}
-    (U^TAU)^T
-    & = U^TA^TU\\
-    & = U^T(-A)U\\
-    & = -U^TAU.
-  \end{align*}
-  That means that the Schur form of $A$ is anti-symmetric too.
-  Since it is anti-symmetric, $A^TA = -A^2 = AA^T$ so it is also normal.
+\exo{3}
+How can we define the notion of canonical angles between two spaces of different dimension?
 
-  By the exercise~3.2, we know that the Schur form is block diagonal.
-  But since $A_s = -A_s^T$,
-  we must have $\alpha_j = -\alpha_j$ which means that $\alpha_j = 0$
-  (note that it imposes no restriction on $\beta_j$).
-  Same for $A_{ii}$.
-\end{solution}
+% i feel like there's some fuckery going on with  matrix dimensions here :(
 
-\exo{1}
 \begin{solution}
-  We simply use the corollary since we remove the last line and column.
-
-  If the $\beta_i$ are nonzero, we can see that it is strict since
-  if $p_i(\lambda) = p_{i-1}(\lambda) = 0$, we have
-  \[ 0 = -\beta_i^2 p_{i-2}(\lambda) \]
-  so $p_{i-2}(\lambda) = 0$.
-  By induction, we have $p_0(\lambda) = 0$ which is absurd.
+	Let $\mathcal{S}_1$ be of dimension $m$ and $\mathcal{S}_2$ be of dimension $n < m$.
+	We can have $n$ canonical angles using
+	\begin{align*}
+	S_1^*S_2
+	& = U_1
+	\begin{pmatrix}
+	\Sigma\\0
+	\end{pmatrix}
+	U_2^*\\
+	& =
+	\begin{pmatrix}
+	U_{1,1} & U_{1,2}
+	\end{pmatrix}
+	\begin{pmatrix}
+	\Sigma\\0
+	\end{pmatrix}
+	U_2\\
+	& = U_{1,1} \Sigma U_2,
+	\end{align*}
+	with $\Sigma = \cos(\Theta)$, where $\theta_i$ are the angles between the $n$ vectors of $S_1U_{1,1}$
+	and the $n$ vectors of $S_2U_2$.
 \end{solution}
 
-\exo{2}
+\exo{4}
+Show that for every positive semidefinite matrix \(H\) and every unitary matrix \(Q\), we have
+\[
+\trace(HQ) \leq \trace(H).
+\]
+
 \begin{solution}
-  See syllabus.
-  You should obtain
-  \[ \sigma_k^2 - \|m_{i:}\|_2^2 \leq \hat{\sigma}_k^2 \leq \sigma_k^2 + \|m_{i:}\|_2^2 \]
-  and not
-  \[ \sigma_k^2 - \|m_{i:}\|_2^2 \leq \hat{\sigma}_k^2 \leq \sigma_k^2 \]
-  like written in the syllabus.
+	\emph{Hint:}
+	Use the property $\trace(AB) = \trace(BA)$.
+
+	Since $H$ is Hermitian, there is a unitary $U$ and diagonal $\Lambda$ such that $H = U \Lambda U^*$.
+	Let's first analyse the RHS
+	\begin{align*}
+	\trace(H) & = \trace(U \Lambda U^*)\\
+	& = \trace(\Lambda U^*U)\\
+	& = \trace(\Lambda)\\
+	& = \sum_{i=1}^n \lambda_i.
+	\end{align*}
+	For the LHS, we first need to analyse $Q$.
+	It is unitary, so its eigenvalues are on the unit circle therefore $\abs{x^* Q x} \leq 1$ for all $x$ such that $\norm{x}=1$.
+
+	Using the triangle inequality and the fact that the $\lambda_i$ are positive
+	\begin{align*}
+	\trace(HQ)
+	& = \trace(U \Lambda U^* Q)\\
+	& = \trace(\Lambda U^* Q U)\\
+	& = \sum_{i=1}^n \lambda_i u_{:i}^* Q u_{:i}\\
+	& \leq \abs{\sum_{i=1}^n \lambda_i u_{:i}^* Q u_{:i}}\\
+	& \leq \sum_{i=1}^n \abs{\lambda_i} \, \abs{u_{:i}^* Q u_{:i}}\\
+	& = \sum_{i=1}^n \lambda_i \abs{u_{:i}^* Q u_{:i}}\\
+	& \leq \sum_{i=1}^n \lambda_i.
+	\end{align*}
 \end{solution}
 
 \exo{3}
-\begin{solution}
-  See syllabus.
-\end{solution}
+How could you extend the polar decomposition to the case \(m \neq n\)?
 
-\exo{2}
 \begin{solution}
-  See syllabus.
-
-  \begin{align*}
-    \sum_{i=0}^\infty \frac{(A+B)^i}{i!}
-    & = \sum_{i=0}^\infty \frac{\sum_{k=0}^i \frac{i!}{k!(i-k)!} A^{i-k}B^k}{i!}\\
-    & = \sum_{i=0}^\infty \sum_{k=0}^i \frac{A^{i-k}}{(i-k)!} \frac{B^k}{k!}\\
-    & =
-    \left(\sum_{i=0}^\infty \frac{A^i}{i!}\right)
-    \left(\sum_{i=0}^\infty \frac{B^i}{i!}\right).
-  \end{align*}
-  because each term $A^aB^b$ is present with the term $\frac{1}{a!b!}$.
+	If $A$ is $m \times n$, then $H$ is $m \times r$ and $Q$ is $r \times n$ for some $r$.
+	We want $H$ to be positive semidefinite so we need $H$ to be square for it to be defined.
+	That means that $Q$ is $m \times n$.
+
+	The problem with the proof when $m \neq n$ is that we cannot say
+	$U \Sigma V^* = U \Sigma U^* U V^*$ since the product is not defined because $U$ is $m \times m$
+	and $V$ is $n \times n$.
+	\begin{itemize}
+		\item If $m < n$, we have
+		\begin{align*}
+		A
+		& = U
+		\begin{pmatrix}
+		\Sigma & 0
+		\end{pmatrix}
+		\begin{pmatrix}
+		V_1 & V_2
+		\end{pmatrix}^*\\
+		& = U \Sigma V_1^*\\
+		& = (U \Sigma U^*) (U V_1^*)
+		\end{align*}
+		with $V_1^*V_1 = I$ (but $V_1V_1^* \neq I$, since $I$ has full rank but $V_1$ does not have full row rank)
+		since
+		\begin{align*}
+		I
+		& =
+		\begin{pmatrix}
+		V_1 & V_2
+		\end{pmatrix}^*
+		\begin{pmatrix}
+		V_1 & V_2
+		\end{pmatrix}\\
+		& =
+		\begin{pmatrix}
+		V_1^*V_1 & V_1^*V_2\\
+		V_2^*V_1 & V_2^*V_2
+		\end{pmatrix}.
+		\end{align*}
+
+		Therefore $QQ^* = (U V_1^*) (U V_1^*)^* = I$ but $Q^*Q = (U V_1^*)^* (U V_1^*) = V_1V_1^* \neq I$.
+		\item If $m > n$, we have
+		\begin{align*}
+		A
+		& =
+		\begin{pmatrix}
+		U_1 & U_2
+		\end{pmatrix}
+		\begin{pmatrix}
+		\Sigma \\ 0
+		\end{pmatrix}
+		V^*\\
+		& = U_1 \Sigma V^*\\
+		& = (U_1 \Sigma U_1^*) (U_1 V^*)
+		\end{align*}
+		with $U_1^*U_1 = I$ but $U_1U_1^* \neq I$ like for the previous point.
+
+		This time it's $Q^*Q = I$ and $QQ^* \neq I$.
+	\end{itemize}
 \end{solution}
 
-\exo{1}
-\begin{solution}
-  We have
-
-  \begin{align*}
-    \begin{pmatrix}
-      \lambda_0 & 1      &        & \\
-                & \ddots & \ddots & \\
-                &        & \ddots & 1\\
-                &        &        & \lambda_0\\
-    \end{pmatrix}
-    & =
-    \lambda_0 I + N
-  \end{align*}
-  for
-  \[
-    N =
-    \begin{pmatrix}
-      & 1 &        & \\
-      &   & \ddots & \\
-      &   &        & 1\\
-      &   &        & \\
-    \end{pmatrix}.
-  \]
-  The rest is a simple consequence of the exercise~3.7.
-
-  It is important to note for the next page that $N^n = 0$.
-  This is indeed a consequence of the fact that $N = J - \lambda_0I$
-  where $J$ is a Jordan block of $\lambda_0$ of size $n$.
-  $J$ is therefore a matrix with only one eigenvalue $\lambda_0$
-  of algebraic multiplicity $n$ but geometric multiplicity $1$.
-  Hence the whole set $\mathbb{C}^n$ is an invariant subspace of $N$
-  which means that $(J - \lambda_0I)^n = 0$.
-
-  We can see for example for $n = 4$ that
-
-  \begin{align*}
-    N & =
-    \begin{pmatrix}
-      0 & 1 & 0 & 0\\
-      0 & 0 & 1 & 0\\
-      0 & 0 & 0 & 1\\
-      0 & 0 & 0 & 0
-    \end{pmatrix}\\
-    N^2 & =
-    \begin{pmatrix}
-      0 & 0 & 1 & 0\\
-      0 & 0 & 0 & 1\\
-      0 & 0 & 0 & 0\\
-      0 & 0 & 0 & 0
-    \end{pmatrix}\\
-    N^3 & =
-    \begin{pmatrix}
-      0 & 0 & 0 & 1\\
-      0 & 0 & 0 & 0\\
-      0 & 0 & 0 & 0\\
-      0 & 0 & 0 & 0
-    \end{pmatrix}\\
-    N^4 & =
-    \begin{pmatrix}
-      0 & 0 & 0 & 0\\
-      0 & 0 & 0 & 0\\
-      0 & 0 & 0 & 0\\
-      0 & 0 & 0 & 0
-    \end{pmatrix}
-  \end{align*}
-  We can see that $e_1$ is an eigenvector of $\lambda_0$ but
-  $e_2$, $e_3$, $e_4$ are not.
-  \begin{itemize}
-    \item $Je_2 = \lambda_0 e_2 + e_1$
-      so $(J - \lambda_0 I)e_2 = e_1$.
-    \item $(J - \lambda_0 I)e_3 = e_2$ and $(J - \lambda_0 I)^2e_3 = e_1$.
-    \item $(J - \lambda_0 I)e_4 = e_3$, $(J - \lambda_0 I)^2e_3 = e_2$
-      and $(J - \lambda_0 I)^3e_4 = e_1$.
-  \end{itemize}
-
-  Note that $J^4 \mathbb{C}^n = \mathbb{C}^n$ if $\lambda_0 \neq 0$.
-  It is not to be mistaken from $(J - \lambda_0 I)^4 \mathbb{C}^n = \{0\}$.
-\end{solution}
+\exo{3}
+Show that the polar decomposition of \(B^\top A\) leads to an optimal rotation \(Q\) that minimizes \(\norm{AQ^\top - B}_F^2\).
 
-\exo{1}
 \begin{solution}
-  \begin{align*}
-    \fdif{}{t}e^{At}
-    & = \fdif{I}{t} + \sum_{i=1}^\infty \frac{A^i}{i!} \fdif{t^i}{t}\\
-    & = \sum_{i=1}^\infty \frac{iA^it^{i-1}}{i!}\\
-    & = A\sum_{i=1}^\infty \frac{(At)^{i-1}}{(i-1)!}\\
-    & = A \sum_{i=0}^\infty \frac{(At)^i}{i!} = A e^{At}\\
-    & = \sum_{i=0}^\infty \frac{A^{i+1} t^i}{i!}\\
-    & = \sum_{i=0}^\infty \frac{(At)^i}{i!} A = e^{At} A.
-  \end{align*}
+	\emph{Hint} Use the property $\norm{A}_F = \trace(A^\top A) = \trace(AA^\top)$ and the property $\trace(AB) = \trace(BA)$.
+
+	We have (also using $\trace(A^\top) = \trace(A)$).
+
+	\begin{align*}
+	\norm{AQ^\top - B}_F^2
+	& = \trace((AQ^\top - B)(AQ^\top - B)^\top)\\
+	& = \trace((AQ^\top - B)(QA^\top - B^\top))\\
+	& = \trace(AQ^\top QA^\top) + \trace(BB^\top) - \trace(AQ^\top B^\top ) - \trace(BQA^\top)\\
+	& = \trace(AA^\top) + \trace(BB^\top) - \trace(B^\top AQ^\top) - \trace(QA^\top B)\\
+	& = \norm{A}_F^2 + \norm{B}_F^2 - \trace(B^\top AQ^\top) - \trace(B^\top AQ^\top)\\
+	& = \norm{A}_F^2 + \norm{B}_F^2 - 2\trace(B^\top AQ^\top)\\
+	& = \norm{A}_F^2 + \norm{B}_F^2 - 2\trace(\tilde{H}\tilde{Q} Q^\top)\\
+	& \geq \norm{A}_F^2 + \norm{B}_F^2 - 2\trace(\tilde{H})
+	\end{align*}
+	using Exercise~3.4 since $\tilde{Q}Q^\top$ is unitary and $\tilde{H}$ is Hermitian positive semidefinite.
+	Taking $Q = \tilde{Q}$, we have equality. It is therefore the optimal rotation.
 \end{solution}
 
-\exo{1}
+\exo{3}
+Construct a matrix \(B\), with \(\mathop{\mathrm{rank}}(B) \leq s\), that is different from (3.15) and reaches the same bound on \(\norm{A - B}\).
+
 \begin{solution}
-  See syllabus.
-\end{solution}
+	For $s_i \in [\sigma_i - \sigma_{s+1}(A), \sigma_i + \sigma_{s+1}(A)]$ for $i = 1, \ldots, s$,
+	and
+	\[ B = \sum_{i=1}^s \mathbf{u}_i s_i \mathbf{v}_i^\top, \]
+	we have
+	\[ A - B = \sum_{i=1}^s \mathbf{u}_i(\sigma_i - s_i)\mathbf{v}_i^\top + \sum_{i=s+1}^r \mathbf{u}_i\sigma_i \mathbf{v}_i^\top. \]
+	which also has its maximum singular value equal to $\sigma_{s+1}(A)$ by definition of the $s_i$.
+\end{solution}