\documentclass[12pt]{article}
\usepackage{fancyhdr,fancybox,amssymb,epsfig,amsmath,latexsym,bm}
\usepackage{enumerate}
\usepackage{tikz}
\usepackage[all]{xy}
\usetikzlibrary{arrows}
\usetikzlibrary{decorations.markings}

\tikzset{big arrow/.style={decoration={markings,mark=at position 0.6
      with {\arrow[scale=2]{>}}}, postaction={decorate}, shorten >=1pt}}

% Code to make an augmented matrix (Usage: \begin{bmatrix}[ccc|c])
\makeatletter
\renewcommand*\env@matrix[1][*\c@MaxMatrixCols c]{%
  \hskip -\arraycolsep
  \let\@ifnextchar\new@ifnextchar
  \array{#1}}
\makeatother

\topmargin=-.5in
\headsep=0.2in
\oddsidemargin=0in
\textwidth=6.7in
\textheight=9.2in
\footskip=.5in

% Change this line to \def\bold{\mathbf} for ``upright boldface.''
\def\bold{\bm}

\newcommand{\F}{\mathbb{F}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\Z}{\mathbb{Z}}
\def\0{\bold{0}}
\def\A{\bold{A}}
\def\B{\bold{B}}
\def\C{\bold{C}}
\def\I{\bold{I}}
\def\M{\bold{M}}
\def\P{\bold{P}}
\def\Q{\bold{Q}}
\def\a{\bold{a}}
\def\b{\bold{b}}
\def\c{\bold{c}}
\def\e{\bold{e}}
\def\p{\bold{p}}
\def\q{\bold{q}}
\def\t{\bold{t}}
\def\u{\bold{u}}
\def\v{\bold{v}}
\def\x{\bold{x}}
\def\<{\langle}
\def\>{\rangle}
\def\hat{\widehat}

\DeclareMathOperator{\Span}{Span}
\DeclareMathOperator{\rank}{rank}
\newcommand{\vv}[2]{\begin{bmatrix} #1 \\ #2 \end{bmatrix}}
\newcommand{\vvv}[3]{\begin{bmatrix} #1 \\ #2 \\ #3 \end{bmatrix}}

\newcommand{\ds}{\displaystyle}

\renewcommand{\footrulewidth}{1pt}
\newenvironment{proof}[1][Proof.]{\smallskip\begin{trivlist}
    \item[\hskip \labelsep {\sffamily #1}]}{\qed\end{trivlist}\bigskip}
\newenvironment{sol}[1][Solution.]{\smallskip\begin{trivlist}
    \item[\hskip \labelsep {\sffamily #1}]}{\qed\end{trivlist}\bigskip}
\newenvironment{fminipage}
{\setlength{\fboxsep}{15pt}\begin{Sbox}\begin{minipage}}
{\end{minipage}\end{Sbox}\fbox{\TheSbox}}
\newcommand{\qed}{\hfill \ensuremath{\Box}}
\newcommand{\disp}{\displaystyle}


\pagestyle{fancy} \lhead{{\sf Homework 9 $|$ Due October 26 (Friday) }}
\rhead{\thepage} \cfoot{{\sf MthSc 311 $|$ Linear Algebra $|$ Fall 2012 
    $|$ M.~Macauley}}


\begin{document}

$\;$

\noindent\emph{Read}: Strang, Section 4.3, 4.4. \\

\noindent\emph{Suggested short conceptual exercises}: Strang, Section
4.3, \#12--16, 25, 26, 29. Section 4.4, \#3, 4, 8, 9, 19.


\begin{enumerate}

  % Strang: Section 4.3, strong modification of the problem sequence #1-11.
\item For this problem, consider the four data points
  $(t_i,b_i)=(0,0)$, $(1,8)$, $(3,8)$, and $(4,20)$. Let
  $\t=(0,1,3,4)$ be the vector of inputs and $\b=(0,8,8,20)$ the
  vector of outputs. Feel free to use a computer to solve any systems
  of equations you encounter throughout this problem.

  \begin{enumerate}
    % See #1
  \item If there were a straight line $b=C+Dt$ through these four
    points, then a certain equation $\A\x=\b$ would have a solution,
    where $\x=(C,D)$. Write this equation in matrix form (that is,
    find $\A$).
    
    % See #1
  \item Instead, we wish to find the ``best fit'' line, which means
    we need to solve $\A\hat{\x}=\p$, where $\p$ is the projection of
    $\b$ onto the column space of $\A$. Write down the \emph{normal
      equations} $\A^T\A\hat{\x}=\A^T\b$, where
    $\hat{\x}=(\hat{C},\hat{D})$, and solve for $\hat{\x}$.
    
    % See #3
  \item Check that $\e=\b-\p$ is orthogonal to both columns of
    $\A$. Compute $||\e||$, which is the shortest distance from $\b$
    to the column space of $\A$. Sketch a diagram of $\e$, $\b$,
    $\p$, and the orthogonal subspaces $C(\A)$ and $N(\A^T)$ to illustrate
    this.
    
  \item Plot the four data points in $\R^2$ (on the $tb$-plane) and
    sketch the best fit line through them that you just found. Clearly
    mark what the vectors $\b=(b_1,b_2,b_3,b_4)$,
    $\e=(e_1,e_2,e_3,e_4)$, and $\p=(p_1,p_2,p_3,p_4)$ represent.

    % See #4
  \item Write down $E:=||\A\x-\b||^2$ as a sum of four squares--the
    last one is $(C+4D-20)^2$, and compute $\partial E/\partial C$ and
    $\partial E/\partial D$. Set these derivatives equal to zero, and
    obtain scalars of the normal equations $\A^T\A\hat{\x}=\A^T\b$.

    % See #5
  \item The method above found the best fit degree-$1$ polynomial
    (line). Now, find the best fit degree-$0$ polynomial (horizontal
    line) $b=C$. Note that this will be a $4\times 1$ system instead
    of a $4\times 2$ system. Compute the vectors $\p$ and $\e$, and
    the (squared) error $||\e||^2$.

    % See #9
  \item Find the best fit parabola (degree-$2$ polynomial)
    $b=C+Dt+Et^2$. On a new set of axes, plot the four data points
    and this parabola. Compute the vectors $\p$ and $\e$, and
    the (squared) error $||\e||^2$.

    % See #10
  \item Find the best fit cubic (degree-$3$ polynomial)
    $b=C+Dt+Et^2+Ft^3$. On a new set of axes, plot the four data
    points and this cubic. Compute the vectors $\p$ and $\e$, and the
    (squared) error $||\e||^2$.
  \end{enumerate}



  % Strang: Section 4.4, #10 (modified)
\item In this problem we will prove that orthonormal vectors are
  linearly independent two different ways.
  \begin{enumerate}
    \item Vector proof: First, suppose that
      $c_1\q_1+c_2\q_2+\cdots+c_k\q_k=\0$. Show that each
      $c_i=0$. [\emph{Hint}: Start by multipling both sides of the
      equation by $\q_i^T$.]
    \item Matrix proof: Let $\Q$ be the matrix whose columns are the
      $\q_i$'s. Show that if $\Q\x=\0$, then $\x=\0$. [\emph{Hint}:
      Since $\Q$ need not be square, you cannot assume $\Q^{-1}$
      exists, but $\Q^T$ of course will.]
  \end{enumerate}


\newpage


  % Strang: Section 4.4, #6, #13 (added the 2nd and 4th part myself)
\item For each of the following, answer either \emph{true} (with a reason) or
  \emph{false} (with a counterexample).
  \begin{enumerate}
  \item If $\Q$ is an orthogonal matrix, then $\Q^{-1}$ is orthogonal. 
  \item If $\Q$ is an orthogonal matrix, then $\Q^T$ is orthogonal. 
  \item If $\Q_1$ and $\Q_2$ are orthogonal matrices, then $\Q_1\Q_2$
    is orthogonal.
  \item If $\Q$ is a matrix with orthonormal columns (need not be square),
    then $||\Q\x||=||\x||$ for every $\x$. 
  \end{enumerate}



  % Strang: Section 4.4, #13 (modified)
\item What multiple of $\A=\vv{1}{1}$ should be subtracted from
  $\b=\vv{6}{0}$ to make the resulting vector $\B$ orthogonal to $\a$?
  Sketch a figure showing $\A$, $\b$, and $\B$. Then normalize $\A$
  and $\B$ to get an orthonormal set, $\q_1$ and $\q_2$.



  % Strang: Section 4.4, #18 (modified)
\item Let $\a$, $\b$, and $\c$ be the (independent) column vectors of
  the matrix
  \[
  \M=\begin{bmatrix}1&2&4\\0&0&5\\0&3&6\end{bmatrix}\,.
  \]
  Use the Gram-Schmidt process to produce an orthonormal basis $\q_1$,
  $\q_2$, and $\q_3$. Then write $\M=\Q\bold{R}$, where $\Q$ is orthogonal
  and $\bold{R}$ is upper-triangular.



  % Strang: Section 4.4, #32, 34 (modified)
\item Recall that if $||\u||=1$, then the rank-$1$ matrix $\u\u^T$ is
  the projection matrix onto $\u$. In this case, $\Q=\I-2\u\u^T$ is a
  \emph{reflection matrix}.
  \begin{enumerate}
    \item Reflecting twice across the same axis is the identity. Verify
      that indeed, $\Q^2=\I$. 
    \item Compute $\Q\u$, and simplify this expression as much as
      possible.
    \item Suppose $\v$ is orthogonal to $\u$. Compute $\Q\v$, and
      simplify as much as possible.
    \item Describe in plain English which subspace $\Q$ is reflecting
      across. Your answer should involve $\u$. Include a sketch. 
    \item Compute the reflection matrix $\Q_1=\I-2\u_1\u_1^T$ where
      $\u_1=(0,1)$. Compute $\Q_1\x_1$, where $\x_1=(a,b)$, and sketch
      the vectors $\u_1$, $\x_1$, and $\Q_1\x_1$ in the plane.
    \item Compute the reflection matrix $\Q_2=\I-2\u_2\u_2^T$ where
      $\u_2=(0,\sqrt{2}/2,\sqrt{2}/2)$. Compute $\Q_2\x_2$, where
      $\x_2=(1,1,1)$, and sketch the vectors $\u_2$, $\x_2$, and
      $\Q_2\x_2$ in $\R^3$.
  \end{enumerate}
    
\end{enumerate}

\end{document}
