\documentclass[twoside]{article}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}

\theoremstyle{definition}
\newtheorem{thm}{Theorem}[section]
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{cor}[thm]{Corollary}

\theoremstyle{definition}
\newtheorem*{defn}{Definition}
\newtheorem*{exmp}{Example}

\theoremstyle{remark}
\newtheorem*{rem}{Remark}
\newtheorem*{note}{Note}
\newtheorem*{exer}{Exercise}

\newtheoremstyle{citing}% name
  {3pt}%      Space above, empty = `usual value'
  {3pt}%      Space below
  {}% Body font
  {}%         Indent amount (empty = no indent, \parindent = para indent)
  {\bfseries}% Thm head font
  {.}%        Punctuation after thm head
  {.5em}%     Space after thm head: " " = normal interword space;
        %       \newline = linebreak
  {\thmnote{#3}}% Thm head spec

\theoremstyle{citing}
\newtheorem*{varthm}{}

\newcommand{\boxedeqn}[1]{%
\[\fbox{%
\addtolength{\linewidth}{-2\fboxsep}%
\addtolength{\linewidth}{-2\fboxrule}%
\begin{minipage}{\linewidth}%
\begin{equation}#1\end{equation}%
\end{minipage}%
}\]%
}

\setlength{\oddsidemargin}{0.25 in}
\setlength{\evensidemargin}{-0.25 in}
\setlength{\topmargin}{-0.6 in}
\setlength{\textwidth}{6.5 in}
\setlength{\textheight}{8.5 in}
\setlength{\headsep}{0.75 in}
\setlength{\parindent}{0 in}
\setlength{\parskip}{0.1 in}

\newcommand{\lecture}[4]{
   \pagestyle{myheadings}
   \thispagestyle{plain}
   \newpage
   \setcounter{page}{1}
   \setcounter{section}{#4}
   \noindent
   \begin{center}
   \framebox{
      \vbox{\vspace{2mm}
    \hbox to 6.28in { {\bf Stat205A:~Probability~Theory~(Fall 2002) \hfill Lecture: #4} }
       \vspace{6mm}
       \hbox to 6.28in { {\Large \hfill #1  \hfill} }
       \vspace{6mm}
       \hbox to 6.28in { {\it Lecturer: #2 \hfill Scribe: #3} }
      \vspace{2mm}}
   }
   \end{center}
   \markboth{#1}{#1}
   \vspace*{4mm}
}

\begin{document}

\lecture{Basic ${\cal L}^2$ Convergence Theorem and Kolmogorov's Law of Large Numbers }{James W. Pitman}{Vinod Prabhakaran {\tt vinodmp@eecs.berkeley.edu } }{8}

\begin{thm}[Basic ${\cal L}^2$ Convergence Theorem] \cite[p. 63, (8.3)]{durrett95}
Let $X_1\;X_2,\;\ldots$ be independent random variables with $E(X_i)=0$ and
$E(X_i^2)=\sigma_i^2<\infty$, $i=1,2,\ldots$, and $S_n=X_1+X_2+\cdots+X_n$.
If $\sum_{i=1}^{\infty}\sigma_i^2<\infty$, then $S_n$ converges a.s. and in
${\cal L}^2$ to some $S_\infty$ with $E(S_\infty^2)=\sum_{i=1}^{\infty}
\sigma_i^2$\end{thm}
\begin{proof} 
First note that ${\cal L}^2$ convergence and existence of
$S_\infty$ is implied by orthogonality of $X_i$'s: $E(X_iX_j)=0,\;i\neq j$
\begin{gather*}
E(S_n^2)=\sum_{i=1}^{n} \sigma_i^2\\
E((S_n-S_m)^2)=\sum_{i=m+1}^{n}\sigma_i^2\;\rightarrow\; 0 \mbox{ as }
m,n\;\rightarrow\;\infty
\end{gather*}
$\therefore$ $S_n$ is Cauchy in ${\cal L}^2$.  Since ${\cal L}^2$ is complete,
there is a unique $S_\infty$ (up to a.s. equivalence) such that
$S_n\;\rightarrow\;S_\infty$ in ${\cal L}^2$. 

Turning to a.s convergence, the method is to show the sequence $(S_n)$ is 
a.s. Cauchy.  The limit of $S_n$ then exists a.s. by completeness of the 
set of real numbers.
The same argument applies more generally to martingale differences 
$X_i$ \cite[p. 252 (4.5) for $p =2$]{durrett95}.
Note that this method gives $S_\infty$ more explicitly,
and does not appeal to completeness of ${\cal L}^2$.

Recall that $S_n$ is Cauchy a.s. means $M_n:=\sup_{p,q\geq n} |S_p-S_q|\;
\rightarrow\; 0$ a.s.  Note that $0\leq M_n(\omega)\downarrow$ implies that
$M_n(\omega)$ converges to a limit in $[0,\infty]$. So, if $P(M_n>\epsilon)\; \rightarrow\;
0,\;\forall\;\epsilon>0$, then $M_n\downarrow 0$ a.s.

Let $M_n^\ast:=\sup_{p\geq n}|S_p-S_n|$. Since by triangle inequality, \[
|S_p-S_q|\leq|S_p-S_n|+|S_q-S_n|\; \Rightarrow\; M_n^\ast\leq M_n\leq
2M_n^\ast \] it is sufficient to show that $M_n^\ast \;
\stackrel{P}{\rightarrow} \; 0$

For all $\epsilon>0$,
\begin{align*}
P\left(\sup_{p\geq n}|S_p-S_n|>\epsilon\right)& = \lim_{N\rightarrow\infty}P\left(\max_{n\leq p\leq N}|S_p-S_n|>\epsilon\right)\\
                                   & \leq \lim_{N\rightarrow\infty} \sum_{i=n+1}^{N} \frac{\sigma_i^2}{\epsilon^2} = \sum_{i=n+1}^{\infty} \frac{\sigma_i^2}{\epsilon^2}
\end{align*}
where the inequality is Kolmogorov's \cite[p. 62, (8.2)]{durrett95}.\\ Since
$\sum_{i=1}^{\infty} \sigma_i^2 < \infty$, \[ \lim_{n\rightarrow\infty}
P\left(\sup_{p\leq n}|S_p-S_n|>\epsilon\right)=0 \qedhere\]
\end{proof}

{\em Remark}:
Just orthogonality rather than independence of the $X_i$ is not enough to 
get an a.s. limit. Counter examples are hard \cite{alexitz61}. According 
to a classical results of Rademacher-Menchoff \cite[Theorems 2.3.2 and 2.3.3]{stout74}, for orthogonal $X_i$  the condition
$\sum_i (\log ^2 i ) \sigma_i^2 < \infty$ is enough for a.s. convergence of
$S_n$, whereas if $b_i \uparrow$ with $b_i = o(\log^2 i)$ there
exist orthogonal $X_i$ such that $\sum_i b_i \sigma_i^2 < \infty$ and
$S_n$ diverges almost surely.

An easy consequence of the Basic ${\cal L}^2$ Convergence Theorem 
is the sufficiency part of Kolmogorov's three-series 
theorem:
\begin{thm}[Kolmogorov] \cite[p.64, (8.4)]{durrett95}
Let $X_1,X_2,\ldots$ be independent. Fix $b>0$. Convergence of the following
three series
\begin{itemize}
\item $\sum_n P(|X_n|>b)<\infty$
\item $\sum_n E(X_n1_{(|X_n|<b}))$ converges to a finite limit
\item $\sum_n \mbox{var}(X_n1_{(|X_n|<b}))<\infty$
\end{itemize}
is equivalent to $P(\sum_n X_n \mbox{ converges to a finite limit})=1$
\end{thm}
\begin{note}If any one of the three series diverges then
\[P\left(\sum_n X_n \mbox{ converges to a finite limit}\right)=0\]
by Kolmogorov's zero-one law \cite[p. 62, (8.1)]{durrett95}.
Note also that if one or more of the series diverges for
some $b$, then one or more of the series must diverge for every $b$, 
but exactly which of the three series diverge may depend on $b$.
Examples can be given of $8$ possible combinations of convergence/divergence.
\end{note}
\begin{proof}[Proof of sufficiency]
That is, convergence of all 3 series implies $\sum_n X_n$ converges a.s..
Let $X'_n=X_n1_{(|X_n| \le b)}$. Since $\sum_n P(X'_n\neq X_n)=\sum_n
P(|X_n|>b)<\infty$, Borel-Cantelli lemma gives $P(X'_n\neq X_n \mbox{
i.o.})=0$ which implies $P(X'_n=X_n \mbox{ ev.})=1$.  Also if
$X'_n(\omega)=X_n(\omega)$ ev., then $\sum_nX_n(\omega)$ converges
$\Leftrightarrow$ $\sum_nX'_n(\omega)$ converges.

$\therefore$ it is enough to show that
\[P\left(\sum_n X'_n \mbox{ converges to a finite limit}\right)=1\]

Now \[\sum_{n=1}^{N} X'_n = \sum_{n=1}^N (X'_n-E(X'_n))\;+\;\sum_{n=1}^N E(X'_n).\]
$\sum_{n=1}^N E(X'_n)$ has a limit as $N\rightarrow\infty$ by hypothesis,
and \[\sum_{n=1}^\infty E((X'_n-E(X'_n))^2)=\sum_{n=1}^\infty
\mbox{var}(X'_n)<\infty \] implies that $\sum_{n=1}^\infty(X'_n-E(X'_n))$
converges a.s. by the basic ${\cal L}^2$ convergence theorem.
For proof of the converse, see \cite[p. 118, Example 4.7]{durrett95}.
\end{proof}

Recall {\em Kronecker's lemma} \cite[p. 64, (8.5)]{durrett95}: If $a_n\uparrow\infty$ and $\sum_{n=1}^{\infty}
{X_n}/{a_n}$ converges a.s., then $(\sum_{m=1}^n X_m)/{a_n}\rightarrow0$
a.s.

Let $X_1,X_2,\ldots$ be independent with mean 0 and
$S_n=X_1+X_2+\cdots+X_n$. If $\sum_{n=1}^{\infty}
{E(X_n^2)}/{a_n^2}<\infty$, then by basic ${\cal L}^2$ convergence theorem,
$\sum_{n=1}^{\infty} {X_n}/{a_n}$ converges a.s. Then ${S_n}/{a_n}\rightarrow0$ a.s.

\begin{exmp}
Let $X_1,X_2,\ldots$ be i.i.d., $E(X_i)=0$, and $E(X_i^2)=\sigma^2<\infty$.\\
Take $a_n=n$,
\[\sum_{n=1}^{\infty}\frac{\sigma^2}{n^2}<\infty\;\Rightarrow\;\frac{S_n}{n}\stackrel{a.s.}{\rightarrow}0\]
Now take $a_n=n^{\frac{1}{2}+\epsilon}$, $\epsilon>0$
\[\sum_{n=1}^{\infty}\frac{\sigma^2}{n^{1+2\epsilon}}<\infty\;\Rightarrow\;\frac{S_n}{n^{\frac{1}{2}+\epsilon}}\stackrel{a.s.}{\rightarrow}0\]
\end{exmp}

The definitive result of this kind is the {\em Law of the iterated logarithm}
\cite[p. 434]{durrett95}.

\begin{thm}[Kolmogorov's Law of Large Numbers]
Let $X,X_1,X_2,\ldots$ be i.i.d. with $E(|X|)<\infty$. Let
$S_n=X_1+X_2+\cdots+X_n$, then ${S_n}/{n}\rightarrow E(X)$ a.s. as
$n\rightarrow\infty$
\end{thm}
{\em Note.} The theorem is true with just pairwise independence instead
of the full independence assumed here \cite[p. 56 (7.1)]{durrett95}.
The theorem also has an important generalization
to stationary sequences (The Ergodic Theorem \cite[p. 341]{durrett95}).


\begin{proof}
{\em Step 1}: Without loss of generality, we can assume $E(X)=0$.

{\em Step 2}: {\em Truncated variables}\\
Define\[\hat{X}_n:=X_n1_{(|X_n|\leq n)}\]
Note that $\hat{X}_n$ are independent.
Define their centered versions $\tilde{X}_n:=\hat{X}_n-E(\hat{X}_n)$\\
{\em Plan}: We will show that\[ (\frac{S_n}{n}\rightarrow0)
\stackrel{a.s.}{\stackrel{=}{\mbox{\tiny (a)}}}
(\frac{\hat{S}_n}{n}\rightarrow0) \stackrel{a.s.}{\stackrel{=}{\mbox{\tiny
(b)}}} (\frac{\tilde{S}_n}{n}\rightarrow0),\] where
$\hat{S}_n=\hat{X}_1+\hat{X}_2+\cdots+\hat{X}_n$ and
$\tilde{S}_n=\tilde{X}_1+\tilde{X}_2+\cdots+\tilde{X}_n$.\\
Then using Kronecker's lemma we will show that $P\left({\tilde{S}_n}/{n}
\rightarrow0\right)=1$.

{\em (a)} $P(X_n=\hat{X}_n\;ev.)=1$ because $P(X_n\neq\hat{X}_n\;i.o.)=0$
which follows from
\[\sum_{n=1}^{\infty}P(X_n\neq\hat{X}_n)=\sum_{n=1}^{\infty}P(|X_n|>n)=\sum_{n=1}^{\infty}P(|X|>n)\leq E(|X|)<\infty\] by Borel-Cantelli lemma.
So, $S_n$ and $\hat{S}_n$ differ only at a finite number of terms.
\[\therefore\; (\frac{S_n}{n}\rightarrow0) \stackrel{a.s.}{=}(\frac{\hat{S}_n}{n}\rightarrow0)\]

{\em (b)}
\[\frac{\tilde{S}_n-\hat{S}_n}{n}=\frac{1}{n}\sum_{m=1}^{n}E(\hat{X}_m)\rightarrow0\]
since
\[E(\hat{X}_n)=E(X1_{(|X|\leq n)})\rightarrow E(X)=0\]
by dominated convergenece theorem. (Dominate by $|X|$ and note $E(|X|)<\infty$.)

To finish, by Kronecker's lemma and basic ${\cal L}^2$ convergence
theorem, it is enough to show that
\[\sum_{n=1}^{\infty}\frac{E(\tilde{X}_n^2)}{n^2}<\infty.\]
\[E(\tilde{X}_n^2)=\mbox{var}(\hat{X}_n)\leq E(\hat{X}_n^2)=E(X^21_{(|X|\leq n)})\]
But, (a fact about real numbers)\footnote{This fact can be shown roughly as
follows \[\sum_{n=1}^{\infty}\frac{x^21_{(|x|\leq n)}}{n^2}\approxeq
x^2\sum_{n=|x|}^\infty\frac{1}{n^2}\approxeq
x^2\frac{1}{|x|}\approxeq|x|\].}
\[\sum_{n=1}^{\infty}\frac{X^21_{(|X|\leq n)}}{n^2}\leq2|X|\]
Take expectations to complete the proof.
\end{proof}
\bibliographystyle{plain}
\bibliography{/saruman/accounts/fac/pitman/search/bm3,/saruman/accounts/fac/pitman/search/general,/saruman/accounts/fac/pitman/search/bm4,/saruman/accounts/fac/pitman/search/bessel,/saruman/accounts/fac/pitman/search/sizebias,/saruman/accounts/fac/pitman/search/pitman,/saruman/accounts/fac/pitman/search/comb,/saruman/accounts/fac/pitman/search/species,/saruman/accounts/fac/aldous/trees/trees,/saruman/accounts/fac/aldous/trees/rwgbook,/saruman/accounts/fac/aldous/trees/misc,/saruman/accounts/fac/aldous/trees/me,/saruman/accounts/fac/aldous/trees/coag}
\end{document}