\documentclass{article}

\usepackage{epsfig}
\usepackage{amsfonts}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{color}

\setlength{\oddsidemargin}{0.25 in}
\setlength{\evensidemargin}{-0.25 in}
\setlength{\topmargin}{-0.6 in}
\setlength{\textwidth}{6.5 in}
\setlength{\textheight}{8.5 in}
\setlength{\headsep}{0.75 in}
\setlength{\parindent}{0 in}
\setlength{\parskip}{0.1 in}

% Use these for theorems, lemmas, proofs, etc.
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}[theorem]{Definition}
%\newenvironment{proof}{{\bf Proof:}}{\hfill\rule{2mm}{2mm}}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{open}[theorem]{Open Problem}

\newcommand{\Prob}{\mathrm{P}}
\newcommand{\E}{\mathrm{E}}
\newcommand{\chat}{{\hat{c}}}
\newcommand{\Uhat}{{\hat{U}}}
\newcommand{\Var}{\mathrm{Var}}

\newcommand{\convas}{\stackrel{\mbox{a.s.}}{\longrightarrow}}
\newcommand{\convLp}{\stackrel{L^p}{\longrightarrow}}
\newcommand{\convp}{\stackrel{p}{\longrightarrow}}
\newcommand{\convd}{\stackrel{d}{\longrightarrow}}
\newcommand{\conv}{\longrightarrow}
\newcommand{\donotconv}{\not\longrightarrow}

\newcommand{\mrm}[1]{{\mathrm #1}}
\newcommand{\mbf}[1]{{\mathbf #1}}
\newcommand{\mcal}[1]{{\mathcal #1}}
\newcommand{\bful}[1]{\underline{\textbf{#1}}}

\def\qedsymbol{\ifmmode\Box\else$\Box$\fi}

\newcommand{\lecture}[3]{
   \pagestyle{myheadings}
   \thispagestyle{plain}
   \newpage
   \setcounter{page}{1}
   \noindent
   \begin{center}
   \framebox{
      \vbox{\vspace{2mm}
    \hbox to 6.28in { {\bf Stat 205A \hfill September, 12, 2002} }
       \vspace{6mm}
       \hbox to 6.28in { {\Large \hfill #1  \hfill} }
       \vspace{6mm}
       \hbox to 6.28in { {\it Lecturer: #2 \hfill Scribes: #3} }
      \vspace{2mm}}
   }
   \end{center}
   \markboth{#1}{#1}
   \vspace*{4mm}
}

\begin{document}

\lecture{Convergence of random variables, and the Borel-Cantelli lemmas}{James W. Pitman}{Jin Kim ({\tt jin@eecs})}

\section{Convergence of random variables}

Recall that, given a sequence of random variables $X_n$,  almost sure (a.s.) convergence,  convergence in $\Prob$,  and convergence in $L^p$ space are true concepts in a sense that $X_n \to X$. In this lecture, we will define weak convergence, or convergence in distribution, $ \Prob_{X_n} \to \Prob_X $, which we write, by abuse of notation, $X_n \convd X$.

\begin{definition}[Convergence in distribution] 
We say $X_n \convd X$ if $\Prob(X_n \le x) \longrightarrow \Prob(X \le x)$ for all $x$ at which the RHS is continuous.
\end{definition}
This weak convergence appears in the central limit theorem.
\begin{theorem}
$X_n \convd X$ 
$\iff$
$\E f(X_n)  \longrightarrow \E f(X)$ for all bounded and continuous function $f$.
\end{theorem}
{\bf Proof } See Durrett. $\qedsymbol$

\begin{theorem}
The following property holds among the types of convergence. 

\setlength{\unitlength}{1cm}
\begin{picture}(10,6.5)
\put(1.5,5){\framebox(3,1){$X_n \convas X$}}
\put(6.5,5){\framebox(3,1){$X_n \convLp X$}}
\put(4,2.5){\framebox(3,1){$X_n \convp X$}}
\put(4,0){\framebox(3,1){$X_n \convd X$}}

%leftarrow
\put(4.5,3.7){\line(-1,1){1.3}}
\put(4.2,3.6){\line(-1,1){1.35}}
\put(4.45,3.5){\line(-3,1){.6}}
\put(4.45,3.5){\line(0,1){.6}}

%rightarrow
\put(6.35,3.7){\line(1,1){1.3}}
\put(6.65,3.6){\line(1,1){1.35}}
\put(6.4,3.5){\line(0,1){.6}}
\put(6.4,3.5){\line(3,1){.6}}
\put(7.5, 4){ $(\ast)$}

%centerarrow
\put(5.4, 2.5){\line(0,-1){1.4}}
\put(5.6, 2.5){\line(0,-1){1.4}}
\put(5.5,1){\line(-1,1){.4}}
\put(5.5,1){\line(1,1){.4}}
\put(7,2){ $(\ast \ast)$}

\end{picture}
\end{theorem}


{\bf Proof }
$(\ast)$ can be proven by Chebychev inequality (with usually $p=2$):
\[ \Prob( \mid X_n -X \mid  > \epsilon) \le  \frac{\E \mid X_n -X \mid ^p}{\epsilon^p} \; , \]
and $(\ast\ast)$ is proven in Durrett. $\qedsymbol$



{\bf Exercise. } counter examples
\begin{itemize}
\item  $\convp$ but not $\convas$ : moving blip

\item   $\convp$ but not $\convLp$ : try $X_n=n \mbf{1}(0, 1/n)$.
        $X_n \convp 0$ but $\E \mid X_n-0 \mid = 1$, thus $X \donotconv 0$ in $L^1$.
\end{itemize}


\begin{proposition}[Inducing a Metric]
$X_n \convas X$  cannot be metrized, but $X_n \convLp X$ and $X_n \convp X$ can be metrized, 
e.g. using $\E ( \mid X_n - X \mid \wedge 1 )$. 
Furthermore, when so metrized, the space of random variables are complete.
\end{proposition}
{\bf Proof} See text (uses BCL).


\begin{definition}[Infinitely Often (i.o.) and Eventually (ev.)]
Let $q_n$ be some statement, e.g., $\mid X_n-X \mid > \epsilon$. We say
 ($q_n$ i.o.) if for all $n$, $\exists m \ge n \; : \; q_m$ is true, 
and ($q_n$ ev.) if $\exists \; n$ \; : \;  for all $ m \ge n \; : \; q_m$ is true.
\end{definition}

{\bf Exercise. }Note that the following holds;
\begin{itemize}
\item $X_n \conv X \hspace{.31cm} \iff \hspace{.31cm} \forall \epsilon > 0, \mid X_n -X \mid < \epsilon$ ev.

\item $X_n \donotconv X \hspace{.31cm} \iff \hspace{.31cm} \forall \epsilon > 0, \mid X_n -X \mid > \epsilon$ i.o.

\item $(q_n \mbox{ i.o.})^\sim \; = \; (q_n \mbox{ ev.})$
\end{itemize}

Similarly, for a sequence of events $A_n$ in a prob space $(\Omega, \mcal{F}, \Prob)$, we can say the following;
\begin{itemize}
\item $ (A_n \mbox{ i.o.}) = \left\{ \omega: \omega \in A_n \; i.o.\right\} = \cap_n \cup_{m\ge n} A_m $
\item $ (A_n \mbox{ ev.}) = \left\{ \omega: \omega \in A_n \; ev.\right\} = \cup_n \cap_{m\ge n} A_m $ 
\item $ (A_n \mbox{ i.o.})^c =  A_n^c \mbox{ ev.}$
\end{itemize}

Main application of the idea of i.o. and ev. is to the proof of a.s. convergence. For example, since
\[(X_n \conv X) \hspace{.31cm} = \hspace{.31cm} \cap_{\epsilon >0 } (\mid X_n - X \mid < \epsilon \mbox{ ev.} ) \; ,  \]
we have
\[ \Prob (X_n \conv X) \hspace{.31cm} = \hspace{.31cm} \lim_{\epsilon \conv 0 } \Prob (\mid X_n - X \mid < \epsilon \mbox{ ev.} ) \; . \]
Since the basic criterion for a.s. convergence can be written as 
\[ (X_n \conv X) \hspace{.31cm} \iff \hspace{.31cm} \forall \epsilon > 0,  \Prob (\mid X_n - X \mid > \epsilon \mbox{ i.o.} )=0 \; , \]
we are interested in conditions in some sequence of events $A_n$ so that $\Prob(A_n) \mbox{ i.o.} = 0$.


\section{Borel-Cantelli Lemma}


\begin{theorem}[Borel-Cantelli Lemma] \textcolor{white}{.}
\begin{enumerate}
\item If $\sum_n \Prob(A_n) < \infty$, then $\Prob(A_n \mbox{ i.o.}) = 0$. \\
\item If $\sum_n \Prob(A_n) = \infty$ and $A_n$ are independent, then $\Prob(A_n \mbox{ i.o.}) = 1$.  
\end{enumerate}
\end{theorem}

There are many possible substitutes for independence in BCL II, including Kochen-Stone Lemma. 

Before prooving BCL, notice that
\begin{itemize}
\item $ \mbf{1}(A_n \mbox{ i.o.}) = \lim_{n \to \infty} \sup \mbf{1}( A_n ) $
\item $ \mbf{1}(A_n \mbox{ ev.}) = \lim_{n \to \infty} \inf \mbf{1}( A_n ) $
\item $  (A_n \mbox{ i.o.}) = \lim_{m \to \infty} \Prob ( \cup_{n > m} A_n ) 
         \hspace{.53cm} (\mbox{ as } m \uparrow, \;  \cup_{n \ge m} A_n \downarrow \; )$  
\item $ (A_n \mbox{ i.o.}) = \lim_{m \to \infty} \Prob ( \cap_{n > m} A_n ) 
         \hspace{.53cm} (\mbox{ as } m \uparrow, \;  \cap_{n \ge m} A_n \downarrow \; )$. 
\end{itemize}
Therefore,
\begin{align*}
  \Prob(A_n \mbox{ ev.}) & \le  \lim \inf_{n \to \infty} \Prob(A_n)  \hspace{.62cm} \mbox{ by Fatou's lemma }\\
   & \le  \lim \sup_{n \to \infty} \Prob(A_n)  \hspace{.62cm} \mbox{ obvious from definition }\\
   & \le  \Prob(A_n \mbox{ i.o.})  \hspace{.62cm} \mbox{ duel of Fatou's lemma (i.e. apply to } (\cdots)^\sim ) 
\end{align*} 

{\bf Pf of BCL I} 
\begin{align*}
  \Prob(A_n \mbox{ i.o.}) & =  \lim_{m \to \infty} \Prob(\cup_{n \ge m} A_n)  \\
   & \le  \lim_{m \to \infty} \sum_{n \ge m}^\infty \Prob(A_n)  \; = \; 0 
        \hspace{.32cm} \mbox{ since } \sum_{i=1}^\infty \Prob(A_n) < \infty . \; \qedsymbol
\end{align*} 
  
{\bf Pf of BCL I} (Alternative method)\\
Consider a random variable $N := \sum_\mbf{1}(A_n)$, i.e. the number of events that occur. Then
$\E[N] = \sum_{n=1}^\infty \Prob(A_n)$ by the Monotone Convergence Theorem, and
\begin{align*}
  \sum_{n=1}^\infty \Prob(A_n) < \infty  & \Longrightarrow    \E[N] < \infty\\
        & \Longrightarrow       \Prob(N < \infty)=1 \\
        & \Longrightarrow       \Prob(N = \infty)=0 \\
        & \Longrightarrow       \Prob(A_n \mbox{ i.o.})=0  \hspace{.63cm} \mbox{ because }(N=\infty) \equiv ( A_n \mbox{ i.o.})  . \; \qedsymbol
\end{align*} 
 
{\bf Pf of BCL II} We will show that $ \Prob(A_n^c \mbox{ ev.}) =0$. \\
\begin{align}
  \Prob(A_n^c \mbox{ ev.}) &=  \lim_{n \to \infty} \Prob(\cap_{m \ge n} A_m^c )  
 \; = \; \lim_{n \to \infty} \prod_{m \ge n} \Prob( A_m^c )  \label{eq:BCL2a}\\
&= \lim_{n \to \infty} \prod_{m \ge n} \left( 1-\Prob( A_m ) \right)
\; \le \; \lim_{n \to \infty} \prod_{m \ge n} \exp{(-\Prob( A_m^c ))}  \label{eq:BCL2b} \\
&= \lim_{n \to \infty} \exp{(-\sum_{m \ge n} \Prob( A_m^c ))} = 0 \nonumber
\end{align}
For~\eqref{eq:BCL2a} we used the following fact (due to the independence of $A_n$); 
\begin{align*}
                \Prob(\cap_{m \ge n} A_m^c ) = \lim_{N \to \infty} \Prob( \cap_{n \le m \le N} A_m^c) =  
         \lim_{N \to \infty} \prod_{n \le m \le N} \Prob( A_m^c) =  \prod_{n \le m} \Prob(A_m^c)
\end{align*} 
 and $1-x \le \exp(-x)$ was used in~\eqref{eq:BCL2b}.  \; \qedsymbol

As a trivial example, consider $A_n = (0,1/n)$ in $(0,1)$. Then, 
$\Prob(A_n) = 1/n$, $\sum \Prob(A_n) = \infty$, but $\Prob(A_n \mbox{ i.o.}) = \Prob( \emptyset ) =0$. 


{\bf Intuitive example } Consider random walk  in $\mbf{Z}^d$, $d=0, 1, \cdots$
$S_n=X_1+\cdots+X_n,, \; n=0, 1, \cdots$ where $X_i$ are independent in $\mbf{Z}^d$. In the simplest case, each $X_i$ has uniform distribution on $2^d$ possible strings. i.e., if $d=3$,  we have $2^3=8$ neighbors
\begin{align*}
   \left\{ 
   \begin{array}{c}
        (+1, +1, +1)\\ \vdots \\ (-1, -1, -1)
   \end{array}
   \right\} \; .
\end{align*}    
Note that each coordinate of $S_n$ does a simple coin-tossing walk independently. We can prove that
\begin{align}
  \Prob(S_n=0 \mbox{ i.o.}) = \left\{ 
  \begin{array}{cll}
        1 & \mbox{if } d=1 \mbox{ or } 2 & \mbox{(recurrent)}  \\
        0 & \mbox{if } d \ge 3 & \mbox{(transient)} \; . \label{eq:randwalk:tran}
  \end{array}   \right.
\end{align}

{\bf Sketch of Pf of \eqref{eq:randwalk:tran} }\\
Let us start with $d=1$, then
\[\Prob(S_{2n}=0) = \left( {2n \atop n} \right) 2^{-2n}  \sim \frac{c}{\sqrt{n}} \mbox{ as } n \conv \infty . \]
where we used the fact, $ n ! \sim  \left( {n \atop e} \right)^n \sqrt{2 \pi n}$.

Note  
\begin{align}
  \sum \left( \frac{1}{\sqrt{n}} \right) ^d = \left\{ 
  \begin{array}{rl}
        \infty & \hspace{.61cm} d=1, 2  \\
        <\infty & \hspace{.61cm} d= 3, 4, \cdots \label{eq:randwalk:tran2}
  \end{array}   \right.
\end{align}
BC II and \eqref{eq:randwalk:tran2} together gives \eqref{eq:randwalk:tran}. \; \qedsymbol


Because
\[X_n \conv X \mbox{ a.s.} 
        \hspace{.81cm} \iff \hspace{.81cm} 
  X_n-X \conv 0 \mbox{ a.s.} \; , \]
thus it is enough to understand as convergence to $0$.

\begin{proposition} The following are equivalent:\\
\begin{enumerate}
  \item $ X_n \conv 0$
  \item $ \forall \epsilon > 0, \Prob( \mid X_n \mid >0 \mbox{ i.o.} ) =0 $
  \item $ M_n \conv 0$ where $M_n := \sup_{n \le k} \mid X_k \mid$
  \item $ \exists \epsilon_n  \downarrow 0 \; : \;  \Prob( \mid X_n \mid > \epsilon_n \mbox{ i.o.} ) =0 $
\end{enumerate}
\end{proposition}
 If we need to show $X_n \convas X$ but do not know $X$, then it might be easier to show instead that \\$\Prob(X_n \mbox{ is a Cauchy sequence}) =1$. This leads to the following;
\begin{lemma}
Let $X_n$ be any sequence of random variables, and define $M_n :=\sup_{n \le m} \mid X_n - X_m \mid$. Then \\
\[ \exists X \; : \; X_n \conv X \mbox{ a.s. } \iff
        M_n \convp 0 \]
\end{lemma}
{\bf Proof } Consider  $M_n^\ast :=\sup_{n \le m, p} \mid X_m - X_p \mid$. Notice $M_n^\ast \downarrow$. Thus 
\[ M_n^\ast \convp 0 \hspace{.61cm} \iff \hspace{.61cm} M_n^\ast \convas 0 \]
Combine with the previous result to finish the proof.  \; \qedsymbol






\end{document}