\input{../courseDefs}

\begin{center}
Fourth Set of Notes
\end{center}

\section{Some remarks on Bayes and Minimax estimators}
We observe $X \sim \Prob_\theta$, $\theta \in \Theta$.
Let $\pi$ be the prior distribution of $\theta$; we assume that the support of $\pi$ is
a subset of $\Theta$.
Suppose the conditional distribution of $X$ given $\theta$ is $\Prob_\theta$, with
corresponding expectation operator $E_\theta$.
The action space is $\calA$, and we seek a decision rule 
$\delta : \calX \rightarrow \calA$.
The risk of a decision $\delta$ is
$R(\theta, \delta) = E_\theta \ell(\theta, \delta(X))$.
Define the {\em average risk} of an estimator $\delta$ to be
\begin{eqnarray}
r_\pi(\delta) &\equiv & E_{X, \theta} \ell(\theta, \delta(X)) \nonumber \\
&=& E[E(\ell(\theta, \delta(X)))|X)],
\end{eqnarray}
where the expectation is with respect to the product measure of $X$ and $\theta$.
The Bayes estimator minimizes the average risk.

The {\em posterior risk} of an action $a$ is
$r_\pi(a, x) = E_\pi(\ell(\theta, a) | X = x)$,
where the subscript $\pi$ is to remind us of the prior, but the expectation
is with respect to the conditional distribution of $\theta$ given $X$, which
is derived from the product measure on $X$ and $\theta$.
Ideally, we would like to find the decision rule 
$\delta_\pi : \bfR \rightarrow \calA$ that minimizes
$r(\delta |x)$ for each $x$; such a rule would also minimize the Bayes risk. 
In general, such a rule need not exist; if one exists, it need not be unique
({\em vide infra}).

An estimator $\delta$ is unbiased for $\tau(\theta)$ if $E_\theta \delta(X) = \theta$.
Recall that an estimator $\delta$ is inadmissible if there exists another estimator
that does at least as well for all values
of $\theta$, and better for some value of $\theta$. 
That is, if there is a $\delta_0$ and $\theta_0$ such that
\beq
R(\theta, \delta_0) = E_\theta(\ell(\theta, \delta(X)) \le R(\theta, \delta) 
\;\;\forall \theta \in \Theta,
\eeq
and
\beq
 R(\theta_0, \delta_0) < \le R(\theta_0, \delta).
\eeq 


One of the nice properties of Bayes estimators is that if they are unique, they
are admissible.


Lehmann, TPE, \S 4.1 Theorem 1.1 states (in slightly different notation)
\begin{Theorem}
Let $\theta$ have distribution $\pi$, and, given $\theta = \gamma$, let $X$ have distribution
$\Prob_\gamma$. Suppose $\ell(\theta, a)$ is nonnegative for all $\theta$, and that
there exists an estimator $\delta_0$ with finite risk for estimating estimating $\tau(\theta)$.
If for almost all $x$ there exists a rule $\delta_\pi(x)$ minimizing
$E_\pi \{\ell(\theta, \delta(x)) | X = x \}$, then $\delta_\pi$ is a Bayes estimator.
\end{Theorem}
\begin{Corollary}
If $\ell(\theta, a) = | a - \tau(\theta)|^2$, then $\delta_\pi = E_\pi \{ \tau(\theta) | X = x \}$.
\end{Corollary}
\begin{Corollary}
If $\ell(\theta, a)$ is strictly convex in $a$, a Bayes estimator $\delta_\pi$ is
unique a.e. $\calP = \{ \Prob_\theta \}$, provided the average risk of $\delta_\pi$ is
finite, and provided the marginal distribution $Q$ of $X$ 
\beq
Q(A) = \int \Prob_\theta \{ X \in A \}d\pi(\theta)
\eeq
is such that a.e. $Q$ implies a.e. $\calP$.
\end{Corollary}

The condition on $Q$ ensures that measures $\Prob_\theta$ that are the only ones
to assign mass to some points $x \in \calX$ 
are not themselves given zero measure by $\pi$.

Note that we typically give up unbiasedness in moving to Bayes decisions:
\begin{Theorem} (Lehmann, TPE, 4.4 Theorem 1.2)
Let $\theta \sim \pi$ and let $\Prob_\theta$ be the conditional distribution of
$X$ given $\theta$. Consider estimating $\tau(\theta)$ for squared-error loss.
If $\delta(X)$ is unbiased, it cannot be Bayes unless
\beq
E_{X, \theta} [ \delta(X) - \tau(\theta)]^2 = 0.
\eeq
\end{Theorem}

\noindent
{\bf Proof.}
Suppose $\delta$ is unbiased and is Bayes for $\tau(\theta)$.
Then $\delta(X) = E_\pi [ \tau(\theta)|X]$ a.e.
Unbiasedness implies $E[\delta(X)| \theta = \gamma ] = \tau(\gamma)$ 
for all $\gamma \in \Theta$.
Conditioning on $X$ gives
\begin{eqnarray}
E[ \tau(\theta) \delta(X) ] &=& E\{\delta(X) E[ \tau(\theta) | X ] \} 
\nonumber \\
&=& E \delta^2(X) .
\end{eqnarray}
Conditioning on $\theta$ gives
\begin{eqnarray}
E[ \tau(\theta) \delta(X) ] &=& E\{\tau(\theta) E[ \delta(X) | \theta ] \} 
\nonumber \\
&=& Eg^2(\theta).
\end{eqnarray}
Thus
\beq
E [ \delta(X) - \tau(\theta) ] = E \delta^2(X) + E g^2(\theta) -
 2 E [ \tau(\theta) \delta(X) ]
= 0.
\eeq
$\Box$.

The Bayes estimator minimizes a weighted average of the risks for different 
possible values of the
parameter $\theta \in \Theta$, where the weight is the prior distribution 
on those values.
In contrast, the minimax decision rule minimizes the largest risk for any 
$\theta \in \Theta$:
\beq
\sup_{\theta \in \Theta} R(\theta, \delta).
\eeq
There is a truly wonderful duality between the risks.
A prior $\pi$ for $\theta$ is {\em least favorable} if the Bayes risk is no larger
for any other prior than for it; {\em i.e.}, if $\delta_\pi$ denotes the 
Bayes estimator
for prior $\pi$ on $\theta$, then $\pi^*$ is {\em least favorable} if
\beq
r_{\pi^*}(\delta_{\pi^*}) \ge r_\pi (\delta_\pi)
\eeq
for all priors $\pi$ on $\Theta$.

\begin{Theorem}
(Lehmann, TPE, 4.2 Theorem 2.1)
Suppose that $\pi$ is a prior distribution on $\Theta$ such that
\beq
E_\pi R(\theta, \delta_\pi) = \sup_{\theta \in \Theta} R(\theta, \delta_\pi),
\eeq
where $\delta_\pi$ is the Bayes decision for prior $\pi$, as before.
Then
\begin{enumerate}
\item $\delta_\pi$ is minimax over $\Theta$.
\item If $\delta_\pi$ is the unique Bayes decision for prior $\pi$, it is the
unique minimax decision.
\item $\pi$ is least favorable.
\end{enumerate}
\end{Theorem}

\noindent
{\bf Proof.}
\begin{enumerate}
\item
Let $\delta$ be a different decision rule.
Then
\begin{eqnarray}
\sup_{\theta \in \Theta} R(\theta, \delta) &\ge& E_\pi R(\theta, \delta) \nonumber \\
&\ge& E_\pi R(\theta, \delta_\pi) \nonumber \\
&=& \sup_{\theta \in \Theta} R(\theta, \delta_\pi) .
\end{eqnarray}
\item same proof as (1), using $>$.
\item
Let $\pi_1$ be another prior distribution on $\Theta$. Then
\begin{eqnarray}
r_{\pi_1}(\delta_{\pi_1}) &=& E_{\pi_1} R(\theta, \delta_{\pi_1}) \nonumber \\
&\le& E_{\pi_1} R(\theta, \delta_\pi) \nonumber \\
&\le& \sup_{\theta \in \Theta} R(\theta, \delta_\pi) \nonumber \\
&=& r_\pi .
\end{eqnarray}
\end{enumerate}

For the Bayes risk of the Bayes estimator to equal the maximum risk of the Bayes
estimator implies that 
\beq
\Prob_\pi \{ R(\theta, \delta_\pi) = 
\sup_{\nu \in \Theta} R(\nu, \delta_\pi) \} = 1.
\eeq
This, together with the theorem, implies that if a Bayes estimator
has constant risk (over $\Theta$), it is minimax. Moreover, if there is a set
$\omega \subset \Theta$ with $\pi(\omega) = 1$ such that $R(\theta, \delta_\pi)$
attains its maximum at all $\theta \in \omega$, then $\delta_\pi$ is minimax.

The preceeding development has tacitly assumed that we are restricting attention to
non-randomized estimators.
When the loss function is strictly convex, the every randomized estimator is
dominated by a non-randomized estimator. When the loss function is merely convex, 
for each randomized estimator, there is a non-randomized estimator whose risk
is no larger than that of the randomized estimator.
Thus in many situations (squared-error loss, in particular) it suffices to
consider non-randomized estimators.

The following material is drawn primarily from TPE.

\begin{Lemma}
Jensen's inequality.  Let $f: \calX \rightarrow \bfR$ be a convex function, and
let $X$ be a random variable taking values in $\calX$.
Then
\beq
f(EX) \le Ef(X).
\eeq
If $f$ is strictly convex, the inequality is strict unless $X$ is almost surely
constant.
\end{Lemma}

\begin{Definition}
A randomized decision rule $\delta$ is a mapping from the sample space $\calX$ to
a random variable $Y(x)$ that takes values in the action space $\calA$ 
(which is assumed to be a
measurable space). 
To each
$x \in \calX$, $\delta$ assigns a random variable $Y(x)$ with {\em known} 
distribution $\Prob_x$.
The decision rule assigns to an observed value $x$ an observation from the
random variable $Y(x) \sim \Prob_x$.
The risk of a randomized decision rule is $E_\theta E_X \ell(\theta, Y(X))$.
\end{Definition}


\begin{Theorem}(Lehmann, TPE, \S1.5, Theorem 5.1)
Suppose $X \sim \Prob_\theta$, $\theta \in \Theta$, and let $T$ be sufficient for
$\Prob_\Theta$. For any estimator $\delta(X)$ of $\tau(\theta)$ there exists
a (possibly randomized) estimator based on $T$ that has the same risk function as
$\delta(X)$.
\end{Theorem}

{\bf Sketch of proof.}
Given $T$, the conditional distribution of $X$ does not depend on $\theta$.
Let $\Prob(\cdot | T=t)$ denote this distribution.
Given $T=t$, one can construct a random variable $X_t'$ that has distribution
$\Prob_(\cdot | T=t)$
The unconditional distributions of $X_t'$ and $X$ are the same: 
$\Prob_\theta \{X_t' \in A \}
= \Prob_\theta\{X \in A\}$ for all measurable subsets 
$A \subset \calX$.
Thus if one knows the value of $T$, performing a subsequent randomization by drawing
from $\Prob_(\cdot | T=t)$, allows one to generate data with the same distribution
as the original experiment gave.  
One can therefore construct an estimator
$\delta'(t)$ that depends on the data only through $T$ and that is risk-equivalent to 
$\delta(x)$ by taking $\delta(t)$ to be $\delta(X_t')$, whose value depends on the data
only through $T$.

\noindent
{\bf Remark.}
Any randomized estimator from data $X$ is equivalent to a non-randomized estimator
from data $X' = (X, U)$, where $U \sim U[0,1]$ is independent of $X$.

\begin{Theorem}The Rao-Blackwell Theorem (see Lehmann, TPE, \S1.6, Theorem 6.4).
Let $X$ have distribution $\Prob_\theta \in \Prob_\Theta = \{ \Prob_\nu : \nu \in \Theta\}$,
and let $T$ be sufficient for $\Prob_\Theta$. Let $\delta : \calX \rightarrow \calA$
be an estimator of $\tau(\theta)$, and let the loss $\ell(\theta, a)$ be
strictly convex in $a$.
Suppose $E_\theta \delta(X) < \infty$ and 
$E_\theta \ell(\tau(\theta), \delta(X)) < \infty$,
$\theta \in \Theta$.
Let the estimator $\eta(t) \equiv E[\delta(X) | T = t]$.
Then
\beq
R(\theta, \eta) < R(\theta, \delta)
\eeq
unless $\delta(X) = \eta(T)$ with probability 1.
\end{Theorem}

\noindent
{\bf Proof.}
If $\ell$ is strictly convex in $a$, then applying Jensen's 
inequality to the conditional
expectation given $T=t$,
\beq
\ell(\theta, \eta(t)) < E\{\ell(\theta, \delta(X))|T=t\},
\eeq
unless $\delta(X) = \eta(t)$ a.s.
Thus
\beq
E_\theta \ell(\theta, \eta(t)) < E_\theta E\{\ell(\theta, \delta(X))|T=t\},
\eeq
which was to be shown.


\begin{Corollary}(Lehmann, TPE, \$1.6, Corollary 6.2)
If the loss function $\ell$ is strictly convex, every randomized
estimator of $\tau(\theta)$ is dominated by a non-randomized estimator.
If $\ell$ is convex, there is a non-randomized estimator whose 
risk function is
pointwise no larger than that of any randomized estimator.
\end{Corollary}

{\bf Proof.}
Any randomized estimator is equivalent to a nonrandomized estimator
based on $(X, U)$, and $X$ is sufficient for $X$.

Note that the ``zero-one'' loss associated with confidence intervals is not convex.
If the loss is 
\beq
\ell(\theta, a) = \left \{ 
\begin{array}{ll}
0, & |\theta - a| \le \chi \\
 1, & | \theta - a| > \chi ,
\end{array}
\right .
\eeq
then the risk of $\delta$ is the non-coverage probability of the 
fixed-length interval $[\delta - \chi,
\delta + \chi]$, which one would like to minimize for a given $\chi$.
This loss is not convex: take $a_0 = \theta$ and $a_1 = \theta + 3\chi$.
Then $\ell(\theta, a_0) = 0$, $\ell(\theta, a_1) = 1$, and 
\beq
\ell(\theta, (a_0 + a_1)/2) = 1 > (\ell(\theta, a_0) + \ell(\theta, a_1))/2 = 1/2.
\eeq

(This loss is, however, {\em quasiconvex}. 
A quasi-convex function $f$ is one for which 
\beq
f(\lambda x + (1-\lambda)y) \le \max\{f(x), f(y)\},
\eeq
for all $x$, $y$, and for all $\lambda \in [0,1]$.
If the inequality is strict whenever $\lambda \in (0, 1)$ and $x \ne y$,
$f$ is strictly quasiconvex.
For any two actions $a_0$ and $a_1$, we have
\beq
\ell(\theta, \lambda a_0 + (1-\lambda)a_1) \le \max (\ell(\theta, a_0), \ell(\theta, a_1)),
\;\; \forall \lambda \in [0, 1],
\eeq
so $\ell$ is quasiconvex (but not strictly) in $a$.
A different characterization of quasiconvex functions is that
$f$ is quasiconvex iff its level sets $\{x : f(x) \le b \}$ are convex
for every $b$.  
A local minimum of a strictly quasiconvex function is a global minimum.)

Lehmann (TPE, 4.2 Example 2.2) gives an example for this zero-one loss
where a randomized decision does better than a non-randomized one.
Suppose we are estimating the probability $p$ of success in $n$ i.i.d. Bernoulli($p$)
trials from the total number $X$ of successes in the trials,
which is a binomially-distributed sufficient statistic.
Suppose the interval half-width is $\chi < 1/(2(n+1))$.
There are only $n+1$ possible data, so a non-randomized rule can take only $n+1$ possible
values. Because the interval is so short, the union of the intervals centered at those
values cannot include all of $\Theta = [0, 1]$, and thus the maximum risk for the minimax
non-randomized rule is 1. (Hence, just picking $\delta(X) = 0$ is minimax among
non-randomized decisions.)
On the other hand, suppose we use the randomized rule $\delta_r(X) \sim U(0,1)$,
independent of the data and
ignoring the data completely.
Then
\beq
\sup_{\theta \in [0, 1]} \Prob \{ | U - \theta | > \chi \} = 1 - \chi < 1.
\eeq
In this case, a randomized rule does uniformly better (as measured by maximum risk
over $\Theta$) than the best non-randomized rule.

\section{Some Math}
Before we begin, some math.

\begin{Definition}
A set $\calX$ is {\em partially ordered} by a relation $\le$ if for 
$x, y, z \in \calX$,
\begin{enumerate}
\item $x \le y$ and $y \le z$ $\Rightarrow$ $x \le z$ (transitivity)
\item $x \le x$ for all $x \in \calX$ (reflexivity)
\item $x \le y$ and $y \le x$ $\Rightarrow$ $x = y$.
\end{enumerate}
A subset $\calX_0$ of $\calX$ is totally ordered by $\le$
if for every $x, y \in \calX$, either $x \le y$ or $y \le x$.
If $\calX_0$ is totally ordered, $x, y \in \calX_0$, $x \le y$, and
$x \ne y$, we write $x < y$.
\end{Definition}
That every nonempty partially ordered set contains
a maximal totally ordered subset is Hausdorff's maximality theorem.

\begin{Definition}
Suppose the sets $\calX$ and $\calY$ are totally ordered.
Let $K(x, y) : \calX \times \calY \rightarrow \bfR$.
We say $K(x,y)$ is {\em sign regular of order $r$} (SR${}_r$) if for every $1 \le m \le r$
there is a constant $\epsilon_m = \pm 1$ such that for
every pair of increasing sets of elements 
 $(x_1 < x_2 < \ldots < x_m)$ and $(y_1 < y_2 < \ldots < y_m)$,
\beq \label{eq:srDef}
\epsilon_m K \left (
\begin{array}{c}
x_1, x_2, \ldots, x_m \\
y_1, y_2, \ldots, y_m
\end{array}
\right ) \equiv
\left | 
\begin{array}{cccc}
K(x_1, y_1) & K(x_1, y_2) & \cdots & K(x_1, y_m) \\
K(x_2, y_1) & K(x_2, y_2)& \cdots & K(x_2, y_m) \\
\cdots & \cdots & \cdots & \cdots \\
K(x_m, y_1) K(x_m, y_2) \cdots K(x_m, y_m) 
\end{array}
\right |
\ge 0 ,
\eeq
where the vertical bars denote the determinant of the matrix.
If the inequality \ref{eq:srDef}
is strict, $K$ is said to be {\em strictly sign regular of
order $r$} (SSR${}_r$).
If all $\epsilon_j$ equal $+1$, $1 \le j \le r$, $K$ is said to be
{\em totally positive of order $r$} (TP${}_r$).
If all $\epsilon_j$ equal $+1$, $1 \le j \le r$, and the inequality 
\ref{eq:srDef} is strict, we say $K$ is {\em strictly
totally positive of order $r$} (STP${}_r$).
If the inequality \ref{eq:srDef} holds for all finite $r$, $r$ is omitted from
the notation, and $K$ is said to be sign regular (SR), strictly sign regular (SSR),
totally positive (TP), or strictly totally positive (STP), respectively.
\end{Definition}

For statistical applications, a very useful fact is that the ``kernel'' $K(x, y)$
associated with the ``density'' of a one-parameter exponential family is
totally positive.
That is, if $\calX$ and $\calY$ are totally ordered subsets of $\bfR$,
the kernel $K(x, y) = \beta(x) e^{xy}$ is totally positive.
This follows from the fact that
an exponential polynomial $\sum_{j=1}^n p_j(y) e^{c_jy}$, where $c_j \ne c_j$ for $i \ne j$,
and $p_j$ is a real polynomial of degree $d_j$, either vanishes identically, or has
at most $n-1 + \sum_{j=1}^n d_j$ zeros (counting multiplicities).

\begin{Definition}
The {\em lower number of sign changes} of
a finite real-valued 
sequence $(x_j)_{j=1}^m$, $S^-((x_j))$, is the number of sign changes in the
sequence, discarding zeros.
The {\em upper number of sign changes} of $(x_j)$,
$S^+((x_j))$, is the maximum number of sign changes
in the sequence when the terms that equal zero are counted as having arbitrary signs.
Let $f$ be a real-valued function defined on a totally ordered subset $\calI$ of $\bfR$.
The {\em lower number of sign changes of $f$}, $S^-(f)$ is
\beq
S^-(f) = \sup_{m < \infty,\;\; \{ x_j \} \subset \calI :
x_1 < x_2 < \ldots x_m } S^-((f(x_j))_{j=1}^m) ,
\eeq
and the {\em upper number of sign changes of $f$}, $S^+(f)$,
is
\beq
S^+(f) = \sup_{m < \infty,\;\; \{ x_j \} \subset \calI :
x_1 < x_2 < \ldots x_m } S^+((f(x_j))_{j=1}^m ).
\eeq
\end{Definition}

A very important result (which we shall use presently)
is that transformations induced by a sign-regular kernel are
{\em variation diminishing}: they do not increase the number of zero-crossings
of a function. 


\begin{Theorem} (Karlin, \S3, Theorem 3.1)
Let $K(x,y): \calX \times \calY \rightarrow \bfR$ be Borel measurable,
where $\calX$ and $\calY$ are totally ordered topological 
spaces.
Let $\mu$ be a sigma-finite regular measure on
$\calY$, such that $\mu(U)>0$ for each open set $U$ for which $U \cap \calY \ne \emptyset$.
Let $X$ be a totally ordered topological space, and let 
$K(x,y): \calX \times \calY \rightarrow
\bfR$ be Borel-measureable, and assume that $\int_\calY K(x,y) d\mu(y)$ exists for every
$x \in \calX$.
Let $f: \calY \rightarrow \bfR$ be a bounded, Borel-measurable
function on $\calY$.
Define the transformation $(Tf): \calX \rightarrow \bfR$ by
\beq
(Tf)(x) = \int_{\calY} K(x, y) f(y) d\mu(y) .
\eeq
\begin{enumerate}
\item
If $K(x,y)$ is SR${}_r$, then if $S^-(f) \le r-1$,
\beq
S^-(Tf) \le S^-(f),
\eeq
If $K$ is TP${}_r$ and $f$ is piecewise continuous, then if
$S^-(f) = S^-(Tf) \le r-1$, $f$ and $Tf$ have the same sequence of signs as their
arguments increase.
\item
If $K$ is SSR${}_r$ and $f \ne 0$ a.e.($\mu$),
\beq
S^+(Tf) \le S^-(f)
\eeq
if $S^-(f) \le r-1$.
\end{enumerate}
\end{Theorem}

A transformation that does not increase the number of zero crossings of
a function is called {\em variation diminishing}.
Because the kernel associated with a one-parameter exponential
family is $TP$, the theorem implies that integration against the 
density of an exponential family is variation diminishing.

For example, we obtain the Normal distribution with unit variance by taking
$\beta(x) = e^{-x^2/2}/\sqrt{2\pi}$ and $d\mu(y) = e^{-y^2/2} dy$.
Suppose $f$ is bounded and Borel-measurable. Let
\begin{eqnarray}
(Tf)(x) &=& \int_\bfR f(y) e^{-x^2/2}/\sqrt{2\pi} e^{xy} e^{-y^2/2} dy \nonumber \\
&=& \int_\bfR f(y) e^{-(x-y)^2/2}/\sqrt{2\pi} dy \nonumber \\
&=& \int_\bfR f(y) \phi(x-y) dy \nonumber \\
&=& f \star \phi,
\end{eqnarray}
where $\phi$ is the density of the standard normal distribution and $\star$ denotes
convolution.
Then $S^-(f \star \phi) \le S^-(f)$.

In this case, $K(x,y) d\mu(y)$ is a probability density for fixed $x$. 
Suppose $Y$ is a random
variable with that density. Then a different notation for
the transformation $T$ is $(Tf)(x) = E_x f(Y)$.

See Karlin, 1968, {\em Total Positivity}, Stanford Univ. Press, Stanford CA, for more on
total positivity.


\end{document}



