\input{../courseDefs}

\begin{center}
Fifth Set of Notes
\end{center}
\section{More on the Bounded Normal Mean}

\begin{Lemma}
Stein's Lemma. (See Evans and Stark, 1996. {\em Ann. Stat., 24}, 809-815, for a
substantial generalization.)
Suppose $X \sim N(\theta,1)$, and that $\delta(\cdot)$ is differentiable, with
$E_\theta |\delta'(X)| < \infty$,  
$\lim_{x \rightarrow \pm \infty} \delta(x)\exp\{-(x-\theta)^2/2\} = 0$,
and that $E_\theta [\delta(X) (X-\theta)]$ is finite. 
Then
\beq
E_\theta [\delta(X)(X-\theta)] = E_\theta \delta'(X).
\eeq
\end{Lemma}

\noindent
{\bf Proof.}
Let $\phi(x)$ be the standard normal density.
Integrate by parts:
\begin{eqnarray}
E_\theta [\delta(X)(X-\theta)] &=& \int_{-\infty}^\infty \delta(x)(x-\theta) \phi(x-\theta) dx 
\nonumber \\
&=&
\int_{-\infty}^\infty \delta(x)(x-\theta) \phi(x) dx \nonumber \\
&=& \left . -\delta(x) \phi(x) \right |_{-\infty}^\infty
+ \int_{-\infty}^\infty \delta'(x) \phi(x) dx \nonumber \\
&=& \int_{-\infty}^\infty \delta'(x) \phi(x) dx \nonumber \\
&=& E_\theta \delta'(X).
\end{eqnarray}


\begin{Lemma}
Consider estimating the mean $\theta$ of a normal distribution with unit variance
using an estimator $\delta$ that satisfies the conditions of Stein's Lemma.
For squared-error loss,
\beq
R(\theta, \delta) = 1- E_\theta (2\psi'(X) - \psi^2(X)),
\eeq
where $\psi(x) = x - \delta(x)$.
\end{Lemma}

\noindent
{\bf Proof.}
\begin{eqnarray}
R(\theta, \delta) &=& E_\theta (\theta - \delta(X))^2 \nonumber \\
&=& E_\theta \left ( \theta - X + X - \delta(X)\right )^2  \nonumber \\
&=& E_\theta \left ( -(X - \theta) + \psi(X) \right )^2 \nonumber \\
&=& E_\theta  \left ( (X - \theta)^2 - 2(X-\theta)\psi(X) + \psi(X)^2 \right )
\nonumber \\
&=& 1 - E_\theta \left ( 2 \psi'(X) - \psi^2(X)\right ),
\end{eqnarray}
using the lemma in the last step.

Bickel (1981, {\em Ann. Stat., 9}, 1301-1309) studies the minimax
problem as the bound goes to infinity and finds the asymptotic form
of the least-favorable
prior.
We have a single observation $X \sim N(\theta, 1)$, with $\theta \in \Theta = [-m, m]$.
The loss function $\ell(\theta, a) = | \theta - a |^2$.
The action space is $\calA = \bfR$ (which we might as well limit to $[-m, m]$ if we
can).
The ``natural'' estimator is $\delta^0(x) = x$,
and the maximum likelihood estimator is the truncation estimator:
\beq
\delta_{MLE}(x) = \left \{
\begin{array}{ll} m, & x \ge m \\
x, & |x| < m, \\
-m, x \le -m 
\end{array}
\right .
\eeq
Clearly, the risk of $\delta^0$ is unity for all $\theta$, the
maximum risk of the MLE is for $\theta = 0$, and the minimum risk
is for $\theta = \pm m$.

\begin{Lemma}
(Bickel, 1981; special case of Brown, 1971)
Suppose $X \sim N(\theta, 1)$, $\theta \in \Theta$, $\theta \sim \pi$.
Let $f_\pi$ be the density of the marginal
distribution of $X$: 
\beq
f_\pi (x) = \phi \star \pi(x) = \int_{-\infty}^\infty
\phi(x - \theta) \pi(d\theta).
\eeq
The Bayes risk for $\pi$ for squared-error loss is
\beq
r(\pi) = 1 - \int_{-\infty}^\infty \frac{(f'_\pi(x))^2}{f_\pi(x)} dx .
\eeq
\end{Lemma}

\noindent
{\bf Proof.}
The Bayes estimator for prior $\pi$ is the posterior mean of $\theta$ given $x$.
The derivative of $f_\pi(x)$ with respect to $x$ is
\begin{eqnarray}
\frac{d}{dx} f_\pi(x) &=& \int_{-\infty}^\infty \frac{d}{dx} \phi(x-\theta) \pi(d\theta) 
\nonumber \\
&=& \int_{-\infty}^\infty (-(x - \theta))\phi(x-\theta) \pi(d\theta) \nonumber \\
&=& -x f_\pi(x) + 
\int_{-\infty}^\infty \theta \phi(x-\theta) \pi(d\theta)
 \nonumber \\
&=& ( -x + E(\theta | x) )f_\pi(x),
\end{eqnarray}
so the posterior mean of $\theta$ given $x$ is
\beq
\delta(x) = \delta_\pi(x) = x + f'_\pi(x)/f_\pi(x) .
\eeq
Applying the previous lemma, with $\psi(x) = x - \delta(x) = -f'_\pi(x)/f_\pi(x)$
gives the risk at $\theta$ of this estimator for squared-error loss to be
\begin{eqnarray}
1 - E_\theta \left ( 2 \psi'(X) - \psi^2(X)\right ) &=&
1 - E_\theta \left ( -2 f''_\pi(X)/f_\pi(X) + 2(f'_\pi(X))^2/f_\pi^2(X)
- (f'_\pi(X))^2/f_\pi^2(X) \right ) \nonumber \\
&=& 1 - E_\theta \left ( f'_\pi(X))^2/f_\pi^2(X)- 2 f''_\pi(X)/f_\pi(X) \right ).
\end{eqnarray}
Taking the expectation with respect to $\pi$ to find the Bayes risk, and
using Fubini's Theorem yields
\begin{eqnarray}
r(\pi) = E_\pi R(\theta, \delta_\pi) &=&
1 - \int_{-\infty}^\infty \int_{-\infty}^\infty 
\frac{(f'_\pi(x))^2 - 2 f''_\pi(x) f_\pi(x)}{f_\pi^2(x)}\phi(x-\theta) \pi(d\theta) dx
\nonumber \\
&=& 1 - \int_{-\infty}^\infty 
\frac{(f'_\pi(x))^2 - 2 f''_\pi(x) f_\pi(x)}{f_\pi^2(x)}f_\pi(x) dx
\nonumber \\
&=& 1 - \int_{-\infty}^\infty 
\frac{(f'_\pi(x))^2 }{f_\pi(x)}dx + \left . 2 f'_\pi(x) \right |_{x=-\infty}^\infty .
\end{eqnarray}
Because $f_\pi$ is the result of convolution with a Gaussian density, its derivatives
of all orders vanish as $x \rightarrow \pm \infty$, so we have
\beq
r(\pi) = 1 - \int_{-\infty}^\infty \frac{(f'_\pi(x))^2 }{f_\pi(x)}dx .
\eeq
Note that for a distribution $F$ with absolutely continuous density $f$, the
Fisher information is
\beq
I(F) = \int_{-\infty}^\infty \frac{(f'(x))^2}{f(x)} dx,
\eeq
so the equality just established is  $r(\pi) = 1 - I(\Phi \star \pi)$,
where $\Phi$ is the standard normal distribution.
Thus we have a relation between the Bayes risk for a given 
prior on $\theta$ and the Fisher information of
the marginal distribution of the observation $X$ for that prior.
The least favorable prior is that for which the Fisher information of $\Phi \star \pi$
is minimal.


Let $\rho(m)$ be the minimax risk for squared-error loss with $\Theta = [-m, m]$,
and let $r(\pi)$ be the Bayes risk for squared-error loss using prior $\pi$ on
$\theta$. 
Bickel (1981) uses the relation between the Bayes risk and
the Fisher information, and properties of the Fisher information, to show that 
if $\pi_1$ is the distribution on $[-1, 1]$ with density
\beq
g_1(s) = \left \{ \begin{array}{ll}
\cos^2(s\pi/2), & |s| \le 1 \\
0, & \mbox{otherwise},
\end{array}
\right .
\eeq
and $\pi_m$ is the distribution on $[-m, m]$ with density
\beq
g_m(s) = m^{-1} g_1(s/m),
\eeq
then the priors $\{ \pi_m \}$ are approximately least favorable.

\begin{Theorem}(Bickel, 1981, Theorem 2.1).
As $m \rightarrow \infty$,
\beq
\rho(m) = r(\pi_m) + o(m^{-2}),
\eeq
and
\beq
r(\pi_m) = 1 - \frac{\pi^2}{m^2} + o(m^{-2}).
\eeq
Let $\pi_m^0$ be the least favorable prior when $\Theta = [-m, m]$,
and let $\pi_1^{(m)}$ be the distribution obtained by scaling
$\pi_m^0$ to $[-1, 1]$: $\pi_1^{(m)}(s) = \pi_m^0(ms)$.
Then $\pi_1^{(m)}$ converges weakly to $\pi_1$.
\end{Theorem}

Perhaps surprisingly, the Bayes estimators $\delta_{\pi_m}(x)$ are not
asymptotically minimax (indeed, $\lim\sup_m R(m,\delta_{\pi_m}) > 1$,
so it does not dominate the naive estimator $\delta(x) = x$).
Bickel (1981) also shows how to modify the estimator to be asymptotically minimax
to order $m^{-2}$.

Casella and Strawderman (1981, {\em Ann. Stat., 9}, 870-878) and
Gatsonis, MacGibbon and Strawderman (1987, {\em Stat. Prob. Lett., 6}, 21-30)
address the estimation of a bounded normal mean, using squared-error loss.
The former paper looks at Bayes estimators for
2-point and 3-point priors, and shows that they are minimax when the bound on the mean
is small; the latter shows that 
a uniform prior performs surprisingly well in a minimax sense.

Let $\pi_m^0$ put mass $1/2$ on $\pm m$.  
Because the loss is squared-error, the Bayes estimator $\delta_m^0$ against
that prior is the posterior mean of $\theta$ given $x$, which we can calculate.
The marginal density of $X$ is
\beq
f_X(x) = \frac{1}{2\sqrt{2\pi}} \left [
e^{-(x-m)^2/2} + e^{-(x+m)^2/2} \right ].
\eeq
Proceeding blithely without concern for rigor, the posterior ``density'' of
$\theta$ given $X=x$ is
\beq
f_{\theta | X=x}(\theta) = 
\frac{ \delta_{\theta - m} e^{-(x-m)^2/2} + 
\delta_{\theta + m} e^{-(x-m)^2/2}}{e^{-(x-m)^2/2} + e^{-(x+m)^2/2}},
\eeq
where $\delta_x$ is the Dirac delta measure (a point-mass at $x=0$).
The posterior mean is 
\begin{eqnarray}
\delta_m^0(x) &=& E(\theta | X=x) \nonumber \\
&=&
\left [ 
\frac{m e^{-(x-m)^2/2} - m e^{-(x-m)^2/2}}{e^{-(x-m)^2/2} + e^{-(x+m)^2/2}}
\right ] \nonumber \\
&=& m \frac{e^{mx} - e^{-mx}}{e^{mx} + e^{-mx}} \nonumber \\
&=& 
 m \tanh (mx) .
\end{eqnarray}

Similarly,
let $\pi_m^\alpha$ put mass $\alpha$ at zero, and mass $(1-\alpha)/2$ at $\pm m$.
The Bayes estimator for that prior is 
\beq
\delta_m^\alpha(x) = \frac{(1-\alpha)m \tanh (mx)}{1 - \alpha + \alpha \exp(m^2/2)
\sech(mx)}.
\eeq

Finally, let $\pi_m$ be the uniform prior on $[-m, m]$. Let's find the corresponding
Bayes estimator.
The conditional distribution of $X$ given $\theta$ is as before, and the marginal
density of $X$ is 
\beq
\frac{1}{2m} \int_{\theta = -m}^m \phi(x-\theta) d\theta = 
\frac{1}{2m}(\Phi(x-m) - \Phi(x+m)),
\eeq
where $\phi(\cdot)$ is the standard normal density and $\Phi(\cdot)$ is the
standard normal cdf.
The posterior density of $\theta$ is 
\beq
f(\theta | x) = 1_{|\theta| \le m} \frac{\phi(x-\theta)}{\Phi(x-m) - \Phi(x+m)}.
\eeq
The Bayes estimator $\delta_m(x)$ is the posterior mean, namely 
\beq \label{eq:pmUnif}
\frac{\int_{-m}^m \theta \phi(\theta-x) d\theta}{\Phi(x-m) - \Phi(x+m)}.
\eeq
Let's work on the numerator:
\begin{eqnarray}
\int_{-m}^m \theta \phi(\theta - x) d\theta &=&
\int_{-m}^m (\theta - x) \phi(\theta - x) d\theta + \int_{-m}^m x \phi(\theta-x) d\theta
\nonumber \\
&=& e^{-(x-m)^2/2} - e^{-(x+m)^2/2} + x (\Phi(x-m) - \Phi(x+m)).
\end{eqnarray}
Thus \ref{eq:pmUnif} is
\beq
\delta_m(x) = 
x + \frac{e^{-(x-m)^2/2} - e^{-(x+m)^2/2}}{\Phi(x-m) - \Phi(x+m)}.
\eeq

One use the results following Stein's lemma to calculate the risks differently.

\begin{Lemma}(Casella and Strawderman, 1981, Lemma 3.1)
The Bayes estimator $\delta_m^0$ has maximum risk
\beq
\max_{\theta \in [-m, m]} R(\theta, \delta_m^0 ) = \max
\left ( R(0,\delta_m^0), R(m, \delta_m^0) \right ) .
\eeq
\end{Lemma}

\noindent
{\bf Proof.}
Let $\delta(x) = \delta_m^0(x)$.
\begin{eqnarray}
\delta'(x) = (d/dx) \delta(x)   &=& m (d/dx)\tanh(mx) \nonumber \\
&=& m^2 - m^2\tanh^2(mx) \nonumber \\
&=& m^2 - \delta^2(x) ,
\end{eqnarray}
and
\beq
\delta''(x) = (d^2/dx^2) \delta(x) = -2\delta(x) \delta'(x) .
\eeq
Note that in general, if $\delta(x)$ is differentiable (any other conditions needed?)
and $\theta$ is a location parameter,
\begin{eqnarray}
\lim_{a \rightarrow 0} \frac{E_{\theta+a}\delta(X) - E_\theta \delta(X) }{a}
&=& \lim_{a \rightarrow 0} \frac{E_\theta\delta(X+a) - E_\theta \delta(X) }{a}
\nonumber \\
&=& \lim_{a \rightarrow 0} \frac{E_\theta(\delta(X+a) - \delta(X)) }{a} \nonumber \\
&=& E_\theta \lim_{a \rightarrow 0} \frac{\delta(X+a) - \delta(X) }{a} \nonumber \\
&=& E_\theta \delta'(X)  \label{eq:diffUnderE}.
\end{eqnarray}
The risk of $\delta$ at $\theta$ is
\beq
E_\theta |\theta - \delta(X)|^2 = \theta^2 - 2 \theta E_\theta \delta(X) + E_\theta \delta^2(X).
\eeq
The derivative of the
risk w.r.t. $\theta$ is (applying \ref{eq:diffUnderE})
\begin{eqnarray}
d/d\theta R(\theta,\delta_m^0) &=& 
2\theta - 2 E_\theta \delta(X) - 2\theta E_\theta \delta'(X) + 2 E_\theta \delta(X)\delta'(X)
\nonumber \\
&=& 2E_\theta \left [ (\theta - \delta(X))(1 - \delta'(X)) \right ]
\nonumber \\
&=& 2 E_\theta \left [ ((\theta - X) + (X - \delta(X)))(1 - \delta'(X)) \right ]\nonumber \\
&=& E_\theta \left [ (X - \delta(X)) - \delta'(X)(X+\delta(X)) \right ].
\end{eqnarray}
Casella and Strawderman use Karlin's change of sign lemma to show that
this can have at most three sign changes; recall that $E_\theta$ is a variation-diminishing
transformation for $N(\theta, 1)$, so the result follows if the argument of the
expectation has at most three sign changes.
The argument is
\beq
(x - \delta(x)) - \delta'(x)(x + \delta(x)),
\eeq
which vanishes at $x=0$.
Its other zeros solve
\beq
x - \delta(x) = \delta'(x)(x + \delta(x)),
\eeq
or equivalently (for $x \ne 0$)
\beq \label{eq:csZeros}
1 - \delta(x)/x = \delta'(x) ( 1 + \delta(x)/x ) .
\eeq
For $x>0$, $\delta(x)/x$ and $\delta'(x)$ are positive and decreasing.
Thus for $x>0$,
$(1 - \delta(x)/x)$ is increasing, and $\delta'(x)(1 + \delta(x)/x)$ is decreasing,
so \ref{eq:csZeros} has at most one solution for $x>0$.
Note that $\delta'(x)$ is an even function, and $\delta(x)$ and $x$ are odd functions,
so
$(x - \delta(x)) - \delta'(x)(x + \delta(x))$ is odd, and
it has at most one zero for $x <0$, and thus at most three zeros counting
the one at $x=0$.
Both $\delta(x)$ and $\delta'(x)$ are bounded, so the argument is negative for $x\rightarrow
-\infty$ and positive for $x \rightarrow +\infty$; hence, the sign sequence of
the argument is $-+-+$. By the change of sign lemma, the expectation also has at most
three sign changes, in the same order. 
Again because the argument is odd,
\beq
E_0 \left [(X - \delta(X)) - \delta'(X)(X + \delta(X)) \right ] = 0.
\eeq
Thus the risk is stationary at $\theta = 0$.
The derivative of the risk at $\theta > 0$ is the negative of the derivative of the
risk at $-\theta$ (as you can see from symmetry).
Thus a local extremum of the risk for some $\theta > 0$ must be a minimum,
and hence the maximum risk is either at $\theta=0$ or at $\theta = \pm m$.
This proves the lemma.

If we can establish that the Bayes risk of the estimator is equal to its
maximum risk, the estimator is minimax.

\begin{Lemma} Casella and Strawderman, 1981, Lemma 3.2.
The function
\beq
f(m) = R(0, \delta_m^0) - R(m, \delta_m^0)
\eeq
has only one sign change as $m$ goes from $0$ to $\infty$.
The sign change is from negative to positive, so there is a unique $m_0 \in \bfR$
s.t. $f(m) \le 0$, $\forall m \le m_0$.
\end{Lemma}

\noindent
{\bf Proof.}
The difference in risks is
\begin{eqnarray}
f(m) &= &  E_0 (\delta(X) - 0)^2 - E_m (\delta(X) - m)^2 \nonumber \\
&=& E_0(m\tanh(mX))^2 - E_m(m\tanh(mx) - m)^2 \nonumber \\
&=& m^2 \left [ E_0 \tanh^2(mX) - E_m(1 - \tanh(mx))^2 \right ] \nonumber \\
&\equiv& m^2 g(m) .
\end{eqnarray}
Differentiating $g(m)$ gives
\beq
\frac{d}{dm} g(m) =
 2 E_0 (X \tanh(mX)\sech^2(mX)) + 2 E_m((X+m)(1-\tanh(mX))\sech^2(mX)).
\eeq
The second expectation is of an argument with but one sign change, from negative
to positive, so if its expectation is positive at $\theta = 0$, it is positive for 
$\theta = m \ge 0$ (the distribution of the argument will be stochastically larger).
Define $Q = X \tanh(mX) \sech^2(mX)$.
We have
\begin{eqnarray}
\frac{d}{dm} g(m) &\ge&
2E_0 (X \tanh(mX)\sech^2(mX)) + 2 E_0((X+m)(1-\tanh(mX))\sech^2(mX))
\nonumber \\
&=& 2 E_0 Q + 2 E_0 \left ( (X+m) \sech^2(mX) - Q \right )
\nonumber \\
&=& 2 E_0 \left ( (X+m) \sech^2(mX) \right ) \nonumber \\
&=& 2 m E_0 \sech^2(mX) + 2 E_0 X \sech^2(mX) \nonumber \\
&=& 2mE_0 \sech^2(mX) \nonumber \\
&\ge& 0 \mbox{ for $m \ge 0$}.
\end{eqnarray}
(The penultimate step uses the fact that $X \sech^2(mX)$ is an odd function.) 
This proves the lemma, and takes us to one of the main theorems of 
Casella and Strawderman:

\begin{Theorem}Casella and Strawderman, 1981, Theorem 3.1.
If $X \sim N(\theta, 1)$, $\theta \in \Theta = [-m, m]$, $0 \le m \le m_0$,
then $\delta_m^0(x) = m \tanh(mx)$ is minimax for squared-error loss, and $\tau_m^0$ is
a least-favorable prior.
\end{Theorem}

\noindent
{\bf Proof.}
The two lemmas show that 
for $m \le m_0$, $\max_{\theta \in \Theta} R(\theta, \delta_m^0) = R(m, \delta_m^0)$.
But for the prior $\pi_m^0$ that assigns mass $1/2$ to $\pm m$, the Bayes risk is
\beq
r(\pi_m^0, \delta_m^0) = \frac{1}{2} R(-m, \delta_m^0) + \frac{1}{2} R(m, \delta_m^0)
= R(m, \delta_m^0).
\eeq
We have already seen that if the Bayes risk of the Bayes estimator for a given prior
equals the maximum risk of the Bayes estimator over the parameter set, the
Bayes estimator is minimax. We are done.

Casella and Strawderman go on to show that $\pi_m^0$ is not least favorable for $m > m_0
\approx 1.05$.
As $m \uparrow$, the least favorable prior concentrates at a larger and larger number
of discrete points in $[-m, m]$.  The three-point priors $\pi_m^\alpha$ are minimax for
some range of values of $m$, including at least about $m \in [1.4, 1.6]$.





\end{document}
