\input{../courseDefs}

\begin{center}
Third Set of Notes
\end{center}
\section{More on Equivariant Confidence Sets}

\begin{Definition} Equivariant Confidence Set.
Suppose that the set of distributions $\Prob_\Theta$ on $\calX$ is preserved under the
group $\calG$, and let $\bar{\calG}$ be the group of transformations on $\Theta$ induced
by the action of $\calG$ on $\calX$.
Suppose that the action of $\calGbar$ on the component $\tau(\nu)$ of the more general
parameter $\nu$ depends only on $\tau(\nu)$; that is, 
$\tau(\gbar(\nu)) = \tau(\gbar(\gamma))$ if $\tau(\nu) = \tau(\gamma)$.
For each $g \in \calG$, let $\tilde{g}S = \{ \tau(\gbar(\nu)): \tau(\nu) \in S \}$.
If $S(x)$ is such that 
\beq
\tilde{g}S(x) = S(gx) \;\; \forall x \in \calX, g \in \calG ,
\eeq
we say $S$ is equivariant under $\calG$.
\end{Definition}

Lehmann (TSH, Ch. 6.11) gives several examples of equivariant confidence sets; the
following is taken from there.

\noindent
{\bf Example.}
Suppose $X = (X_1, X_2)$ has independent, unit variance, normally distributed
components with mean $\theta = (\theta_1, \theta_2)$.  Let $\calG$ be the group of
rigid motions of the plane (translations and rotations, but not ``distortions'').
That is 
\beq
\Prob_\Theta = \{ \mbox{ bivariate normal distributions with independent,
unit variance
components, and mean }\; \theta \in \bfR^2 \},
\eeq
and $\Theta = \bfR^2$.
The sample space is $\calX = \bfR^2$ as well.
The transformation $\bar{g} \in \calGbar$
on $\gamma = (\gamma_1, \gamma_2) \in \Theta$ 
induced by the action of $g \in \calG$ on $x = (x_1, x_2) \in \calX$ is
just $g$ itself.
One equivariant family of confidence sets for $\theta$ is
\beq
S(x) = \{ \gamma \in \Theta : (x_1 - \gamma_1)^2 + (x_2 - \gamma_2)^2 \le c \}
\eeq
(circles centered at the data).
To see this, note that for $g$ a rigid-body transformation of the plane,
$\gamma \in S(x) \Longleftrightarrow \bar{g}\gamma \in S(gx)$.
Thus
\begin{eqnarray}
\tilde{g}S(x) &=& \{ \bar{g}\gamma \in \bfR^2 : \gamma \in S(x) \} \nonumber \\
&=& \{ g\gamma : \gamma \in S(x) \} \nonumber \\
&=& \{ g\gamma : g\gamma \in S(gx) \} \nonumber \\
&=& \{ \nu : \nu \in S(x) \} \nonumber \\
&=& S(x).
\end{eqnarray}


\begin{Definition}
A family of $1-\alpha$ confidence sets for $\tau(\theta)$ is {\em uniformly most
accurate equivariant under $\calG$} if it minimizes
\beq
\Prob_\theta \{ \tau(\gamma) \in S(X) \} \;\; \forall \gamma \in \Theta \mbox{ s.t. }
\tau(\gamma) \ne \tau(\theta) 
\eeq
among all confidence sets $S(x)$ that are equivariant under $\calG$.
\end{Definition}


\begin{Lemma}
Lehmann, TSH, 6.11 Lemma 5.
Suppose that for each $\nu \in \Tau = \tau(\Theta)$ 
there is a group $\calG_\nu$ of transformations
under which the problem of testing $H: \tau(\theta) = \nu$.  
Let $\calG$ be the group of
transformations generated by $\{\calG_\nu \}_{\nu \in \Tau}$.
Suppose $S(x)$ is a $1-\alpha$ confidence procedure that is
equivariant w.r.t. $\calG$.
Let $A(\nu) = \{ x : \nu \in S(x) \}$.
\begin{enumerate}
\item
The set $A(\nu)$ is the acceptance region of a level $\alpha$ test
of $H$, and it is invariant under $\calG_\nu$ for each $\nu \in \Tau$.
\item
If, for each $\nu$, $A(\nu)$ is a UMP invariant level $\alpha$ test of
$H: \tau(\theta) = \nu$, $S(x)$ are confidence level $1-\alpha$
uniformly most accurate equivariant (w.r.t. $\calG$) confidence
sets.
\end{enumerate}
\end{Lemma}
Thus if one has a family of tests $A(\nu)$ that are UMP and invariant w.r.t. $\calG_\nu$,
and $S(x) =  \{ \nu \in \Tau : x \in A(\nu) \}$, then $S(x)$ are uniformly most accurate
equivariant.
That is, uniformly most accurate equivariant confidence sets result from inverting
a family of UMP invariant tests.

However, not all problems admit uniformly most accurate equivariant confidence sets,
or UMP invariant tests.

\noindent
{\bf Example.} (Lehmann, 6.12 Ex. 20.)
Suppose $X = (X_1, \cdots, X_n)$ is an i.i.d. sample from a univariate normal distribution,
with parameter $\theta = (\mu, \sigma^2) \in \bfR \times \bfR^+$ unknown.
We want to estimate $\tau(\theta) = \sigma^2$.
This problem is invariant under the group $\calG$ whose elements
translate of all of the components of $X$ by the same
constant $a$.
The statistic $S^2 = \sum_j (X_j - \bar{X})^2$ is sufficient for $\sigma^2$, and is
invariant under $\calG$.
The problem of finding a confidence set for $\sigma^2$ is invariant under positive
scale changes:
$X_j \mapsto bX_j$, $S \mapsto bS$, $\sigma \mapsto b\sigma$, for $b>0$.
If $\sigma^2 \in C(S^2)$ (note change of confidence set to $C$ to keep the
traditional definition of $S$) is an equivariant family of confidence sets,
we need $b^2 C(S^2) = C(b^2S^2)$, which gives
\beq \label{eq:sigma1}
\sigma^2 \in C(S^2) \Leftrightarrow \sigma^2/S^2 \in 1/S^2 C(S^2) = C(1).
\eeq
Thus for a confidence set to be equivariant, it must be of the form
\beq
\sigma^2/S^2 \in C(1),
\eeq
where
\beq \label{eq:sigma2}
\Prob_{(\mu, 1)} \{ S^{-2} \in C(1) \} = 1-\alpha.
\eeq

This family of confidence sets does not contain one that minimizes the 
false coverage probability.


\noindent
%<a name="problem_2"></a>
{\bf Assignment 2.}
Show that there is no uniformly most accurate confidence set among
the collection of confidence sets that satisfy \ref{eq:sigma1} and
\ref{eq:sigma2}.

This leads one to consider other loss functions for confidence regions
(such as the expected volume, which you explored using the Ghosh-Pratt
identity in the last assignment).

Lehmann (TSH, 6.12) discusses some optimality measures to use in conjunction with
the restriction to equivariant confidence sets, for example, minimizing the measure
\beq
\int_{C(1)} d \nu
\eeq
(with $\nu$ Lebesgue measure on $\bfR$),
or
minimizing the scale-invariant measure
\beq
\int_{C(1)} \nu^{-1} d\nu.
\eeq
This measure has the advantage that if the optimal confidence interval for
$\sigma$ is $(\sigma_-, \sigma_+)$, then the optimal confidence interval
for $\sigma^r$ is $(\sigma_-^r, \sigma_+^r)$.

Even when there is no group structure to the problem, considering
similar measures of the
size of a confidence set can lead to practical confidence sets.
For example, consider estimating the mean $\theta$ of a univariate, unit-variance
normal from the
observation $X \sim N(\theta, 1)$. We have $\Theta = \bfR$ and $\calX = \bfR$.
Suppose we want to minimize among all confidence {\em intervals} $S(X)$ the maximum
expected length of the interval, whatever be $\theta$.
That is, define $R(\theta, S) = E_{\Prob_\theta} |S(X)|$, where $|S|$ is the diameter of
$S$.
The minimax procedure $S^*$ minimizes
\beq
\sup_{\theta \in \Theta} R(\theta, S)
\eeq
among all $\Prob_\Theta$-measurable mappings from $\calX = \bfR$ to intervals of $\bfR$,
which we might parametrize by the two (measurable) functions $\ell$, $u$ that map $x$ to 
$S(x) = (\ell(x), u(x))$.  
Then one way to pick a confidence interval procedure is to minimize
\beq
\sup_{\theta \in \Theta} R(\theta, S) = 
\sup_{\theta \in \Theta}E{\Prob_\theta} |u(X) - \ell(X)|,
\eeq
subject to
\beq
\inf_{\theta\in\Theta} \Prob_\theta \{ (\ell(X), u(X)) \ni \theta \} .
\eeq
If we let $A(\nu) = (\nu - z_{\alpha/2}, \nu + z_{\alpha/2})$, the smallest possible
range of observations will be in the acceptance region of each $\nu$, among level
$\alpha$ tests.
On inverting the tests to get confidence intervals, we would get $\ell(x) = x - z_{\alpha/2}$
and $u(x) = x + z_{\alpha/2}$.
If we had chosen the acceptance region differently for some $\nu$ (for example,
picking the region to be an asymmetric interval about $\nu$ subject to the level $\alpha$
restriction), the acceptance region would
have ``reached'' to more distant observations, and there would have been some values of $x$
that produced longer confidence intervals.  If the set on which we chose asymmetrical intervals
had strictly positive measure, this would result in a larger value of the expected length for
some value of $\nu$. (For other values, we might have ended up with expected length less than
$2 z_{\alpha/2}$, but we are interested in the maximum expected length.)
{\em This is not a proof,} but it suggests one.

The ``moral,'' if there is one, is that a quite different consideration from group
equivariance or accuracy leads again to the same natural confidence interval.
However, this approach through optimization extends quite generally to situations in which
there is no group structure, in which $\Theta$ is restricted in unusual ways, and in which 
there is no most powerful test to exploit.
The direction this leads is to restrict the class of procedures (confidence sets)
through their functional dependence on the data.
For example, one might restrict attention to confidence sets that are intervals
whose endpoints are affine functionals of the data:
$\ell(x) = a + bx$, $u(x) = c + dx$, or even to intervals
$\ell(x) = -a + bx$, $u(x) = a + bx$.

In general, the restriction to such procedures can cost a lot, in that the minimax risk
over affine procedures might be much larger than the minimax risk over all measurable 
procedures.
However, in some problems, it is possible to bound the ``inefficiency'' of affine 
procedures relative to more general nonlinear ones.

Consider, for example, the problem of estimating the mean $\theta$ of a unit-variance normal
from a single observation $X \sim N(\theta, 1)$, when $\theta$ 
is known to lie in the interval $\Theta = [-c, c]$.
This is called the ``bounded normal mean'' problem.
This problem is a cartoon of many inference problems in science in which there are
extrinsic physical constraints on the parameter of interest.
For example, a spectral absorption coefficient must be between zero and one, and energies
must be nonnegative, and can sometimes be bounded above using physical arguments.
Donoho (1994, {\em Ann. Stat., 22}, 238-270) shows how to reduce some 
inference problems about functionals of an infinite-dimensional parameter to
questions about the bounded normal mean; we shall visit his work more extensively
later.

For the moment, let's consider point estimates that minimize
mean-squared error rather, than interval estimates.
We seek the estimator $\hat{\theta}^*$ that minimizes
\beq
\sup_{\theta \in \Theta} E_\theta (\hat{\theta}(X) - \theta)^2
\eeq
among all $\Prob_\Theta$-measurable estimators 
$\hat{\theta}$, where $E_\theta$ is shorthand for
$E_{\Prob_\nu}$.

The natural estimator ignoring the constraint $\theta \in \Theta = [-c, c]$
has maximum risk 1 over $\Theta$.
The ``truncation'' estimate 
\beq
\hat{\theta}_t (x) \equiv \left \{
\begin{array}{ll}
x, & |x| \le c \\
c, & x > c \\
-c, & x < -c 
\end{array}
\right .
\eeq
has maximum risk for $\theta = 0$, for which the risk is
\beq
\int_{x=-c}^c x^2 \phi(x) dx + 2c^2 \Phi(-c),
\eeq
where $\phi(x)$ is the standard normal density and $\Phi(x)$ is the standard normal
cdf.

The minimax affine estimate has risk
\begin{eqnarray}
\label{eq:affineMinimaxMSE}
R_A &=& \min_{a, b} \max_{\theta \in \Theta} E_\theta | aX + b - \theta |^2 \nonumber \\
&=&
\min_{a, b} \max_{\theta \in \Theta} E_\theta | aX +b - \theta |^2 \nonumber \\
&=& 
\min_{a, b} \max_{\theta \in \Theta} E_\theta | a(X - \theta) + (1-a)\theta + b |^2
\nonumber \\
&=& \min_{a, b} \max_{\theta \in \Theta} \{a^2 \mbox{Var}(X) + 
((1-a)\theta + b)^2\} .
\end{eqnarray}
We have $\mbox{Var}(X) = 1$.
The risk is quadratic in $\theta$ with a positive leading coefficient,
so the maximum will be attained at either $-c$ or $c$:
\begin{eqnarray}
\max_{\theta \in [-c, c]} a^2 + ((1-a)\theta + b)^2 &=&
a^2 + \max \left \{ ((1-a)c + b)^2, (-(1-a)c + b)^2 \right \} \nonumber \\
&\ge& a^2 + ((1-a)c)^2,
\end{eqnarray}
so the optimal value of $b=0$.
Stationarity then gives the optimal $a$ as the solution to
\beq
2a - 2c^2(1-a) = 0 \;\; \Rightarrow a = \frac{c^2}{1+ c^2} ,
\eeq
and thus
\begin{eqnarray}
\min_a \max_{\theta \in \Theta} E_\theta | aX - \theta |^2 & = &
\left [ \frac{c^2}{1+ c^2} \right ]^2 + \left [ 1 - \frac{c^2}{1+ c^2} \right ]^2 c^2
\nonumber \\
&=&
\left [ \frac{c^2}{1+ c^2} \right ]^2 + \left [ \frac{1}{1+ c^2} \right ]^2 c^2 \nonumber \\
= c^2\frac{1 + c^2}{(1+ c^2)^2} \nonumber \\
&=& \frac{c^2}{1+c^2}.
\end{eqnarray}
Donoho, Liu, and McGibbon (1990, {\em Ann. Stat., 18}, 1416-1437) show that
for this problem the minimax affine risk is no larger than 5/4 of the minimax nonlinear
risk.

Let's consider confidence intervals.  Suppose we restrict attention to {\em fixed-length}
confidence intervals.  That is, we consider intervals of the form
\beq
\calI(x)  = \calI(\delta, \chi) = [\delta(x) - \chi, \delta(x) + \chi],
\eeq
where $\chi$ does not depend on the data.
To guarantee $1-\alpha$ coverage probability over $\Theta$, we need
\beq
\inf_{\theta\in\Theta} \Prob_\theta \{ \calI \ni \theta \} \ge 1-\alpha .
\eeq
The risk is $\chi$.
The minimax risk is
\beq
\chi_{N, \alpha}(c) = \inf_{\delta \mbox{ measurable}} \left \{ \chi : 
\inf_{\theta\in\Theta} \Prob_\theta \{ \calI \ni \theta \} \ge 1-\alpha \right \},
\eeq
and the affine minimax risk is
\beq
\chi_{A,\alpha}(c) = 
\min_{a, b} \left \{ \chi : 
\inf_{\theta\in\Theta} \Prob_\theta \{ [aX + b - \chi, aX + b + \chi] \ni \theta \} 
\ge 1-\alpha \right \}.
\eeq
Because the loss (the half-length of the confidence interval)is not random, 
the loss is the same as the risk.
Clearly $\chi_{N,\alpha}(c) \le \chi_{A, \alpha}(c)$.
Because picking $a=0$, $b=0$, allows one to take $\chi = c$,
$\chi_{A,\alpha} (c) \le c$.
Furthermore, taking $a=1$, $b=0$ allows one to take $\chi = z_{\alpha/2}$,
so $\chi_{A,\alpha} (c) \le z_{\alpha/2}$.
Thus
\beq
\chi_{N,\alpha}(c) \le \chi_{A, \alpha}(c) \le \min(c, z_{\alpha/2}).
\eeq
Suppose $c < z_\alpha$.
Then clearly, $\chi_{N, \alpha}(c) \le \chi_{A,\alpha}(c) \le z_\alpha$, because
the interval $[-z_\alpha, z_\alpha]$ always covers.
However, when the constraint is this restrictive, there is nothing better one can
do than pick $[-z_\alpha, z_\alpha]$.

To see this, first note that it suffices to consider $\delta(x)$ monotone in $x$
and symmetric about $x=0$ ($\delta(-x) = - \delta(x)$).
(Why?)
Suppose the optimal (nonlinear) $\chi_{N,\alpha} = \chi <  c < z_\alpha$.
Then there would be no loss in assuming $\delta(x) \le c -\chi$, $x \ge c-\chi$,
$\delta(x) \ge -c + \chi$, $x \le -c$. 
In order to have coverage probability $1-\alpha$ when $\theta = -c$, the interval must
be centered at some value of $x < \chi - c < 0$ whenever $z_\alpha - c > 0$, which
is a contradiction, because of the monotonicity and symmetry requirements.
The linear rule $\delta(x) = 0$ can be used with $\chi = c$ to get 100\% coverage,
so
$\chi_{N, \alpha}(c) = \chi_{A, \alpha}(c) = c$, $c < z_\alpha$.
Furthermore, as the constraint $c \rightarrow \infty$, $\Theta$ becomes less informative,
and $\chi_{A,\alpha}(c)$ and $\chi_{N,\alpha}(c) \rightarrow z_{\alpha/2}$;
Both the minimax and affine minimax risks are clearly monotone in $c$.
Thus for $c < z_\alpha$, $\chi_{N, \alpha}(c) = \chi_{A, \alpha}(c)$, and
for $c \ge z_\alpha$,
\beq
z_\alpha \le \chi_{N,\alpha}(c) \le \chi_{A,\alpha}(c) \le z_{\alpha/2},
\eeq
which implies
\beq
\frac{\chi_{A,\alpha}(c)}{ \chi_{N,\alpha}(c)} \le \frac{z_{\alpha/2}}{z_{\alpha}} .
\eeq
For $\alpha = 0.05$, this ratio is about $1.96/1.645 = 1.19$.
Thus in this problem,
the minimax affine fixed-length 95\% confidence interval is at most about 20\% longer
than the minimax fixed-length 95\% confidence interval.

%<a name="problem_3"></a>
\noindent
{\bf Problem.}
(not assigned)
Show that it indeed suffices to consider monotone, symmetric rules $\delta(x)$ at
which to center a fixed-length confidence interval for the bounded normal mean.

It is a rather unsatisfactory property of the minimax fixed-length interval in
this problem that if the prior information is sufficiently strong ({\em e.g., if
$c < z_\alpha$), the optimal procedure ignores the data and just returns the
prior information.  This is an artifact of looking only at the worst-case
behavior.

In contrast, one might consider intervals whose lengths depend on the data, for
example, the truncation interval
\beq
\calI_t (x) = [x - z_{\alpha/2}, x + z_{\alpha/2} ] \cap [-c, c] .
\eeq
This interval has random length, but the half-length never exceeds
$\min\{c, z_{\alpha/2}$.
A different criterion of optimality one might consider (rather than the fixed-length)
is
\beq
R^+(\calI) \equiv \sup_{\theta \in \Theta} E_\theta | \calI(X)| .
\eeq
This is the largest expected length of the confidence interval.

\noindent
%<a name="problem_4"></a>
{\bf Assignment 3.}
Find $R^+(\calI_t)$ for $\alpha = 0.05$, $c = 1/2, 1, 2, 5, 10$.
Compare with $R^+(\mbox{affine minimax fixed-length procedure})$.
Note that the risk of the affine minimax procedure is just its fixed length,
whether measured by $R^+$ or by the loss function for which it was derived.
{\em Hint.} For what value of $\theta \in \Theta$ is the maximum expected length
attained?

\section{Bayesian Credible Regions}
The Bayesian analogue of a confidence set is a {\em credible region}.
To construct a credible region, one must think of the parameter $\tau(\theta)$ as
itself being a random variable.  Denote its (prior) distribution, which is assumed to
be known completely, by $\Pr$.

\begin{Definition}
A level $1-\alpha$ credible region for the parameter $\tau(\theta)$ is a
set $S(x)$ such that
\beq
\Pr \{ \tau(\theta) \in S(x) | X = x \} \ge 1-\alpha .
\eeq
That is, $S$ is a set such that the posterior probability that $S$ contains
$\tau$, given the data, is at least $1-\alpha$.
\end{Definition}

Note that there is not a unique $S$ with this property; a criterion often used to
obtain a unique $S$ is to take $S$ to be a level set of the posterior distribution
of $\theta$.
Another is to introduce a loss function associated with a measure of the ``size'' or
``volume'' of the confidence set (as we have been discussing in a
frequentist context), and to find the region that minimizes that loss
(or the risk)
subject to the posterior coverage constraint.

\noindent
% <a name="problem_5"></a> Problem 5.
{\bf Assignment 4: Bounded Normal Mean}
Suppose $\Prob_\Theta = \{ N(\gamma, 1): \gamma \in [-a, a]\}$, 
$X \sim N(\theta, 1)$, 
$\Theta = [-a, a]$, $\theta \sim U[-a, a]$.
Let $\calD$ be the set of Lebesgue-measurable subsets of $\bfR$.
Let $L(\theta, d) = \mu(d)$, where $\mu(d)$ is the Lebesgue measure of the set $d$.
(1) Characterize (give equations that determine it) 
a $1-\alpha$ credible region $d$ for $\theta$ that
minimizes $L$ given $X = x$.
(2) Is it sufficient to assume that $d$ is an interval?
(3) Does this approach also minimize the risk $E_{\Pr} L = E_{\Pr} \mu(d)$?
(4) Find the credible region explicitly for $a=1$, $x=0, 2, 5$.
(5) For $a = 1$, find explicitly or estimate
by simulation the risk $E_{\Pr} L = E_{\Pr} \mu(d)$
of this procedure.



\end{document}
