\input{../courseDefs}

\begin{center}
 First Set of Notes
\end{center}
\section{Hypothesis Testing and Confidence Sets}
\subsection{Set-up}
We are to collect a vector of data $X \in \calX$, which has probability
distribution $\Prob_\theta$, with (possibly infinite-dimensional)
parameter $\theta$ unknown, except that $\theta \in \Theta$,
where $\Theta$ is a known set.
Typically, $\calX = \bfRn$, but it might instead be a more general 
measurable space of 
possible observations. 
We are interested in making statistical inferences about $\tau(\theta)$,
which might be $\theta$ itself, or a function of $\theta$ (for example,
for a univariate normal we might have $\theta = (\mu,\sigma^2)$, and be
interested in $\tau(\theta) = \mu$).
Let 
\beq
\Tau \equiv \tau(\Theta) = \{ \gamma : \exists \eta \in \Theta {\rm s.t.} \gamma = \tau(\eta)
\},
\eeq
and 
\beq
\Prob_\Theta = \{ \Prob_\eta: \eta \in \Theta \}.
\eeq
We wish to test the {\em null hypothesis}
$H: \tau(\theta) \in \Tau_H \subset \Tau$ against an alternative $K$ not yet specified.
In a deliberate ``overloading'' of notation, let $H$ also stand for
$\{ \Prob_\eta, \eta \in \Theta : \tau(\eta) \in \Tau_H \}$ (
the set of probability distributions
for which the null hypothesis $H$ is true),
and let $K$ also stand for
$\{ \Prob_\eta, \eta \in \Theta : \tau(\eta) \in \Tau_K \}$ 
(the set of probability distributions
for which the alternative hypothesis $K$ is true).
We shall typically assume that $H \cup K = \Prob_\Theta$.

\begin{Definition}{}
If $\{ \Prob_\eta \in H \}$ be a singleton set
(just one distribution), we say the null hypothesis $H$ is
{\em simple}. If the alternative $K$ be a singleton set, we say $K$ is {\em simple}.
If an hypothesis is not simple, it is {\em composite}.
\end{Definition}


\begin{Definition}{}
A (significance) level $\alpha$ test of the hypothesis 
$\tau(\theta) \in \Tau_H$ is a (possibly random) measurable decision rule 
$\delta(X) : \calX \rightarrow \{\mbox{ accept, reject}\}$ such that
\beq
\sup_{\{\Prob_\eta \in H\}}\Prob_\eta \{\delta(X) = \mbox{reject} \} \le \alpha.
\eeq
\end{Definition}
The constant $\alpha$ is (an upper bound on) the probability of a false rejection.

The most common decision rules (deterministic rules)
reject when the data $X$ fall outside a set
$A = A_H$ that satisfies
\beq
\sup_{\{\Prob_\eta \in H\}} \Prob_\eta 
\{ X \not\in A_H \} \le \alpha,
\eeq
The set $A_H$ is called the
{\em acceptance region} of the test; $A_H^C$ is the
{\em rejection region} of the test.
Under the Neyman-Pearson paradigm, the term ``acceptance region'' is a
misnomer---one never ``accepts'' the null hypothesis; one merely
fails to reject it given certain data (evidence) $X$.
I shall often blur the notational distinction between a test and its
acceptance region.

Another family of decision rules performs a random experiment that depends on the
observed value of $X$, such that for each $x$, the null hypothesis is rejected with
probability $\phi(x)$ and not rejected with probability $1-\phi(x)$.
To have a significance level $\alpha$ randomized test, we need
\beq
\sup_{\{\Prob_\eta \in H\}} E_\eta \phi(X) = \int \phi(x) d\Prob_\eta(x) \le \alpha.
\eeq
Deterministic rules correspond to decision functions $\phi$ that take only the values
0 (do not reject, with probability 1) and 1 (reject, with probability 1).

Typically, the set $A_H$ is defined in two steps: first, one selects
a statistic
$T(X)$ (a function of $X$ that is $\Prob_\gamma$-measurable for all
$\gamma \in \Theta$, and that does not depend on $\theta$), then 
one defines a subset $A_{T_H}$ of the range of $T$, with the 
property that 
\beq
\sup_{\{\Prob_\eta \in H\}} \Prob_\eta \{ T(X) \not\in 
A_{T_H} \} = 
\alpha.
\eeq
Thus $A_H$, a subset of $\calX$, is the pre-image under
$T$ of $A_{T_H}$, a subset of the range of $T$.
(In symbols, $A_H = T^{-1}(A_{T_H})$.)

Suppose that the range $\calX$ of $X$ is endowed with a distance
\begin{eqnarray}
d(\cdot, \cdot) : \calX \times \calX & \rightarrow & \bfR^+ \nonumber \\
(x, y) & \mapsto & d(x, y) ,
\end{eqnarray}
where $\bfR^+$ are the nonnegative reals.
(Recall that a distance $d(\cdot, \cdot)$ on a set $\calX$ must satisfy
\begin{enumerate}
\item $0 \le d(x, y) \le \infty$; $d(x, y) = 0 \iff x = y$ (positive definiteness)
\item $d(x, y) = d(y, x)$ (symmetry)
\item $d(x, z) \le d(x, y) + d(y, z)$ (triangle inequality)
\end{enumerate}
for all $x, y, z$ in $\calX$.)

\begin{Definition}{}
The {\em diameter} of a set $A$ on which a metric
$d$ is defined is
\beq
|A| \equiv \sup_{x, y \in A} d(x,y).
\eeq
The {\em radius of $A$ relative to the point $x$} is
\beq
|A|_\theta \equiv \sup_{y \in A} d(x, y) .
\eeq
\end{Definition}

One natural criterion of optimality of an acceptance region is
that its diameter be minimal.
This is related to (but not equivalent to) the power of the test
against a family of alternatives; {\em vide infra}.

\begin{Definition}{}
A {\em family of tests for $\tau \in \Tau$} is a set-valued
function $A_\gamma$ such that for each $\gamma \in \Tau$, 
$A_\gamma$ is
the acceptance region for a level $\alpha$ test of the 
hypothesis $H: \tau = \gamma$.
\end{Definition}

{\bf Examples.}
\begin{enumerate}
\item
Suppose that $\Prob_\theta$ is the normal distribution with mean
$\theta$ and unit variance, that $\Theta = \bfR$, $\tau(\theta) = \theta$,
and that we 
observe $X \sim \Prob_\theta$. 
Let $z_\lambda$ be the $\lambda$ critical
value of the standard normal distribution; that is,
\beq
\Prob_0 \{ X \ge z_\lambda \} = \lambda.
\eeq
Then 
\beq
A_\gamma \equiv (\gamma - z_{\alpha/2}, \gamma + z_{\alpha/2})
\eeq
is a family of level $\alpha$ tests for $\tau(\theta) = \theta \in \bfR$.

\item
Suppose $\Prob_\Theta$ is the family of distributions on $\bfR$ that
are continuous with respect to Lebesgue measure.
Let $\tau(\theta)$ be the 90th percentile of the distribution parametrized by
$\theta$. We observe $X = \{X_j\}_{j=1}^n$ i.i.d. $P_\theta$.
Let $T_\gamma: \bfRn \rightarrow \bfN$ equal $\# \{ X_j \ge \gamma \}$.
($\bfN$ are the nonnegative integers).
For all $\nu$ such that $\tau(\nu) = \gamma$, the probability distribution of
$T_\gamma(X)$ is binomial with parameters $n$ and $p = 0.1$.
Thus for any $\gamma$, we can find integers $a_- = a_-(\gamma, n, \alpha)$ 
and $a_+ = a_+(\gamma, n, \alpha)$ such that
\beq
\Prob_\nu \{ T_\gamma(X) \not\in [a_-, a_+] \} \le \alpha 
\;\; \forall \nu {\rm s.t.} \tau(\nu) = \gamma.
\eeq
Such a pair of mappings defines a family of level $\alpha$ tests for $\tau(\theta) \in \bfR$.

\item
Suppose that $\Prob_\Theta$ is the set of probability distributions
on $\bfR$
that are continuous with respect to Lebesgue measure; let $\theta$ be
the distribution function of the ``true'' measure, and suppose
we are interested in $\tau(\theta) = \theta$.
We observe $X = \{X_j \}_{j=1}^n$ i.i.d.
$\Prob_\theta$.
Let $\hat{\theta}_n$ denote the empirical distribution
\beq
\hat{\theta}_n \{(-\infty, x]\} \equiv \frac{1}{n} \sum_{j=1}^n 1_{x \ge X_j},
\eeq
where $1_{B}$ is the indicator function of the event $B$.
For any two probability distributions $\Prob_1$, $\Prob_2$, on 
$\bfR$, define the
Kolmogorov-Smirnov distance
\beq
d_{KS}(\Prob_1, \Prob_2) \equiv 
\| \Prob_1 - \Prob_2 \|_{KS} \equiv \sup_{x \in \bfR} | 
\Prob_1\{(-\infty, x]\} - \Prob_2\{(-\infty, x]\} |.
\eeq
There exist universal constants $\chi_\alpha$
so that for every continuous (w.r.t. Lebesgue measure) distribution
$\theta$,
\beq
\Prob_\theta 
\left \{ \| \theta - \hat{\theta}_n \|_{KS} \ge \chi_n(\alpha) \right \}
= \alpha.
\eeq
This is the Dvoretzky-Kiefer-Wolfowitz inequality.
Moreover, Massart ({\em Ann. Prob., 18}, 1269--1283, 1990) showed that the constant
\beq
\chi_n(\alpha) \le \sqrt{\frac{\ln \frac{2}{\alpha}}{2n}}
\eeq
is {\em tight}.
For $y = (y_1, \cdots, y_n) \in \bfR^n$,
let $\hat{y}_{n}$ be the probability measure on $\bfR$
whose distribution function is 
$1/n \sum_{j=1}^n 1_{x \ge y_j}$.
Then 
\beq
A_\gamma \equiv \{ y \in \bfR^n : 
\| \gamma - \hat{y}_n \|_{KS} \le \chi_\alpha \}
\eeq
is a family of level $\alpha$ tests for $\theta \in \Theta$.

\end{enumerate}


\subsection{Most Powerful Tests}
\begin{Definition}{}
The {\em power $\beta$ of the test $\delta$ of $H$ against the alternative $K$} is
\beq
\beta = \beta(\delta, K) \equiv \inf_{\Prob_\nu \in K} \Prob_\nu \{ \delta(X) = \mbox{reject} \} .
\eeq
That is, $\beta(\delta, K)$ is the smallest probability of rejecting the null 
hypothesis when the value
of the parameter of interest, $\tau(\theta)$, is in the alternative set $\Tau_K$.
\end{Definition}

In the Neyman-Pearson paradigm for hypothesis testing, one is concerned with the probabilities
of two kinds of errors: rejecting the null hypothesis $H$ when it is in fact true (a Type I
error), and failing to reject the null hypothesis when it is in fact false (a Type II error).
The significance level of a test is a bound on the probability of a Type I error; the
power of the test against the alternative $K$ is 
$1 - \sup_{\Prob_\nu \in K} \Prob_\nu \{\mbox{Type II error}\}$.

For a given bound $\alpha$ on the chance of a Type I error, one is naturally led to maximize
the power $\beta(K)$. 
This can be thought of as a more general statistical decision problem with two zero-one
loss functions:
Define 
\beq
L_1 (\theta, \mbox{reject}) = \left \{
\begin{array}{ll}
0, & \Prob_\theta \not \in H \\
1, & \Prob_\theta \in  H
\end{array}
\right .
\eeq
\beq
L_1 (\theta, \mbox{accept}) = 
0, \forall \theta \in \Theta,
\eeq
and
\beq
L_2 (\theta, \mbox{reject}) = 0, \forall \theta \in \Theta,
\eeq
\beq
L_2 (\theta, \mbox{accept}) = 
\left \{
\begin{array}{ll}
0, & \Prob_\theta \in H \\
1, & \Prob_\theta \not \in  H
\end{array}
\right .
\eeq
Then the problem of finding the most powerful test is to find the
decision rule $\delta$ that minimizes $EL_2(\theta, \delta(X))$ subject to
the constraint $EL_1(\theta, \delta(X)) \le \alpha$.

For the case $H$ and $K$ are simple, 
let $\Prob_H = H$ and $\Prob_K = K$.
Considering first nonrandomized tests, one wants to find $A_H$ to maximize
\beq
\beta = \int_{x \not \in A_H} d\Prob_K(x)
\eeq
subject to
\beq
\int_{x \not \in A_H} d\Prob_H(x) \le \alpha.
\eeq
Subject to a bound on the chance of a Type I error, the best points to exclude from
$A_H$
are those that are most probable under $K$ relative to their probability under $H$.
Let $r(x) = d\Prob_K(x) / d\Prob_H(x)$.
Then the most powerful nonrandomized level $\alpha$ test $\delta$ has 
\beq \label{eq:lrtest}
A_H = \{ x: r(x) > c \},
\eeq
where $c$ solves
\beq
\Prob_H \{ X \not\in A_H \} = \int_{x: r(x) > c} d\Prob_H(x) = \alpha.
\eeq
If $\Prob_H$ contains atoms, it can happen that for some values of $\alpha$,
the most
powerful deterministic decision rule $\delta$ that attains exactly level $\alpha$
is not given by the likelihood ratio region \ref{eq:lrtest} for some 
special values of $\alpha$ (for a given value of $c$, the level would be
too large, while for infinitesmaly larger $c$, the level would be too small).
If one allows
randomized decisions, that problem does not occur; one makes a deterministic decision when
$r < c$ or $r > c$, and makes a random decision for $r=c$, with probability of rejection chosen
s.t. the overall level is $\alpha$.
A more common approach (essentially ubiquitous in practice)
is to choose $\alpha$ to avoid such pathology.

\begin{Theorem}{Fundamental Lemma of Neyman and Pearson}
(See Lehmann, TSH, 3.2, Theorem 1.)
Suppose $\Prob_H$ and $\Prob_K$ have densities $p_H$ and $p_K$ relative to a measure
$\mu$ ({\em e.g.}, $\Prob_H + \Prob_K$).
Then
\begin{enumerate}
\item
There is a decision function $\phi$ and a constant $c$ such that
\beq \label{eq:lp1}
E_H \phi(X) = \alpha,
\eeq
\beq \label{eq:lp2}
\phi(x) = \left \{
\begin{array}{ll}
1, & p_K(x) > cp_H(x) \\
0, & p_K(x) < cp_H(x).
\end{array}
\right .
\eeq
(The value of $\phi$ for $p_K(x) = cp_H(x)$ is adjusted to give $E_H \phi(X) = \alpha$;
depending on $\alpha$, $H$, and $K$, this can result in a randomized decision rule.)

\item 
If a decision function $\phi$ satisfies \ref{eq:lp1} and \ref{eq:lp2} for some $c$,
it is most powerful for testing $H$ against $K$ at level $\alpha$.

\item
If $\phi$ is the most powerful decision function for testing $H$ against $K$,
then for some $c$
it satisfies \ref{eq:lp2} a.e.$(\mu)$, and it satisfies \ref{eq:lp1} unless there
is a level $< \alpha$ test of $H$ against $K$ with $\beta = 1$.
\end{enumerate}
\end{Theorem}

The fundamental lemma of Neyman and Pearson applies just to simple null 
and alternative hypotheses.
One might hope that when $H$ and $K$ were composite, the same test would be most
powerful for all $\Prob_\eta \in H$ against all $\Prob_\eta \in K$; 
unfortunately, that is not typically the
case. 
Such a test, when it exists, is called {\em uniformly most powerful} (UMP).

There is an important
class of distributions with real parameters for which UMP tests exist.
Suppose $\Prob_\eta$, 
$\eta \in \Theta = \bfR$ 
has density
$p_\eta(x)$. 

\begin{Definition}
The set of densities $p_\eta$ has {\em monotone likelihood ratio (in $T(x)$)} 
if there exists a 
function $T: \calX \rightarrow \bfR$ such that for $\nu < \eta$
\begin{enumerate}
\item
$\Prob_\nu \ne \Prob_\eta$, and
\item
$p_\eta(x)/p_\nu(x)$ is a monotone non-decreasing function of $T(x)$.
\end{enumerate}
\end{Definition}

\begin{Theorem}
(See Lehmann, TSH, 3.3, Theorem 2.)
Suppose $\theta \in \Theta = \bfR$ and $X$ has density $p_\theta(x)$ 
with monotone likelihood ratio in $T(x)$. 
Let $H = \{ \Prob_\eta : \eta \le \eta_H \}$
and $K = \{ \Prob_\eta : \eta > \eta_H\}$. 
(Such a $K$ is called a {\em one-sided alternative}.)
Then
\begin{enumerate}
\item
A UMP level $\alpha$ test of $H$ against $K$ exists.
\item
The decision function $\phi$ for the UMP test is 
\beq
\phi(x) = \left \{ \begin{array}{ll}
1, & T(x) > c \\
b & T(x) = c \\
0, & T(x) < c ,
\end{array}
\right .
\eeq
with $b$ and $c$ chosen to satisfy
\beq
E_{\Prob_{\eta_H}}\phi(X) = \alpha .
\eeq
\item
For this test, the power 
\beq
\beta(\Prob_\theta) = E_{\Prob_\theta} \phi(X)
\eeq
is a strictly increasing function of $\theta$ at all points for which $0 < \beta(\Prob_\theta) < 1$.
\item
For all $\gamma$, this test is UMP for testing $\theta \le \gamma$ against $\theta > \gamma$
at level $\beta(\Prob_\gamma)$.
\item
For any $\theta < \eta_H$, the test minimizes $\beta(\Prob_\theta)$ among all level $\alpha$ tests.
\end{enumerate}
\end{Theorem}

\begin{Definition}
Let $\Prob_\theta$, $\theta \in \Theta \subset \bfR$ have density
\beq
p_\theta(x) = C(\theta) e^{Q(\theta)T(x)}h(x)
\eeq
relative to some measure $\mu$.
Then $\{ \Prob_\theta : \theta \in \Theta\}$ is a {\em one parameter exponential family}.
\end{Definition}

\noindent
{\bf Remark.}
The one-parameter exponential families have monotone likelihood ratio in $T(x)$ if
$Q(\theta)$ is strictly monotone.

\noindent
{\bf Remark.}
Lehmann refers to a converse due to Pfanzagl (1968) that under weak regularity conditions,
if there exist level $\alpha$ UMP tests against one-sided alternatives for all sample sizes,
$\Prob_{\Theta}$ is an exponential family.
 
\subsection{Confidence Regions.}

\begin{Definition}{}
A $1-\alpha$ confidence region for $\tau(\theta)$ is a random set $S(X) \subset \Tau$
satisfying
\beq
\Prob_\theta \{ S(X) \ni \tau(\theta) \} \ge 1- \alpha.
\eeq
\end{Definition}

The most common way to construct a $1-\alpha$ confidence region for $\tau(\theta)$ is
by ``inverting'' a family of tests for the hypotheses $\tau(\theta) = \gamma$:

\begin{Theorem}{Duality between Tests and Confidence Regions.}
(See Lehmann, TSH, 3.5, Theorem 4).
Let $A_\gamma$ be a family of acceptance regions for level $\alpha$ tests of
the hypotheses $\tau(\theta) = \gamma$. For each value of $x \in \bfRn$, define
\beq
S(x) = \{ \gamma \in \Tau : x \in A_\gamma \} .
\eeq
Then $S(X)$ is a confidence region for $\tau(\theta)$ with confidence level $1-\alpha$.
\end{Theorem}

\begin{Theorem}{The Ghosh-Pratt Identity.}
(See Pratt, J.W., 1961. Length of confidence intervals,
 {\em JASA, 56}, 549--567; Ghosh, J.K., 1961.
On the relation among shortest confidence intervals of different types, 
{\em Calcutta Stat. Assoc. Bull.}, 147--152.)
For a set $S(x) \subset \Theta$, let
\beq
\mu(S(x)) \equiv \int_{\gamma \in S(x)} d\mu(\gamma),
\eeq
for some measure $\mu$ on $\Theta$.
Then
\beq
E_{\Prob_\eta}\mu(S(X)) = \int \Prob_\eta \{S(X) \ni \gamma \} d\mu(\gamma).
\eeq
\end{Theorem}

The Ghosh-Pratt identity relates the expected ``volume'' (w.r.t. the measure $\mu$)
of a confidence set to the
probability that points other than the true parameter are in the set:
the right hand side is the integral of the ``false coverage'' probability.
That is in turn related to the power of the tests to which $S$ is dual
against the alternative with respect to which the expectation and the
probability are calculated. 
For example, suppose that $\Theta = \bfR^m$, that $\mu$ is Lebesgue measure 
(so the expectation on the left is the ``ordinary'' expected volume 
of the confidence set) and that
$S$ is the dual of a family of tests that are most powerful against the alternative
$\theta = {\bf 0}$. That is, the sets $A_\nu$ minimize 
$\Prob_\nu \{{\bf 0} \ni A_\nu \}$.
Then the confidence set $S(X)$ has minimal expected volume when the true value of
$\theta$ is $\bf 0$ among all confidence sets.

Brown, Casella and Hwang (Optimal Confidence Sets, Bioequivalence, and the
Limacon of Pascal, Brown Univ. Tech. Rept. BU-1205-M, 1993, rev.1994) 
use this result to develop confidence sets
for assessing bioequivalence. Each test in the family
is that of a simple null hypothesis
against the simple alternative $\theta = 0$, so a likelihood ratio
test is most powerful; Brown and Hwang 
obtain closed-form expressions for the shape of the resulting 
confidence sets.

\noindent
{\bf Problem.} Find a formula for a $1-\alpha$ confidence set for the mean of
a Poisson distribution from $n$ i.i.d. observations, with minimal expected volume
when the true mean $\theta = 1$. Is the set always an interval?
Give the confidence set that results
when $X = 2$. It might help to read Brown {\em et al.}





\end{document}

