%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%234567890123456789012345678901234567890123456789012345678901234567890123456789
%        1         2         3         4         5         6         7         


\documentclass[letterpaper, 10 pt, conference]{IEEEconf}  % Comment this 
                                                          % line out if
                                                          % you need a4paper
%\documentclass[a4paper, 10pt, conference]{IEEEconf}      % Use this line for 
                                                          % a4 paper
\usepackage{amsfonts}
\IEEEoverridecommandlockouts                              % This command is 
                                                          % only needed if
                                                          % you  want to 
                                                          % use the \thanks 
                                                          %command

% See the \addtolength command later in the file to balance the column lengths 
% on the last page of the document



% The following packages can be found on http:\\www.ctan.org
%\usepackage{graphics} % for pdf, bitmapped graphics files
%\usepackage{epsfig} % for postscript graphics files
%\usepackage{mathptmx} % assumes new font selection scheme installed
%\usepackage{times} % assumes new font selection scheme installed
%\usepackage{amsmath} % assumes amsmath package installed
%\usepackage{amssymb}  % assumes amsmath package installed

\def\fsh{F_{\hat\Sigma}}
\def\axts{{\cal A}_{x,{\cal T}}^{\hat\Sigma}}
\def\vts{{\cal V}_{{\cal T}}^{\hat\Sigma}}
\def\rr{\mathbb{R}}
\def\R{\rr}
\def\Om{\Omega}
\newcommand{\eps}{\varepsilon}
\def\equaldef{{\buildrel \rm def \over =}}
\def\proofend{\hfill$\diamondsuit\,\,$}
\newtheorem{itlemma}{Lemma}[section] %number by section 
\newtheorem{itremark}[itlemma]{Remark}
\newtheorem{itdefinition}[itlemma]{Definition}
\newenvironment{definition}{\begin{itdefinition}\rm}{\end{itdefinition}}
\newtheorem{itproposition}[itlemma]{Proposition}
\newenvironment{proposition}{\begin{itproposition}}{\end{itproposition}}
\newtheorem{theorem}[itlemma]{Theorem}
\newtheorem{itexample}[itlemma]{Example}
\newenvironment{example}{\begin{itexample}\rm}{\end{itexample}}
\newenvironment{remark}{\begin{itremark}\rm}{\end{itremark}} %no-italics
\title{\LARGE \bf
Uniqueness results for the value function via direct
trajectory-construction methods
}
\author{ \parbox{3 in}{\centering H\'ector J. Sussmann\\
Department of Mathematics\\
Rutgers, the State University of New Jersey\\
Hill Center---Busch Campus\\
110 Frelinghuysen Road\\
Piscataway, NJ 08854-8019, USA\\ 
{\tt\small sussmann@hilbert.rutgers.edu}}}



\begin{document}



\maketitle
\thispagestyle{empty}
\pagestyle{empty}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{abstract}

We present some new \mbox{results},
together with a number of particularly \mbox{simple} and user-friendly 
versions of results obtained in \mbox{recent} years by the \mbox{author} 
and \mbox{M.~Malisoff}, on the \mbox{uniqueness} of \mbox{solutions} of the 
Hamilton-Jacobi-Bellman \mbox{equation} (HJBE) 
for \mbox{deterministic} finite-dimensional \mbox{optimal} control
\mbox{problems} under \mbox{non-standard hypotheses}. Our
\mbox{approach} is completely control-theoretic and totally
self-contained, using the systematic \mbox{construction} of 
\mbox{special} \mbox{trajectories} of \mbox{various} kinds, and not
\mbox{involving} any PDE methods.  We do%definitely 
not assume that the \mbox{Lagrangian} is positive, or 
that the \mbox{dynamics} is Lipschitz-continuous.

\end{abstract}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{INTRODUCTION}

\noindent We consider autonomous {\em Lagrangian \mbox{optimization} problems}
involving a {\em state variable} $x$ with values in an open subset 
$\Omega$ of $\rr^n$,  a {\em control variable} $u$ taking values in a 
{\em control space} $U$, and a {\em target set}
${\cal T}$, which is a closed subset of the closure of $\Omega$
disjoint from $\Omega$ itself.  The dynamics is given by
an O.D.E.

\vspace{-6mm}
\begin{equation}\label{dyna}
\dot x=f(x,u)\,,
\end{equation}
the cost functional to be minimized is
\begin{equation}\label{lagra}
J=\int_{\tau_-(\xi)}^{\tau_+(\xi)}L(\xi(t),\eta(t))\,dt\,,
\end{equation}
(where $\tau_-(\xi)$, $\tau_+(\xi)$ are
the initial and \mbox{terminal} times of the trajectory $\xi$),
and for each \mbox{$x\in\Omega$} the \mbox{minimization} is 
over the set $\axts$ of all pairs \mbox{$\Xi=(\xi,\xi_0)$} such that
\begin{itemize}
\item[(i)]  \mbox{$\Xi$} \mbox{consists}  of a trajectory
$\xi$ of (\ref{dyna}) (i.e., a \mbox{locally} \mbox{absolutely} 
continuous (LAC) function  $\xi$ that satisfies $\dot\xi(t)=f(\xi(t),\eta(t))$
for almost all $t$) \mbox{corresponding} to some $U$-valued control 
$\eta$, and a ``\mbox{running} cost'' function $\xi_0$ corresponding to 
$\xi$ and $\eta$ (i.e., a LAC
function  $\xi_0$ such that \mbox{$\dot\xi_0(t)=L(\xi(t),\eta(t))$}
for almost all $t$);
\item[(ii)]  \mbox{$\xi$} starts at $x$,  and ``ends at the target'' in a 
sense  to be defined precisely later.
\end{itemize}
We will call a pair $\Xi=(\xi,\xi_0)$ for which (i) above 
holds an ``augmented trajectory'' of our \mbox{system}, 
\mbox{because} it really is a trajectory of the ``\mbox{augmented} 
\mbox{control} system''  

\vspace{-2mm}
\begin{equation}\label{dynaaug}
\dot x=f(x,u)\,,\qquad\dot x_0=L(x,u)
\end{equation}

\noindent obtained from (\ref{dyna}) by ``adding the cost as
an extra \mbox{variable}'' in a well known way.  We will write
$J(\Xi)$, rather than just $J$, for the left-hand side of
(\ref{lagra}), \mbox{because} it is easy to see that the natural
arguments for our cost functional $J$ are really augmented
\mbox{trajectories}, \mbox{since} (i)~one cannot just regard $J$ as a
functional of $\xi$ \mbox{only}, because the integral of (\ref{lagra})
involves the control $\eta$ as well as $\xi$, but on the other hand
(ii)~the cost is \mbox{completely} \mbox{determined} once we know
$\xi$ and a \mbox{running} cost $\xi_0$, \mbox{because} in that case
$L(\xi(t),\eta(t))=\dot\xi_0(t)$. 

The infimum $V(x)$ of the costs $J(\Xi)$ of all 
\mbox{augmented} trajectories
$\Xi\in\axts$ is the {\em value}  of our problem at $x$. (If 
$\axts$ is empty, then  of course 
\mbox{$V(x)=+\infty$}.)  The function 
$V:\Omega\mapsto\rr\cup\{-\infty,+\infty\}$ is the {\em value
function}.

Our goal is to present uniqueness theorems, \mbox{showing} that a
viscosity solution of the HJBE that satisfies an \mbox{appropriate}
boundary condition is necessarily the value \mbox{function}. 
``\mbox{Uniqueness}'' is to be understood as ``\mbox{uniqueness}
within a class defined by some additional properties,'' such as that
of all functions that are continuous and bounded below.

We will work with a class of systems which is \mbox{sufficiently} general to
capture some new and  \mbox{interesting} \mbox{phenomena},
and restricted \mbox{enough}
to make it \mbox{possible} to prove strong \mbox{theorems}.  In 
particular,  we will asume that the \mbox{sets} 
\mbox{$\fsh(x,U)=\{(f(x,u),L(x,u)):u\in U\}$} are closed and convex,
but will not require them to be \mbox{compact}, and will instead
impose the ``local coerciveness'' \mbox{condition} that
locally,  an inequality of the form $\|f(x,u)\|^r\le L(x,u)+C$,
with $C>0$ and $r>1$,  hold uniformly with respect to $u$.  We 
also require $f(x,u)$ and $L(x,u)$ to be continuous
with respect to $x$, uniformly with respect to $u$ locally in $x$. 

On the other hand, we will most definitely {\em not} require 
that $f(x,u)$ be Lipschitz-continuous with respect to $x$,
since one of the main purposes of this work is to clarify the exact
role of the Lipschitz-continuity assumptions often made in the viscosity
literature.  The answer we  propose is as follows: 
\begin{itemize}
\item[(a)] Without any Lipschitz-continuity hypotheses, one can 
prove, for continuous viscosity solutions $V$ of the HJBE, an 
{\em existence theorem for \mbox{trajectories}}, \mbox{asserting}
that, starting 
at every point of $x$ of $\Omega$, there is a maximally defined
``augmented \mbox{trajectory} of steepest descent,'' that is, 
a maximally defined pair $\Xi=(\xi,\xi_0)$ defined on a interval 
$I$ such that \mbox{$0=\min\,I$}, for which 
the \mbox{inequality} 

\vspace{-3.5mm}
\begin{equation}\label{steepdesc}
V(\xi(t))+\xi_0(t)\ge V(\xi(s))+\xi_0(s)
\end{equation}

\vspace{-2.5mm}
\noindent holds whenever $s,t\in I$ and $s\le t$. 
\item[(b)] As a trivial corollary of the existence of \mbox{steepest} 
\mbox{descent} trajectories (applied to $-V$ and $-L$), we get the 
existence of ``DPI \mbox{trajectories}'' (where ``DPI'' is an acronym 
for ``\mbox{Dynamic} \mbox{Programming} \mbox{Inequality}''), i.e., 
\mbox{augmented}
\mbox{trajectories} \mbox{$\Xi=(\xi,\xi_0)$} along which 
the inequality 

\vspace{-5mm}
\begin{equation}\label{steepasc}
V(\xi(t))+\xi_0(t)\le V(\xi(s))+\xi_0(s)
\end{equation}

\vspace{-2mm}
\noindent (that is, the exact opposite of (\ref{steepdesc}))
is satisfied.
\item[(c)] The existence result of (b) says that for every 
``\mbox{sufficiently} nice'' (e.g., piecewise constant) \mbox{control}
$\eta$ {\em there exists a trajectory of $\eta$ 
with the given \mbox{initial} condition along which the DPI holds}.
This is \mbox{almost}, but not quite, 
what is needed to prove that $V$ is bounded above by the 
value function. 
\item[(d)] The gap between the existence result for DPI 
\mbox{trajectories} and what would actually 
be needed to prove that $V$ is bounded above by the 
value \mbox{function} is that to achieve the latter goal one needs 
the DPI to hold for {\em all} trajectories, and it is not enough
to have just one DPI trajectory for every intial condition and every 
control. 
\item[(e)] The gap described in (d) clearly does not exist when 
there is uniqueness of trajectories for every given control and initial 
condition, and in particular when,
for \mbox{every} admissible control $\eta$,  the corresponding 
time-varying vector field $(x,t)\mapsto f(x,\eta(t))$ 
satisfies a Lipschitz-Carath\'eodory condition that guarantees
uniqueness of trajectories, or a 
weaker condition such as a local bound

\vspace{-5mm}
\begin{equation}\label{weakerbo}
\langle f(x,\eta(t))-f(y,\eta(t)),x-y\rangle 
\le k(t)\|x-y\|^2\,,
\end{equation}

\vspace{-2mm}
\item[] with $k$ integrable.
\item[(h)] Even more generally, the only property that really matters
is that, if we pick a sequence $\{\eta_j\}_{j=1}$ of 
piecewise constant controls such that our \mbox{augmented}
trajectory $\Xi$ can be approximated by  \mbox{augmented}
trajectories $\Xi_j$ corresponding to the $\eta_j$---with, say, 
the same initial condition---then the $\Xi_j$ converge to $\Xi$ uniformly 
{\em no matter how the $\Xi_j$ are chosen}. We call such trajectories 
``uniquely \mbox{limiting},'' and use this concept in the statement of our 
main theorem. 
\end{itemize}


\begin{remark}
Our approach owes a great deal to the book
\cite{subbotin} by A.I. Subbotin.  Subbotin, however, \mbox{only} studies
\mbox{viscosity} \mbox{solutions} of PDEs of the form $F(x,u(x),Du(x))=0$,
where the Hamiltonian $F(x,u,p)$ is required to be globally
\mbox{Lipschitz} with respect to $p$ (cf. \mbox{Equation} (2.2) in
\cite{subbotin}, p.~9).  A somewhat weaker \mbox{hypothesis} is also
considered later but, even with the \mbox{weakened}
\mbox{requirement}, the results do not cover, for example, coercive
problems of the kind discussed here, such as linear quadratic
\mbox{optimal} \mbox{control}. \proofend
\end{remark}


\section{The main theorem}\label{sec-standard}
%\subssection{Systems and augmented systems.} 
\noindent If $n$ is a positive  integer, an {\em $n$-dimensional
control \mbox{system}} is a triple $\Sigma=(\Omega,U,f)$ such that
$\Omega$ (the {\em \mbox{state} space} of $\Sigma$) is an open subset 
of $\R^n$,  $U$ (the {\em control \mbox{space}} of $\Sigma$) 
is a nonempty set, and $f$ (the {\em dynamics} of $\Sigma$) 
is a map \mbox{$\Omega\times U\ni(x,u)\mapsto f(x,u)\in\R^n$}. 

An {\em $n$-dimensional augmented 
control system} is a \mbox{4-tuple} $\hat\Sigma=(\Omega,U,f,L)$ such that
$\Sigma=(\Omega,U,f,L)$ is an $n$-dimensional control system 
and $L$ (the {\em Lagrangian} of $\hat\Sigma$) is a map 
$\Omega\times U\ni(x,u)\mapsto L(x,u)\in\rr$. 
(In that case, the state space, control space, and dynamics 
of $\Sigma$ are also called the state space, control space, and dynamics 
of $\hat\Sigma$.)

An augmented control system $\hat\Sigma=(\Omega,U,f,L)$ is 
{\em \mbox{continuous}} if the maps 
$\Omega\ni x\mapsto f(x,u)\in\R^n$ and 
$\Omega\ni x\mapsto L(x,u)\in\R$ 
are continuous for each fixed $u$. We call $\hat\Sigma$
{\em \mbox{uniformly} continuous} on a subset $S$ of $\Omega$
if there exists a function $\omega:\,]0,\infty[\mapsto[0,\infty]$ 
such that $\lim_{s\downarrow 0}\omega(s)=0$, having the property that
$$
\|f(x,u)-f(y,u)\|+|L(x,u)-L(y,u)|\le\omega(\|x-y\|)
$$
whenever $x,y\in S$, $u\in U$. We call 
$\hat\Sigma$ {\em \mbox{locally} \mbox{uniformly} \mbox{continuous}} if it is 
uniformly \mbox{continuous} on every compact subset of $\Omega$, and
{\em globally  \mbox{uniformly} \mbox{continuous}} if it is 
uniformly continuous on $\Omega$. We call $\hat\Sigma$
{\em Lipschitz continuous} if the maps 
\mbox{$\Omega\ni x\mapsto f(x,u)\in\R^n$} and 
\mbox{$\Omega\ni x\mapsto L(x,u)\in\R$} 
are Lipschitz continuous for each fixed $u$. We call $\hat\Sigma$
{\em \mbox{uniformly} Lipschitz continuous} on a subset $S$ of $\Omega$
if there exists a positive constant $C$ such that
$$
\|f(x,u)-f(y,u)\|+|L(x,u)-L(y,u)|\le C\|x-y\|
$$
whenever $x,y\in S$ and $u\in U$. We call $\hat\Sigma$
{\em \mbox{locally} \mbox{uniformly} Lipschitz continuous} if it is 
uniformly \mbox{Lipschitz} continuous on every compact subset of $\Omega$, and
{\em globally  uniformly \mbox{Lipschitz} continuous} if it is 
uniformly Lipschitz continuous on $\Omega$. 

The augmented control system $\hat\Sigma=(\Omega,U,f,L)$ is {\em
\mbox{coercive}} on a subset $S$ of $\Omega$ if there exist real constants
$r$, $A$, $C$, such that $A>0$, $C>0$, $r>1$, and 
\begin{equation}\label{iscoer}
\|f(x,u)\|^{r}\le AL(x,u)+C\,\,{\rm for\,\,all}\,\,
x\in S\,,\,u\in U\,.
\end{equation}
We call $\hat\Sigma$ {\em locally coercive} if it is coercive on every 
\mbox{compact} subset of $\Omega$, and {\em globally coercive} if 
it is \mbox{coercive} on $\Omega$.

\begin{remark}
If $\hat\Sigma$ is coercive on a set $S$, then {\em it is 
\mbox{always} possible
to choose $C$, $r$, such that $C>0$, $r>1$, and 
(\ref{iscoer}) holds with $A=1$.} \proofend
\end{remark}

For an augmented control system $\hat\Sigma=(\Omega,U,f,L)$, 
we define a map $\fsh:\Om\times U\mapsto\R^{n+1}$
(called the {\em augmented dynamics} of $\hat\Sigma$) by 
letting $\fsh(x,u)=(f(x,u),L(x,u))$ for $x\in\Omega$, $u\in U$. 
We say that $\hat\Sigma$ {\em satisfies the convexity and 
upper semicontinuity condition} if, for each $x\in\Omega$, 
\begin{equation}\label{convus}
\fsh(x,U)=\bigcap_{\eps>0}\overline{{\rm co}}\Big(\bigcup\{\fsh(x',U):
\|x'-x\|\le\eps\}\Big)\,,
\end{equation}
where ``$\overline{{\rm co}}$'' stands for ``closed convex hull,''
and $\fsh(x,U)\equaldef\{(f(x,u),L(x,u)):u\in U\}$.

\begin{remark}
It is easy to see that {\em if $\hat\Sigma=(\Omega,U,f,L)$ is locally 
uniformly continuous, then $\hat\Sigma$ satisfies the \mbox{convexity} and 
upper semicontinuity condition iff
the set  $\fsh(x,U)$ is closed and convex for every $x\in\Omega$.}
\proofend
\end{remark}

A {\em target} for an augmented control system 
\mbox{$\hat\Sigma=(\Omega,U,f,L)$}
is a closed subset ${\cal T}$ of $\R^n$ 
such that \mbox{${\cal T}\subseteq{\rm Closure}\,\Omega$} and 
${\cal T}\cap\Omega=\emptyset$.

%\subsection{Trajectories}

A {\em trajectory} of $\hat\Sigma=(\Omega,U,f,L)$ 
is a LAC  curve 

\vspace{-3mm}
\begin{equation}\label{trajo}
I\ni t\mapsto\xi(t)\in\Omega\,,
\end{equation}

\vspace{-2.5mm}
\noindent defined on a nonempty subinterval $I$ of $\R$, such that 
\mbox{$\dot\xi(t)\in f(\xi(t),U)$} for almost every \mbox{$t\in I$}. 
An {\em augmented trajectory} of $\hat\Sigma$ is a LAC curve 
\begin{equation}\label{augmtrajo}
I\ni t\mapsto\Xi(t)=(\xi(t),\xi_0(t))\in\Omega\times\R\,,
\end{equation}
defined on a subinterval $I$ of $\R$,
having the property that $\dot\Xi(t)\in\fsh(\xi(t),U)$ for
almost every $t\in I$. 

The {\em initial time}, or {\em starting time} of a trajectory $\xi$ 
(\mbox{resp.} an augmented trajectory $\Xi=(\xi,\xi_0)$)  with domain $I$ 
is the time $\tau_-(\xi)\equaldef\min\,I$ (resp.
$\tau_-(\Xi)\equaldef\min\,I$)), if the \mbox{minimum} exists,
i.e., if $I$ is bounded below and \mbox{its} infimum belongs to $I$.  If
the initial time of $\xi$ (\mbox{resp.} $\Xi$) exists, 
then (i)~the point $x_-(\xi)\equaldef\xi(\tau_-(\xi))$
(\mbox{resp.} $x_-(\Xi)\equaldef\xi(\tau_-(\Xi))$)
is the {\em starting point}, or {\em \mbox{initial}  point}, of
$\xi$ (resp. $\Xi$), and (ii)~the ordered pair 
$\partial_-(\xi)\equaldef(\tau_-(\xi),x_-(\xi))$ 
(resp. $\partial_-(\Xi)\equaldef(\tau_-(\Xi),x_-(\Xi))$) is the
{\em initial condition} of $\xi$ (resp.  $\Xi$). 
If $\partial_-(\xi)=(t,x)$ (resp. 
$\partial_-(\Xi)\!=\!(t,x)$), we say that $\xi$ (resp. $\Xi$) 
{\em starts at $x$ \mbox{at time $t$}}. 

If ${\cal T}$ is a target for $\hat\Sigma=(\Omega,U,f,L)$, then 
a trajectory $\xi$ or augmented trajectory $\Xi=(\xi,\xi_0)$
with domain $I$ {\em ends at  ${\cal T}$} if the limit
$\xi(\uparrow)\equaldef\lim_{t\uparrow\sup\,I}\xi(t)$
exists and belongs to ${\cal T}$. 

For each $x\in\Omega$, we let $\axts$ be the set of \mbox{all}
\mbox{augmented} trajectories $\,\Xi=(\xi,\xi_0)$ of $\hat\Sigma$ such that 
(i)~$\partial_-(\Xi)=(0,x)$, (ii)~$\xi_0(0)=0$, (iii)~$\Xi$ ends at the 
target, and (iv)~the limit 
$\xi_0(\uparrow)\equaldef\lim_{t\uparrow\sup\,{\rm domain}\,\Xi}\xi_0(t)$
exists. 

If $\Xi=(\xi,\xi_0)\in\axts$ then the {\em cost}
of $\Xi$ is the number
\begin{equation}\label{thecost}
J(\Xi)\equaldef \xi_0(\uparrow)\,.
\end{equation}

\medbreak The {\em value function} of the optimal control \mbox{problem}
\mbox{defined}
by $\hat\Sigma$ and the target ${\cal T}$ 
is the function 
\mbox{$\vts:\Omega\cup{\cal T}\mapsto\R\cup\{-\infty,+\infty\}$}
given by
$$
\vts(x)=\left\{\begin{array}{lll}
\inf\{J(\Xi):\Xi\in\axts\}&{\rm if}&x\in\Omega\\
0&{\rm if}&x\in{\cal T}\,.\end{array}\right.
$$

\def\sdsv{{\cal SD}_{\hat\Sigma,V}}
\def\sdsvx{{\cal SD}_{\hat\Sigma,V,x}}
\def\msdsv{{\cal MSD}_{\hat\Sigma,V}}
\def\msdsvx{{\cal MSD}_{\hat\Sigma,V,x}}

If $V:\Omega\mapsto\R$ is a function, then 
an \mbox{augmented} trajectory $\Xi=(\xi,\xi_0)$ of $\hat\Sigma$ 
is said to be {\em of steepest descent with respect to $V$} 
if $\xi_0(s)+V(\xi(s))\ge\xi_0(t)+V(\xi(t))$
whenever $s\le t$ and $s,t\in{\rm domain}\,\Xi$.
We use $\sdsv$ to denote the set of all  augmented 
\mbox{trajectories} of $\hat\Sigma$ of steepest descent with respect to $V$,
and $\sdsvx$ to denote the set of all  $\Xi\in\sdsv$ 
such that $\partial_-(\Xi)=(0,x)$. 

\medbreak If $x\in\Omega$, a {\em maximal augmented trajectory of
$\hat\Sigma$ from $x$ of steepest \mbox{descent} with respect to $V$}
is a $\Xi\in\sdsvx$ that \mbox{cannot} be \mbox{extended} to a
$\tilde\Xi\in\sdsvx$ defined on an interval which is strictly larger
that the domain of $\Xi$.  We use $\msdsvx$ to denote the set of all
maximal augmented \mbox{trajectories} of $\hat\Sigma$ from $x$ of steepest
descent with respect to $V$.

\begin{definition}\label{rightunb}.
If $\Omega$ is an open subset of $\R^n$, and 
\mbox{$\xi\!:\!I\!\mapsto\!\Omega$} is a curve, we say that $\xi$ is 
\mbox{{\em right-unbounded} if}
\begin{itemize}
\item[(i)] the interval $I$ is open on the right (that is, 
if \mbox{$\tau=\sup\,I$}, then either (a)~$\tau=+\infty$ or
(b)~$\tau$ is finite and does not belong to $I$),
\item[(ii)] if $\tau$ is finite, then for every compact subset $K$
of $\Omega$ there exists a $\tau_K$ such that $0\le\tau_K<\tau$ and
\mbox{$\xi(t)\notin K$} whenever $\tau_K<t<\tau$. \proofend
\end{itemize}
\end{definition}

\medbreak The following observation is a completely trivial
\mbox{consequence} of Zorn's lemma, \mbox{given} our 
definitions, but we state it explicitly as a
\mbox{separate} \mbox{result} for future \mbox{reference}. 

\begin{proposition}\label{trivialprop}
{\em If $\hat\Sigma=(\Omega,U,f,L)$ is an \mbox{augmented} control system,
$V:\Omega\mapsto\R$ is a function, 
and $x\in\Omega$, then the set $\msdsvx$ of maximal \mbox{augmented}
\mbox{trajectories} of $\hat\Sigma$ from $x$ of steepest \mbox{descent}
with respect to $V$ is nonempty.}\proofend
\end{proposition}


\medbreak An {\em augmented arc} is an augmented trajectory whose 
\mbox{domain} is a compact interval. If $\Xi=(\xi,\xi_0)$ is an 
\mbox{augmented} arc 
with domain $[a,b]$, then an \mbox{{\em improvement}} of $\Xi$ 
is an augmented arc  $\Xi'=(\xi',\xi'_0)$, with domain
$[a',b']$, that satisfies \mbox{$\xi'(a')=\xi(a)$}, $\xi'(b')=\xi(b)$, 
and \mbox{$\xi_0'(b')-\xi_0'(a')\le\xi_0(b)-\xi_0(a)$}.



\medbreak If $\hat\Sigma=(\Omega,U,f,L)$ is an \mbox{augmented} 
control system, then an augmented arc $\Xi=(\xi,\xi_0)$ of
$\hat\Sigma$ with \mbox{domain} $[a,b]$ is 
{\em uniquely limiting} if there exists a sequence 
$\{\eta_j\}_{j=1}^\infty$ of piecewise constant functions
$\eta_j:[a,b]\mapsto U$ such that 
\begin{itemize}
\item[(*)] {\em if $\{\Xi_j\}_{j=1}^\infty$ is an arbitrary sequence
of \mbox{maximally} defined augmented trajectories of $\hat\Sigma$ 
such that \mbox{$a\in{\rm domain}(\Xi_j)$} and $\Xi_j(a)=\Xi(a)$
for every $j$, then $[a,b]\subseteq{\rm domain}(\Xi_j)$ if $j$ is large enough,
and $\Xi_j\to\Xi$ uniformly on $[a,b]$ as $j\to\infty$.}
\end{itemize}

\begin{example}\label{exalips}
Suppose $\Xi=(\xi,\xi_0)$  is an augmented arc  of
$\hat\Sigma$ with \mbox{domain} $[a,b]$ such that 
\begin{itemize}
\item[(\#)] {\em there exist a positive  $\delta\in\rr$, 
a function \mbox{$\eta:[a,b]\mapsto U$}, 
and an integrable function $\varphi:[a,b]\mapsto[0,\infty]$, such that
\begin{itemize}
\item[(i)] $\dot\Xi(t)=\Big(f(x,\eta(t)),L(x,\eta(t))\Big)\,\,$ for a.~e.
$t\in[a,b]$, 
\item[(ii)] the map $t\mapsto\Big(f(x,\eta(t)),L(x,\eta(t))\Big)\,\,$, on the 
\mbox{compact} set 
$I_x\equaldef\{t:a\le t\le b\wedge \|x-\xi(t)\|\le\delta\}$, 
%\item[] $\qquad I_x\equaldef\{t:a\le t\le b\wedge \|x-\xi(t)\|\le\delta\}$, 
\item[] is measurable for each $x\in\Omega$, 
\item[(iii)] the map $x\mapsto\Big(f(x,\eta(t)),L(x,\eta(t))\Big)\,\,$, on the 
\mbox{compact} set $I^t\equaldef\{x\in\Omega:\|x-\xi(t)\|\le\delta\}$, 
%\item[] $\qquad I^t\equaldef\{x\in\Omega:\|x-\xi(t)\|\le\delta\}$, 
\item[] is continuouse for each $t\in[a,b]$, 
\item[(iv)] \mbox{$\Big\langle f(x,\eta(t))-f(x',\eta(t)),x-x'\Big\rangle
\le\varphi(t)\|x-x'\|^2$}
whenever $t\in[a,b]$, $\|x-\xi(t)\|\le\delta$, and
$\|x'-\xi(t)\|\le\delta$,
\item[(v)] the inequality $|L(x,\eta(t))|\le\varphi(t)$ 
holds \mbox{whenever} $t\in[a,b]$ and $\|x-\xi(t)\|\le\delta$.
\end{itemize}}\end{itemize}

\medbreak\noindent Then $\Xi$ is uniquely limiting. \proofend
\end{example}


\medbreak An augmented trajectory $\Xi=(\xi,\xi_0)$ with domain $I$ is 
{\em locally uniquely limiting} if for every compact \mbox{subinterval} 
$I'$  of $I$ the restriction of $\Xi$ to 
$I'$ is uniquely limiting.

\medbreak\noindent
An augmented trajectory $\Xi=(\xi,\xi_0)$ with domain $I$ is {\em almost
uniquely limiting} if there exists 
a finite subset $B$ of $I$ such that the restriction of
$\Xi$ to every subinterval of $I\backslash B$ is 
locally uniquely limiting.

\medbreak\noindent If $\Omega$ is an open subset of $\R^n$, we say that 
a function \mbox{$V:\Omega\mapsto\R$} {\em satisfies the inequality}

\vspace{-3mm}
\begin{equation}\label{loweq}
\sup\{-\langle \nabla V(x),f(x,u)\rangle-L(x,u):u\in U\}\ge 0
\end{equation}

\medbreak\noindent on $\Omega$ {\em in the viscosity sense} if 

\begin{center}\begin{minipage}{7.4cm}
\begin{itemize}
\item[(V$_+$)] {\em whenever $x\in\Omega$ and $p\in\R^n$ is a 
\mbox{subdifferential} of $V$ at $x$, it follows that}

\vspace{-3mm}
$$
\sup\{-\langle p, f(x,u)\rangle-L(x,u):u\in U\}\ge 0\,.
$$
\end{itemize}
\end{minipage}\end{center}

\medbreak\noindent(We recall that, if $\Omega$ is open in $\R^n$, then
a {\em subdifferential} of a function $V:\Omega\mapsto\R$ at a point
$\bar x\in\Omega$ is a vector $p\in\R^n$ such that

$$
\lim\inf_{x\to\bar x}\frac{V(x)-V(\bar x)-p\cdot(x-\bar x)}{
\|x-\bar x\|}\ge 0\,.\,)
$$

\medbreak Similarly, we say that $V$ {\em satisfies the inequality}

\begin{equation}\label{upeq}
\sup\{-\langle \nabla V(x),f(x,u)\rangle-L(x,u):u\in U\}\le 0
\end{equation}

\medbreak\noindent on $\Omega$ {\em in the viscosity sense} if 

\begin{center}\begin{minipage}{7.4cm}
\begin{itemize}
\item[(V$_-$)] {\em whenever $x\in\Omega$ and $p\in\R^n$ is a 
\mbox{superdifferential} of $V$ at $x$, it follows that}

\vspace{-7mm}
$$
\sup\{-\langle p, f(x,u)\rangle-L(x,u):u\in U\}\le 0\,.
$$
\end{itemize}
\end{minipage}\end{center}

\noindent (A {\em superdifferential} of $V$ at $x_*$ is a vector $p$ 
such that $-p$ is a subdifferential of $-V$ at $x_*$.) 

\medbreak We say that $V$ {\em satisfies the equation}

\begin{equation}\label{eqeq}
\sup\{-\langle \nabla V(x),f(x,u)\rangle-L(x,u):u\in U\}=0
\end{equation}

\medbreak\noindent on $\Omega$ {\em in the viscosity sense} if 
it satisfies (\ref{loweq}) and (\ref{upeq}) in the viscosity 
sense. 

\begin{remark}
The definition of ``viscosity solution'' given here 
is known to be equivalent to the more \mbox{common} one involving 
test functions, cf. \cite{BARD}.\proofend
\end{remark}

Our main result is the following theorem:

\begin{theorem}\label{mainres}
{\em Let $\hat\Sigma=(\Omega,U,f,L)$ be an \mbox{augmented} 
\mbox{control} system, let ${\cal T}$ be a target for $\hat\Sigma$, and
let \mbox{$V:\Omega\cup{\cal T}\mapsto\R$} be a function. Assume that
\begin{itemize}
\item[(1)] $\hat\Sigma$ is locally uniformly continuous, locally coercive,
and such that $F_{\hat\Sigma}(x,U)$ is closed and convex for \mbox{every} 
\mbox{$x\in\Omega$}. 
\item[(2)] $V$ is \mbox{continuous}.
\item[(3)] $V$ satisfies (\ref{eqeq}) on $\Omega$ in the
\mbox{viscosity} sense.
\item[(4)] $V$ vanishes on ${\cal T}$.
%\item[(5)] $V$ is is bounded below.
\item[(5)] Every augmented arc has an almost locally uniquely 
limiting improvement. 
\item[(6)] Whenever $x\in\Omega$, $\Xi=(\xi,\xi_0)\in\msdsvx$, 
and $\xi$ is right-unbounded, it follows that $\Xi\in\axts$.
\end{itemize}
Then $V\equiv\vts$.}\proofend
\end{theorem}

\begin{remark} 
Condition (6) was essentially introduced by M. Malisoff, cf.  
especially \cite{MALI}.\proofend
\end{remark}


\section{Examples}\label{exasec}

\begin{example} {\em (Linear-quadratic optimal control.)} 
\mbox{Consider} the standard linear-quadratic optimal \mbox{control} problem, in which 
$x$, $u$ take values in $\R^n$, $\R^m$, \mbox{respectively},
the dynamical law is
\begin{equation}\label{dynaquad}
\dot x=Ax+Bu\,,
\end{equation}
the Lagrangian is given by 
$$
L(x,u)=x^\dagger Rx+u^\dagger Su\,,
$$
the square matrices $R$, $S$ are strictly positive \mbox{definite}, and the pair 
$(A,B)$ is stabilizable. We take the \mbox{target} set ${\cal T}$ to consist 
of the origin of $\R^n$.  (In order to \mbox{satisfy} the condition 
that $F_{\hat\Sigma}(x,U)$ is convex for \mbox{every} 
$x\in\Omega$, we add a new scalar nonnegative control \mbox{variable}
$v$, in such a way that the dynamical law (\ref{dynaquad}) remains \mbox{unchanged}
but the Lagrangian $L$ is replaced by $\tilde L$, where 
$\tilde L(x,u,v)\equaldef L(x,u)+v$.) The crucial \mbox{technical} issue here is the
fact that the Lagrangian is not bounded away from zero. 
The \mbox{hypotheses} of our main \mbox{theorem} (including the coerciveness,
which \mbox{follows} from the \mbox{positive} \mbox{definiteness} of $S$) are easily 
verified as long as $V$ is bounded below.  The only 
nontrivial point is the \mbox{verification} of condition (6). To prove that 
this  holds, let $\Xi:[0,\tau\,[\,\mapsto\R^n\times\R$ be a 
right-unbounded \mbox{maximal} \mbox{trajectory}
of steepest descent with respect to $V$ that does not end
at the target, and write $\Xi=(\xi,\xi_0)$ in the usual way. Then 
$\tau$ has to be infinite, because if $\tau$ was \mbox{finite} then
the boundedness of the cost (\mbox{arising} from the fact that 
$\Xi=(\xi,\xi_0)$ is of steepest descent and $V$ is bounded below)
would \mbox{trivially} imply an $L^2$ bound on the control, from which
it would follow that 
$\Xi$ can be extended to the closed \mbox{interval} $[0,\tau]$, and 
then the \mbox{assumption} that $\Xi$ does not end at the 
\mbox{target} would
\mbox{enable} us to % use Proposition \ref{trivialprop} and
%Theorem \ref{maintech} (\mbox{applied} with $\Omega=\R^n\backslash\{0\}$)
%to 
\mbox{extend} $\Xi$ even \mbox{further}, contradicting maximality. 
So $\tau$ is \mbox{infinite}. On the other hand, the fact that
$V$ is bounded \mbox{below} and $\Xi$ is of \mbox{steepest} descent implies
that the integral 
$$
\int_0^\infty\Big(\xi(t)^\dagger R\xi(t)+
\eta(t)^\dagger S\eta(t)\Big)dt
$$
is finite, if $\eta$ is an open-loop control that generates $\Xi$.
But then $\xi$ and $\eta$ are square-integrable, so the 
\mbox{condition} that \mbox{$\dot\xi=A\xi+B\eta$}
implies that $\xi$ is square-integrable and has
a square-integrable derivative, and then Barbalat's lemma
implies that  $\xi$  ends at the \mbox{target}, as desired. \proofend
\end{example}



\begin{example} {\em (Fuller's problem, cf., e.g., 
\cite{ZB}.)}  This is the optimal control problem for the dynamical law
$$
\dot x=y\,,\qquad\dot y=u\,,
$$
with control constraint $-1\le u\le 1$. The target set ${\cal T}$ consists 
of the origin of $\R^2$.  The Lagragian is $L(x,y,u)=x^2$.
The crucial \mbox{technical} issue here is the
fact that the Lagrangian is not bounded away from zero, and in fact has a 
whole line of zeros.  The \mbox{hypotheses} of our main theorem are easily 
verified as long as $V$ is bounded below. The only 
nontrivial point is the \mbox{verification} of condition (6). To prove that 
this  holds, let $\Xi:[0,\tau\,[\,\mapsto\R^3$ be a right-unbounded
\mbox{maximal} \mbox{trajectory}
of steepest descent with respect to $V$ that does not end
at the target, and write $\Xi=(\xi,\xi_0)$ in the usual way. Then 
$\tau$ has to be infinite, because if $\tau$ was \mbox{finite} then
the boundedness of the control would \mbox{trivially} imply that 
$\Xi$ can be extended to the closed \mbox{interval} $[0,\tau]$, and 
then the assumption that $\Xi$ does not end at the target
enables us to %use Proposition \ref{trivialprop} and
%Theorem \ref{maintech} (applied with $\Omega=\R^2\backslash\{(0,0)\}$)
\mbox{extend} $\Xi$ even \mbox{further}, contradicting maximality. 
So $\tau$ is \mbox{infinite}. On the other hand, the fact that 
$V$ is bounded \mbox{below} and $\Xi$ is of \mbox{steepest} descent implies
that the integral $\int_0^\infty x(t)^2dt$ is finite, if we write
\mbox{$\xi(t)=(x(t),y(t))$}. But then $x(\cdot)$ is a square-integrable function
on $[0,\infty\,[\,$  whose second derivative is bounded.�
By a \mbox{straightforward}  \mbox{generalization} of Barbalat's lemma, 
this implies that 
both $x(\cdot)$ and $y(\cdot)$ go to zero, i.e., that 
$\xi$  ends at the target, as desired. \proofend
\end{example} 

\begin{example} {\em (The reflected brachistochrone problem.)} This is
the minimum time problem for the dynamical law
\begin{eqnarray*}
\dot x&=&u\sqrt{|y|}\,,\\
\dot y&=&v\sqrt{|y|}\,,
\end{eqnarray*}
with control constraint $u^2+v^2\le 1$. The target set ${\cal T}$ consists 
of a single point $B\in\R^2$.  The crucial \mbox{technical} issue here is the
fact that the dynamical law is not Lipschitz-continuous with respect to the 
state.  The \mbox{hypotheses} of our main theorem are easily verified. The only 
nontrivial point is the verification of condition (5). To prove that this 
holds, we pick an arbitrary \mbox{integral} arc $\xi:[a,b]\mapsto\R^2$,
and observe that either (i)~$\xi(t)$ never belongs to the $x$ axis
\mbox{$X=\{(x,y)\in\R^2:y=0\}$} or (ii)~there exist $t_-,t_+\in[a,b]$
such that $t_-\le t_+$, \mbox{$\xi(t)\notin X$} whenever $a\le t<t_-$
or $t_+>t\le b$, $\xi(t_-)\in X$, and $\xi(t_+)\in X$. If (i) holds, 
then $\xi$ \mbox{satisfies} the conditions  of  Example \ref{exalips}, so 
$\xi$ is uniquely \mbox{limiting}.  If (ii) holds and $t_+=t_-$, then the 
\mbox{restriction} of $\xi$ to each of the \mbox{intervals} $[a,t_-\,[+,$, 
$\,]\,t_+,b]$, is \mbox{locally} uniquely \mbox{limiting}, so 
$\xi$ is \mbox{almost} locally uniquely \mbox{limiting}.  Finally, if (ii) 
holds and $t_+<t_-$, then the restriction $\tilde\xi$ 
of $\xi$ to the interval $[t_-,t_+]$ is such that
the set \mbox{$S=\{t\in[a,b]:\xi(t)\notin X\}$} is the union of a finite
or countable infinite collection ${\cal I}$ of pairwise disjoint 
relatively open subintervals of $[a,b]$. If $I\in{\cal I}$, then 
the restriction $\tilde\xi_I$ of $\tilde\xi$ 
to $I$ is entirely contained in the
open upper half-plane or in the open lower half-plane. By reflecting 
$\tilde\xi_I$ with respect to $X$, if necessary, we may 
assume that $\tilde\xi_I$ is entirely contained in the
open upper half-plane for every $I\in{\cal I}$. Then 
$\tilde\xi$ is a trajectory of our system entirely 
\mbox{contained}
in the closed upper half-plane. It is well known that the problem 
in the closed upper half-plane $H_+$ 
is the famous ``brachistochrone problem,''
whose time-optimal \mbox{trajectories} $\zeta:[\alpha,\beta]\mapsto H_+$
are cycloids such that $\zeta(t)$ is an interior point of $H_+$ 
whenever $\alpha<t<\beta$.  It follows that we can always replace 
$\tilde\xi$ by a cycloid $\zeta$, thereby \mbox{obtaining} an
almost locally uniquely limiting \mbox{improvement} of $\xi$. \proofend
\end{example} 

\begin{example} {\em (An example with a continuous \mbox{non-Lipschitz}
dynamics where uniqueness fails.)} Let $\varphi:[0,1]\mapsto\rr$ be a 
nonnegative continuous function such that (a)~the set $\{x\in[0,1]:
\varphi(x)=0\}$ is exactly the Cantor set, and
(b)~$\int_0^1\frac{dx}{\varphi(x)}<\infty$.  (For example, we may take
$\varphi$ to be given by $\varphi(x)={\rm dist}(x,C)^\rho$, where $C$
is the Cantor set and $\rho$ is a positive number such that
$\rho<1-{\rm log}_{_3}2$.  An explicit calculation shows that
$\int_0^1\frac{dx}{\varphi(x)}=(1-\rho)^{-1}2^\rho\sum_{j=1}^{\infty}
\theta^j$, where $\theta=\frac{2}{3}\times 3^\rho$. Our choice of
$\rho$ guarantees that $\theta<1$, so the integral is finite.)  Extend
$\varphi$ to a function defined on $\rr$ by making it periodic of
period $1$. Then consider the optimal control problem on $\rr$ whose
dynamics is $\dot x=u\varphi(x)$, $|u|\le 1$, and where the goal is to
reach the origin in minimum time.  It is easy to see that the optimal 
trajectory from each point $x$ exists and is obtained by ``moving towards the 
target as fast as possible.'' Precisely, this means the we use the control 
$u=-1$ as long as we are to the right of the origin, and we use
$u=1$ if we are to the left. This, however, does not suffice to specify 
the optimal trajectories, because of the lack of uniqueness of solutions. 
The complete specification of the optimal trajectories is as follows.
Suppose $\bar x<0$. Define a function 
$\tau:[\bar x,0]\mapsto\rr$ by letting 
$\tau(x)=\int_{\bar x}^x\frac{dy}{\varphi(y)}$. Then 
$\tau$ is aboslutely continuous, strictly increasing, and such that
$\tau(\bar x)=0$. Therefore $\tau$ maps the interval 
$[\bar x,0]$ homeomorphically onto the interval 
$[0,\tau(0)]$. Let $\xi$ be the inverse function, so 
$\xi$ maps  $[0,\tau(0)]$ homeomorphically onto $[\bar x,0]$. 
Then $\xi$ is absolutely continuous, and 
$\dot\xi(t)=\varphi(\xi(t))$ for almost all $t\in[0,\tau(0)]$.
So $\xi$ is a trajectory of our system which goes from 
$\bar x$ to $0$ in time $\tau(0)$, and it is easy to see that $\xi$ 
is the optimal trajectory from $\bar x$ to $0$. It follows that optimal
time to go from $\bar x$ to $0$ is $\tau(0)$, that is, 
$\int_{\bar x}^0\frac{dy}{\varphi(y)}$.  A similar contruction
applies when  $\bar x>0$. Then the value function $\bar V$ for our problem
is given by
$$
\bar V(x)=\int_{\min(x,0)}^{\max(x,0)}\frac{dy}{\varphi(y)}\,.
$$
The HJBE for our problem is
\begin{equation}\label{isasol}
|V'(x)|\varphi(x)-1=0\,.
\end{equation}
The function $\bar V$ is a solution of this equation on $\rr\backslash\{0\}$
in the \mbox{viscosity} sense.  (This follows from the fact that, for
problems such as this one, the value function is \mbox{automatically} a
viscosity solution of the HJBE. In \mbox{addition}, one can also verify this
directly.  Let ${\cal O}$ be the set of points where
$\varphi(x)>0$. Then on ${\cal O}$ the function $\bar V$ is smooth,
and its derivative is $\frac{1}{\varphi}$ when $x<0$, and
$-\frac{1}{\varphi}$ when $x>0$, so (\ref{isasol}) holds. At points
$x$ where $\varphi(x)=0$, the viscosity solution requirements say that
$-1\ge 0$ whenever $p$ is a subdifferential of $\bar V$ at $x$, and
$-1\le 0$ whenever $p$ is a superdifferential of $\bar V$ at $x$. The
second condition is trivially true. To verify the first condition,
we need to show that it is satisfied vacuously, i.e., that there are 
no subdifferentials of $\bar V$ at $x$. But this easy. Suppose, say, that 
$x<0$. The difference quotient 
$\frac{1}{h}(V(x+h)-V(x))$ is equal, if $h>0$, to 
$-\frac{1}{h}\int_x^{x+h}\frac{dy}{\varphi(y)}$, which is bounded above 
by $\zeta(h)=-\frac{1}{\max\{\varphi(y):y\in[x,x+h]\}}$. 
Since $\varphi(x)=0$, $\zeta(h)$ goes to 
$-\infty$ as $h\to 0$.  This shows that the right derivative of $\bar V$ 
at $x$ is equal to $�\infty$, from which it follows easily 
that there exist no \mbox{subdifferentials} of $\bar V$ at $x$. 
A similar argument shows that if $x>0$ the left derivative of $\bar V$ 
at $x$ is equal to $+\infty$, from which it follows once again 
that there are no subdifferentials of $\bar V$ at $x$.

We now show that there exist nonnegative \mbox{continuous} functions
$\hat V:\rr\mapsto\rr$ other than $\bar V$ that satisfy the HJBE on
$\rr\backslash\{0\}$ and are such that $\hat V(0)=0$.  To see this, we
let $W$ be a continuous monotonically nondecreasing real-valued
function on $[0,\infty\,[\,$ such that (a)~$W(0)=0$, (b)~$W$ is
constant on each connected component of the set
\mbox{$\{x:x>0\wedge\varphi(x)>0\}$}, and (c)~$W(x)<W(y)$ \mbox{whenever}
$0\le x<y$ and the interval $[x,y]$ contains a zero of $\varphi$.
(Such a function is easily constructed using the well known Cantor
function.)  We then \mbox{extend} $W$ to all of $\rr$ by defining
$W(x)=W(-x)$ when $x<0$. Using $W$, we define $\hat V=\bar V+W$.  Then
$\hat V$ is \mbox{continuous}, \mbox{$\hat V(0)=0$}, and $\hat
V(x)>\bar V(x)$ whenever $x\neq 0$.  Let us show that $\hat V$ is also
a solution of the HJBE for our \mbox{problem} on $\rr\backslash\{0\}$.  Near
points $x$ such that $\varphi(x)>0$, the functions $\bar V$ and $\hat
V$ differ by a constant, so the fact that $\bar V$ satisfies the HJBE
implies that the same is true for $\hat V$.  If $x\neq 0$ but
$\varphi(x)=0$, the \mbox{viscosity} \mbox{solution} requirements say that 
$-1\ge 0$ whenever $p$ is a \mbox{subdifferential} of $\hat V$ at $x$,
and $-1\le 0$ whenever $p$ is a superdifferential of $\hat V$ at $x$,
and the second one of these conditions is trivially true.  As for the
first \mbox{condition}, if $x<0$ then we have already shown that the
right \mbox{derivative} of $\bar V$ at $x$ is equal to $�\infty$, and
this clearly \mbox{implies} that the right derivative of $\hat V$ at
$x$ is equal to $�\infty$ as well, since $\hat V=\bar V+W$ and $W$ is
\mbox{monotonically} \mbox{nonincreasing} near $x$. Hence there 
\mbox{exist} no \mbox{subdifferentials} of $\hat V$ at $x$. A similar
argument applies if $x>0$, and we conclude that first one of the
viscosity requirements is satisfied vacuously.

It follows that for our example the value function is not the unique
continuous nonnegative function that vanishes at the target and
satisfies the HJBE. In the example, the reason for the failure of
uniqueness is easy to \mbox{understand}, and clearly related to the
non-uniqueness of trajectories. Notice that the spurious value
\mbox{function} $\hat V$ is bounded below by the true value function,
so what goes wrong is the other inequality, which is related to the
\mbox{dynamic} programming inequality (DPI). And, \mbox{indeed}, the
DPI fails, and this makes it impossible to draw the conclusion that
$\hat V\le\bar V$. Furthermore, the failure of the DPI happens exactly
as described in our general \mbox{analysis}: given any control
$u(\cdot)$ and any initial \mbox{condition} $x_0$, it is easy to
construct a maximal trajectory $\xi$ for $u(\cdot)$ starting at $x_0$
along which the DPI for $\hat V$ holds.  (It \mbox{suffices} to follow
the only possible trajectory for $u(\cdot)$ as long as $\varphi\neq
0$, and stopping at $\bar x$ and staying there for ever as soon as we
reach the first point $\bar x$ where $\varphi$ \mbox{vanishes}.)  This
$\xi$ is not, however, the only \mbox{trajectory} for $u(\cdot)$
\mbox{starting} at $x_0$.  And the fact that the DPI holds along $\xi$
does not imply that that the DPI holds for \mbox{every} trajectory for
$u(\cdot)$ that starts at $x_0$.  (Indeed, if for \mbox{example}
$x_0<0$ and $u(t)\equiv 1$, then in addition to the $\xi$ given by our
construction we could also consider $\xi_{opt}$, the optimal
trajectory described earlier. The DPI for $\hat V$ clearly fails along
the curve $\xi_{opt}$, because if it was true it would follow that 
$\hat V(x_0)\!\le\!\bar V(x_0)$, whereas we know that 
$\hat V(x_0)\!>\!\bar V(x_0)$.) \proofend
\end{example} 

\section{ACKNOWLEDGMENTS}
The  author's work was supported in part by NSF Grant 
DMS01-03901. He wrote the final version while he was a guest of the
Institut Mittag-Leffler, in Djursholm, Sweden, during the Spring
semester of 2003, and is very grateful to the Institute for its kind
hospitality.  He particularly thanks Anders Lindquist for encouraging
him to give the lectures on the value function that eventually
resulted in this paper.

\vspace{6mm}

\begin{thebibliography}{10}


\bibitem{BARD}
Bardi, M., and I. Capuzzo-Dolcetta, {\em Optimal \mbox{Control} and
Viscosity Solutions of Hamilton-Jacobi-Bellman Equations}, Birkh{\"a}user,
Boston, 1997.

%\bibitem{BDL97}
%Bardi, Martino, and Francesca Da Lio,  ``On the Bellman equation
%for some unbounded \mbox{control} problems.'' {\em NoDEA Nonlinear
%Differential \mbox{Equations} Appl.} {\bf 4} (1997), pp.~276-285.

%\bibitem{FLEM}
%Fleming, W.H., and H.M. Soner, {\em Controlled Markov Processes and Viscosity
%Solutions}, Springer, New York, 1993.

%\bibitem{Cla-Le-a}
%Clarke, F.H., and Yu.S. Ledyaev, ``Mean value \mbox{inequalities}.''
%{\em   Proc. Amer. Math. Soc.} {\bf 122}  (1994),  no. 4, 
%pp.~1075--1083. 

%\bibitem{Cla-Le-b}
%Clarke, F.H., and Yu.S. Ledyaev, ``Mean value \mbox{inequalities} in 
%Hilbert space.''  {\em Trans. Amer. Math. Soc.}  {\bf 344}  (1994), 
%no. 1, pp.~307--324. 

%\bibitem{Cla-Le-c}
%Clarke, F.H., Yu.S.  Ledyaev, R.J. Stern, and P.R. Wolenski. {\em Nonsmooth 
%analysis and control theory}. Graduate Texts in Mathematics, 178. 
%Springer-Verlag, New York, 1998.

\bibitem{Mdis}
Malisoff, M., ``Viscosity Solutions of the \mbox{Bellman} \mbox{Equation} for
Optimal Control Problems with \mbox{Exit} Times.'' Ph.D. Thesis, Rutgers
University, New Brunswick, NJ, January 2000.

\bibitem{M99b}
Malisoff, M., ``Viscosity solutions of the Bellman \mbox{equation} for
exit time optimal control problems with vanishing Lagrangians.''
{\em SIAM J. Control \mbox{Optim.}}, to appear.

\bibitem{MALI}
Malisoff, M., ``Viscosity solutions of the Bellman \mbox{equation} for
exit time optimal control \mbox{problems} with non-Lipschitz dynamics.''
{\em ESAIM: Control, \mbox{Optimization} and Calculus of Variations},
{\bf 6} (2001),   pp.~415-441.

\bibitem{M01}
Malisoff, M., ``Bounded-from-below solutions of the
Hamilton-Jacobi equation for optimal control \mbox{problems} with exit
times: Vanishing \mbox{Lagrangians}, eikonal \mbox{equations}, and
shape-from-shading.'' \mbox{submitted}.

\bibitem{subbotin}
Subbotin, A.I., {\em Generalized solutions of first-order 
PDEs: the dynamical optimization perspective.} Birkh\"auser,
Boston, 1995. 

\bibitem{Su-Japan}
Sussmann, H.J., ``From the Brachystochrone \mbox{problem} to the maximum
principle.'' in {\em Proc. 35th IEEE CDC}, IEEE, New York, 1996,
pp.~1588-1594.

\bibitem{ZB}
Zelikin, M.I., and V.F. Borisov, {\em Theory of \mbox{Chattering} Control
with Applications to \mbox{Astronautics}, Robotics, Economics, and
\mbox{Engineering}.} Birkh{\"a}user, Boston, 1994.
\end{thebibliography}
\end{document}




\begin{remark}
Naturally, the concepts of continuity,   \mbox{uniform} continuity,
Lipschitz continuity,  and uniform Lipschitz continuity,  
also make sense for a control \mbox{system} $\Sigma=(\Omega,U,f)$,
by taking the same definitions given above and omitting 
the parts that refer to $L$. In order to avoid having to make 
a similar remark for other concepts to be introduced
in the future, we adopt the convention that any concept $X$
that we \mbox{define} for an augmented control system 
$\hat\Sigma=(\Omega,U,f,L)$ is \mbox{automatically}
\mbox{understood} to apply to a control system $\Sigma=(\Omega,U,f)$, 
in the sense that ``$X$ of $\Sigma$'' means 
``$X$ of the augmented system $\hat\Sigma=(\Omega,U,f,0)$.''\proofend
\end{remark}

