%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%  
%             This is a LaTeX file, containing the paper 
%
%                
%             FURTHER RESULTS ON THE BELLMAN EQUATION FOR 
%                 EXIT TIME OPTIMAL CONTROL PROBLEMS 
%                    WITH NONNEGATIVE LAGRANGIANS:
%		     THE CASE OF FULLER'S PROBLEM
%
%                 by M. Malisoff and H. J. Sussmann
%
%            This version will appear in the conference 
%	       proceedings of the 39th IEEE CDC, to be
%               held in Sydney, Australia, Dec. 2000.
%  
%              Date of this version: September 15, 2000
%
%                 The paper is 3 printed pages long.
%
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newcommand{\T}{{\cal T}}
\newcommand{\dist}{{\rm dist}}
\newcommand{\ds}{\displaystyle}
\newcommand{\beq}{\begin{equation}}
\newcommand{\eeq}{\end{equation}}
\newcommand{\beqn}{\begin{eqnarray}}
\newcommand{\eeqn}{\end{eqnarray}}
\newcommand{\beqno}{\begin{eqnarray*}}
\newcommand{\eeqno}{\end{eqnarray*}}
\newcommand{\bequot}{\begin{quotation}}
\newcommand{\equot}{\end{quotation}}
\newcommand{\rf}[1]{[\ref{#1}]} %bibliographical references
\newcommand{\text}[1]{\hbox{\rm \ #1\ \/}}
%\newcommand{\beql}[1]{\begin{equation}\label{#1}}
\newcommand{\bt}{\begin{theorem}}
\newcommand{\bl}{\begin{lemma}}
\newcommand{\br}{\begin{remark}}
\newcommand{\bc}{\begin{corollary}}
\newcommand{\bp}{\begin{proposition}}
\newcommand{\bd}{\begin{definition}}
\newcommand{\bfact}{\begin{fact}}
\newcommand{\bex}{\begin{example}}
\newcommand{\ber}[1]{\begin{exercise}\label{#1}}
\newcommand{\bem}[1]{\begin{example}\label{#1}}  %Yes, 2 different ones...
\newcommand{\ec}{\end{corollary}}
\newcommand{\el}{\end{lemma}}
\newcommand{\er}{\end{remark}}
\newcommand{\et}{\end{theorem}}
\newcommand{\ed}{\end{definition}}
\newcommand{\ednoen}{\end{definition}}
\newcommand{\eprop}{\end{proposition}}
\newcommand{\epr}{\end{proof}}
\newcommand{\bpr}{\begin{proof}}
\newcommand{\calu}{{\cal U}}
\newcommand{\scr}{\scriptscriptstyle}
\newcommand{\N}{{\rm I \kern-.2em N}}

\def\tto{\mbox{$\,\,\rightarrow\kern-13pt\rightarrow\,\,$}}
\def\transv{{\cap\kern-5.2pt\lower
    -2.4pt\hbox{$|$}\kern-5.6pt\lower -8pt\hbox{$-$}}}
\def\baba{|\kern-1.9pt|} 
\def\stransv{{\cap\kern-5.7pt\lower
    -2.4pt\hbox{$\baba$}\kern-6.3pt\lower -8pt\hbox{$-$}}}
\def\ihpnote{Research supported in part by NSF CCR91-19999.  This author
last revised this paper on September  15, 2000.}
%Most of this work was done while the author was a visitor
%at the Centre \'Emile Borel of the Institut Henri Poincar\'e (IHP), in
%Paris, France. The author is very grateful for the hospitality
%provided by the IHP.}
\def\supportnote{Research supported in part by %
NSF Grant DMS95-00798 and AFOSR Grant 0923.}
\def\bdel{\bar\delta}
\def\ffi{\varphi}
\def\beps{\bar\varepsilon}
\def\co{{\rm co}}
\def\cco{\overline{{\rm co}}}
\def\togr{{\buildrel \rm gr \over \to}}
\def\bfact{\begin{quotation}}
\def\efact{\end{quotation}}
\def\cvfadomi{TVVF_{AD}(\Om,I)}
\def\cvfomi{TVVF(\Om,I)}
\def\cvf{TVVF}
\def\omi{\Omega\!\times\!I}
\def\msr{{\rm meas}}
\def\bloi{{\cal BL}(\Omega,I)}
\def\mbloiy{{\cal M}_e(\bloi,Y)}
\def\ndc{{\bf W}}
\def\eps{\varepsilon}
\def\bbxr{{\cal B}(X\times\rr)}
\newcommand{\mybox}{\hfill \mbox{$\Box$}} %put at right margin (white square)
\def\dom{{\rm Dom}}
\newcommand{\pubnote}{\vskip-5.0cm
\framebox{Proc.\ {\it IEEE Conf. Decision and Control, Tucson, Dec.\ 1992}} 
\vskip4.5cm}
\font\myverysmall=cmr5
\def\midbreak{\vskip 0.01true cm}
\def\steer#1#2{$#1$\steero$#2$}
\def\steert#1#2#3{\hbox{$#1$\hbox{\lower0.4ex\hbox{\steero}}\kern-0.59cm\raise0.53ex\hbox{$#2$}}\kern0.35cm$#3$}
\def\Om{\Omega}
\def\compo{\hbox{$\,${\myverysmall o}$\,$}}
\def\plplus{(+\kern-0.02in\compo\kern -0.02in\oplus)}
\def\om{\omega}
\def\vphi{\varphi}
\def\mtraj{{\rm MTraj\,}}
\def\indi{\raise 1.5pt\hbox{$\chi$}}
\def\lowsub#1{\lower 2pt\hbox{${_{#1}}$}}
\def\pfo#1{{\cal P}_{#1}^{\rightarrow}}
\def\pba#1{{\cal P}_{#1}^{\leftarrow}}  
\def\aabx{{\cal A}(a,b,X)}
\def\icone{IC\kern0.1pt$^1$}
\def\iconeit{IC\kern0.5pt$^1$}
\def\mdno{\medbreak\noindent}
\def\inn{\int_{0}^{\tau}}
\def\bel{\beta}
\def\al{\alpha}
\def\ctraj#1{{\rm CTraj}(#1)}
\def\carc#1{{\rm CArc}(#1)}
\def\traj{{\rm Traj}}
\def\arc#1{{\rm Arc}(#1)}
\def\restr{\lceil}
\def\ep{\varepsilon}
\def\dki{{\partial\,\,\,\over\partial\kappa^i}}
\def\dkia{\partial_i^\kappa}
\def\dkonea{\partial_1^\kappa}
\def\dkna{\partial_n^\kappa}
\def\pdto{\,\rightarrow\kern -12pt ^{pd}\,}
\def\tvto{\,\rightarrow\kern -12pt ^{tv}\,}
\def\opdto{\,\longrightarrow\kern -18pt ^{opd}\,}
\def\pdtvto{\,\hbox to 1.1true cm{\rightarrowfill}\kern -20pt ^{pdtv}\,\,\,}
\def\tpdtvto{\,\hbox to 1.1true cm{\rightarrowfill}\kern -23pt ^{tpdtv}\,\,\,}
\def\adjo#1{{\rm Adj}(#1)}
\def\adjonul#1{{\rm Adj}^{\nu,L}(#1)}
\def\cch{\overline{co}}
\def\rrinf{\bar{\rr}}
\def\nn{{\rm I}\kern-.2em{\rm N}}
\def\nninf{\bar{\nn}}
\def\rrsmall{{\mysmall I}\kern-.2em{\mysmall R}}
\def\compo{\hbox{$\,${\myverysmall o}$\,$}}
\newcommand{\Psiu}{\underline{\Psi}}
\newcommand{\Psiud}{\underline{\dot\Psi}}
\newcommand{\Deltau}{\underline{\Delta}}
\newcommand{\psiu}{\underline{\psi}}
\newcommand{\Phiu}{\underline{\Phi}}
\newcommand{\thetau}{\underline{\theta}}
\newcommand{\sigmau}{\underline{\sigma}}
\newcommand{\varphiu}{\underline{\varphi}}
\newcommand{\nuu}{\underline{\nu}}
\newcommand{\hu}{\underline{h}}
\newcommand{\Deltaud}{\underline{\dot\Delta}}
\newcommand{\psiud}{\underline{\dot\psi}}
\newcommand{\Phiud}{\underline{\dot\Phi}}
\newcommand{\thetaud}{\underline{\dot\theta}}
\newcommand{\sigmaud}{\underline{\dot\sigma}}
\newcommand{\varphiud}{\underline{\dot\varphi}}
\newcommand{\nuud}{\underline{\dot\nu}}
\newcommand{\hud}{\underline{\dot h}}
\newtheorem{defi}{Definition}%[section]
\newtheorem{definition}{Definition}
\newtheorem{exam}{Example}%[section]
\newtheorem{propos}{Proposition}
\newtheorem{lemma}{Lemma}
\newtheorem{remark}{Remark}
\newtheorem{theorem}{Theorem}
\newcommand{\rr}{{\rm I}\kern-.2em{\rm R}}
\newcommand{\pa}{\partial}
\newcommand{\raw}{\rightarrow}
\newcommand{\vare}{\varepsilon}
\newcommand{\nis}{\cal N}
\def\beginproof{\medbreak\noindent{\sl PROOF.\enspace}}
\def\equaldef{{\buildrel \rm def \over =}}
\def\equivdef{{\buildrel \rm def \over \equiv}}
\def\proofend{\mybox}
\def\bi{\begin{itemize}}
\def\ei{\end{itemize}}
\def\br{\begin{remark}}
\def\er{\end{remark}}
\def\bt{\begin{theorem}}
\def\et{\end{theorem}}
\def\bp{\begin{proposition}}
\def\ep{\end{proposition}}
\documentstyle{article}
\renewcommand{\baselinestretch}{0.95} %to fine-tune - use .95 if you need more

\newcommand{\auone}{
\vspace{-15pt}
\begin{center}
H\'ector J. Sussmann$^1$ and Michael A. Malisoff $^2$\\
%\footnote[1]{\supportnote}\\
%\footnote{\supportnote}\\
Department of Mathematics; Rutgers, the State University of New Jersey\\
Hill Center---Busch Campus, 110 Frelinghuysen Road, Piscataway, NJ
08854-8019, USA\\ 
\vskip5pt
 Email:\ \  $\mathtt{\{sussmann, malisoff\}@\, math.rutgers.edu}$\quad
\quad
Web site:\ \  $\mathtt{http://www.math.rutgers.edu/\sim sussmann}$

\end{center}
\vskip5pt
} 

\newcommand{\authors}{
\auone
}

\renewcommand{\title}{
\vspace{-15 pt}

Further results on the Bellman equation for exit time optimal control
problems with nonnegative Lagrangians: The case of
Fuller's
Problem}%\footnote{test}

%$\,^{1,2}$}

\def\spaceabovetitle{\hspace{1cm}}
\def\spacebelowtitle{\vspace{2pt}}
\def\spaceafterauthors{}

\def\titlestuff{
%{\small \null\vspace{-5mm}This paper will be presented at the 39th 
%IEEE Conference on Decision and Control, to be held in
%Sydney, in December 2000,  and  will appear in the conference 
%proceedings. The date of this version is August 8, 2000.}\\
\vspace{-5.6mm}
\null\vspace{4mm}
\vspace{2mm}
\begin{center}
\spaceabovetitle
\begin{center}
{\Large \bf \title}\\
\end{center}
\spacebelowtitle
\authors
\spaceafterauthors
\end{center}}
\tolerance=20 %allow VERY LITTLE tolerance in extending after line.
\overfullrule=0pt %no dark lines if overfull

\newcommand{\ssection}[1]{\vspace{0.25cm}\setcounter{subsection}{0}
  \protect\addtocounter{section}{1}%
  \centerline{{\bf \thesection.~#1}}\vspace{.1cm}}
\newcommand{\subssection}[1]{\vspace{0.16cm}
  \protect\addtocounter{subsection}{1}
  \noindent{{\bf\thesubsection.~#1}}\vspace{0.1cm}} 
\topmargin -.45cm %I added this to try to make the file print in SBH 211.
%\topmargin -2.3cm %This is what was in HJS's file.
\textheight 24.4cm
\oddsidemargin -0.3in
\evensidemargin -0.3in
\textwidth 178mm

\sloppy
\flushbottom
\parindent 1em
\leftmargini 2em
\leftmarginv .5em
\leftmarginvi .5em

\marginparwidth 48pt 
\marginparsep 10pt 
\columnsep 10mm

%\pagestyle{empty}
%\renewcommand{\thefootnote}{\fnsymbol{footnote}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Macros needed for this sample paper
\def\vbar{\mathchoice{\vrule height6.3ptdepth-.5ptwidth.8pt\kern-.8pt}
   {\vrule height6.3ptdepth-.5ptwidth.8pt\kern-.8pt}
   {\vrule height4.1ptdepth-.35ptwidth.6pt\kern-.6pt}
   {\vrule height3.1ptdepth-.25ptwidth.5pt\kern-.5pt}}
\def\fudge{\mathchoice{}{}{\mkern.5mu}{\mkern.8mu}}
\def\bbc#1#2{{\rm \mkern#2mu\vbar\mkern-#2mu#1}}
\def\bbb#1{{\rm I\mkern-3.5mu #1}}
\def\bba#1#2{{\rm #1\mkern-#2mu\fudge #1}}
\def\bb#1{{\count4=`#1 \advance\count4by-64 \ifcase\count4\or\bba A{11.5}\or
   \bbb B\or\bbc C{5}\or\bbb D\or\bbb E\or\bbb F \or\bbc G{5}\or\bbb H\or
   \bbb I\or\bbc J{3}\or\bbb K\or\bbb L \or\bbb M\or\bbb N\or\bbc O{5} \or
   \bbb P\or\bbc Q{5}\or\bbb R\or\bbc S{4.2}\or\bba T{10.5}\or\bbc U{5}\or
%   \bbb P\or\bbc Q{5}\or\bbb R\or\bba S{8}\or\bba T{10.5}\or\bbc U{5}\or
   \bba V{12}\or\bba W{16.5}\or\bba X{11}\or\bba Y{11.7}\or\bba Z{7.5}\fi}}
%
\def\R{{\bb R}}
\def\C{{\bb C}}

\newcommand{\new}{\newcommand}
\new{\rnew}{\renewcommand}
\new\Empty{/\mkern-9.5mu O}
\newcommand{\Rn}[1]{\R^{#1}}
\renewcommand{\a}{\alpha}
\renewcommand{\b}{\beta}
\renewcommand{\c}{\gamma}
\newcommand{\Ca}{\Gamma}
\renewcommand{\d}{\delta}
\newcommand{\D}{\Delta}
\newcommand{\e}{\varepsilon}
\newcommand{\G}{\bigtriangledown}
\newcommand{\k}{\kappa}
\renewcommand{\l}{\lambda}
\renewcommand{\L}{\Lambda}
\newcommand{\p}{\varphi}
\newcommand{\s}{\sigma}
\renewcommand{\t}{\theta}
\newcommand{\w}{\omega}
\newcommand{\tx}{\tilde x}
\new{\ty}{\tilde y}
\new\ttt{\tilde t}
\new{\te}{\tilde e}
\new{\bx}{\bar{x}}
\new{\by}{\bar{y}}
\new{\bz}{\bar{z}}
\new{\dx}{\dot{x}}
\new{\dy}{\dot{y}}
\new{\dz}{\dot{z}}
\new\sat{\rm sat}
\new\sign{\rm sign}
\newcommand{\dd}[2]{\frac{d#1}{d#2}}
\newcommand{\pdo}[1]{\frac{\partial}{\partial#1}}
%\renewcommand{\vec}[2]{(#1_1,\cdots,#1_{#2})}
\newcommand{\vecc}[3]{(#1_{#2},\cdots,#1_{#3})}
\newcommand{\vecf}[3]{(#1_1#3,\cdots,#1_{#2}#3)}
\newcommand{\row}[2]{#1_1,\cdots,#1_{#2}}
\newcommand{\roww}[3]{#1_{#2},\cdots,#1_{#3}}
\newcommand{\newt}{\newtheorem}
\newcommand{\rarrow}{\rightarrow}
\newtheorem{Corollary}[lemma]{Corollary}
%\newcommand{\beq}{\vspace{-0.18cm}\begin{equation}\vspace{-0.12cm}}
%\newcommand{\eeq}{\end{equation}}
%\newcommand{\beqn}{\begin{eqnarray}}
%\newcommand{\eeqn}{\end{eqnarray}}
\newcommand\yangspace{\vspace{-2.45mm}}
\newcommand\yangspacebb{\vspace{-2.25mm}}
\newcommand\yangspacebbc{\vspace{-2.10mm}}
\newcommand\yangspaceb{\vspace{-1.4mm}}
\newcommand\yangspacec{\vspace{-1.2mm}}
\newcommand\yangspaced{\vspace{-.85mm}}
\newcommand{\brem}{\yangspacebb\begin{remark}}
\newcommand{\bedef}{\yangspacebb\begin{defi}}
%\def\bd{\bedef}
%\def\ed{\edef}
\newcommand{\begexam}{\yangspacebb\begin{exam}}
\newcommand{\termiexam}{\end{exam}\yangspacebb}
\newcommand{\beprop}{\yangspacebb\begin{propos}}
\newcommand{\termiprop}{\end{propos}\yangspacebb}
\newcommand{\termidef}{\end{defi}\yangspacebb}
\newcommand{\erem}{\end{remark}\yangspacebb}
\newcommand{\shsection}{\ssection}
\newcommand{\proof}{\noindent {\sc Proof.}\ }
\newcommand{\niindent}{\vspace{1mm}\noindent}
\new{\eproof}{\hfill $\Box$}
\newcommand{\dg}{\dagger}
\newcommand{\dfn}{\stackrel{\rm def}{=}}

\def\pmb#1{\setbox0=\hbox{$#1$}%
  \kern-.025em\copy0\kern-\wd0
  \kern.05em\copy0\kern-\wd0
  \kern-.025em\raise.0433em\box0 }

\new{\bfs}{{\pmb \s}}
\new{\bfl}{{\pmb l}}
\new{\num}{\mu}
\newcommand{\evle}{\le_{\rm ev}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%    PAPER STARTS HERE    %%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}

\setcounter{section}{0}

\twocolumn[\titlestuff]
\vskip4.5cm

\ssection{Introduction}

\noindent 

The theory of viscosity solutions forms the basis for much current work
in optimal control and numerical analysis (cf. \cite{BCD97, BSP99, FS92}).
In two recent 
\footnotetext[1]{\supportnote}\footnotetext[2]{\ihpnote}
papers (cf. \cite{M99a}, \cite{M99b}),
we proved theorems characterizing the value function in deterministic
optimal control as the unique viscosity solution of the Bellman equation
that satisfies appropriate side conditions.  The results applied to a very
general class of problems whose dynamics and Lagrangians can
be 
unbounded and fully nonlinear, including cases where the targets are
unbounded and the dynamics are continuous but not Lipschitz
continuous.  In particular, the results apply to Fuller's Problem
(cf. \cite{ZB94} and $\S$2 below).
Uniqueness
characterizations of this kind have been used extensively to study
singular perturbations, synthesis of optimal controls, convergence of
numerical algorithms for
approximating value functions and optimal trajectories, and much more
(cf. \cite{BCD97}, \cite{FS92}, and the hundreds of references
therein).   However, these results do not apply to 
exit time problems whose dynamics and
Lagrangians  are unbounded and whose Lagrangians vanish at some points
outside the target for certain control values.  In fact, one easily
finds 
exit time problems with vanishing Lagrangians whose Bellman equations
admit more than one proper viscosity solution that vanishes on the
target, where by properness of a function $w$, we mean the condition
$w(x)\to +\infty$ as $||x||\to\infty$.
Indeed, recall the following case from \cite{M99a}:

\medbreak

\noindent{\bf Example}\ \   Use the dynamics $\dot x=u\in[-1,+1]$ and
use
$
\ell(x,a)=(x-2)^2 (x-1)^2 x^2 (x+1)^2 (x+2)^2
$
as the Lagrangian.  
We consider the problem 

\vspace{-.4em}

\[
\text{Minimize} 
\int_0^{t^\star}\ell(y_x(s,u),u(s))ds \text{\ over\ } \calu
\]

\vspace{-.4em}


\noindent for each $x\in\R^2$, where $\calu$ is the set of all measurable
functions 
$u:[0,\infty)\to[-1,+1]$, 
$y_x(\cdot,u)$ is the trajectory for the input $u\in\calu$ starting at
$x$, and   
$t^\star$
is the infimum of those times $s$ at which $y_x(s,u)\in \T_1:=\{0\}$.
 The value function $v$ for this problem, which is defined

\vspace{-.5em}

\[
v(x):= 
\inf\left\{\int_o^{t^\star}\ell(y_x(s,u),u(s))ds:\, u\in \calu,\, 
t^\star<\infty\right\},
\]

\vspace{-.5em}

\noindent is a proper viscosity solution of the corresponding Bellman
equation $|Dv(x)|-\ell(x)=0$ on $\R\setminus\{0\}$.  
The value function for the same problem but with the target $\T_1$ 
replaced by
$\T_2=\{-2,0,+2\}$ is also a proper viscosity solution of this Bellman   
equation.\eproof
\medbreak


The paper \cite{M99a} gave a uniqueness characterization for viscosity
solutions of Bellman equations for exit time problems  whose Lagrangians 
vanish for some
points outside the target.  The result of that paper applies to 
a very general class of 
problems
whose dynamics give positive running costs over any interval where the
state is outside the target, including Fuller's Problem,  and shows
that the value function is the
unique proper viscosity solution of the Bellman equation  which
vanishes at the target.  For related results, see \cite{S99}.

This note gives 
a different approach  which improves special cases of the 
result of \cite{M99a} by
proving 
that the 
value function for a class of problems including Fuller's Problem is the
unique viscosity solution
of the Bellman equation that vanishes at the target and is 
bounded below. We use the fact that all trajectories of these
problems whose total running costs over $[0,\infty)$ are finite 
tend to the origin (cf. $\S$3).   For the preceding example, this
condition is of course not satisfied, since the trajectory $x(t)\equiv 1$
generates a zero total running cost without reaching $\T_1$ or $\T_2$.   
\medbreak \noindent{\bf Remark}\ \  
The paper \cite{SM00} generalizes the result of this note to 
variable interest 
problems
with
discontinuous exit costs.  For such problems, one
minimizes    

\vspace{-.4em}


\[
\int_0^{t^\star}\! \! \ell(y_x(s,\beta),\beta(s))
e^{-\delta(s,x,\beta)}ds+
e^{-\delta(t^\star,x,\beta)}
g(y_x(t^\star,\beta)), 
\]


\vspace{-.4em}

\noindent over the measurable functions $\beta:[0,\infty)\to A$, where
$\delta(s,x,\beta):=\int_0^s h(y_x(r,\beta),\beta(r))dr$, $t^\star$ is the
exit time $\inf\{s\ge 0: y_x(s,\beta)\in \T\}$ to a general closed target
$\T$, and $\T$ and the control set $A$ may both be unbounded.   The
corresponding 
Bellman
equation is

\vspace{-.4em}

\[
\sup_{\scr a\in A}\{-f(x,a)\cdot Dv(x)-\ell(x,a)+h(x,a)\cdot v(x)\}=0,
\]

\vspace{-.4em}

\noindent where $f$ is the dynamics.  We can allow cases where 
$h$ is not bounded below by a positive constant.\eproof  


\ssection{Statement of Result}
Recall that if $F:\R^N\times\R^N\to\R$ is continuous and if
$\Omega\subseteq\R^N$ is open, then we say that a continuous function
$w:\R^N\to\R$ is a {\bf viscosity solution} of the equation $F(x,Dw(x))=0$
on $\Omega$
provided these conditions hold:
\begin{itemize}
\item[(1)] 
$F(\bar x, D\phi(\bar x))\le 0$ for every $C^1$ function
$\phi:\Omega\to\R$ and for every local maximum $\bar x$ of $w-\phi$.
\item[(2)] 
$F(\bar{\bar x}, D\mu(\bar{\bar x}))\ge 0$ for every $C^1$ function
$\mu:\Omega\to\R$ and for every local minimum $\bar{\bar x}$ of $w-\mu$.  
\end{itemize}
We say that a continuous function $\alpha:\R\to [0,\infty)$ is of {\bf 
class MK} provided $\alpha(0)=0$ and $\alpha$ is even and
strictly increasing on $[0,\infty)$.  We will take class MK
functions as our Lagrangians.  Note that functions of class
MK need not be bounded, convex, or Lipschitz.  

For each function $\alpha$ of class MK, we consider the following problem,
which we denote by $P_\alpha$:  

\vspace{-.5em}

\[
\text{Minimize} \int_0^{t_p(\beta)}\alpha[x_p(t,\beta)]dt \text{over} 
\beta\in\calu
\text{subject to}\]

\vspace{-2em}

\[ 
\dot x_p(t,\beta)=y_p(t,\beta),\; \;  
\dot y_p(t,\beta)=\beta(t), \; \; 
(x_p,y_p)(0)=p,
\]

\vspace{-.1em}

\noindent where  $t_p(\beta):=\inf\{t\ge
0:  x_p(t,\beta)=y_p(t,\beta)=0\}$, $\calu$ is as above, 
the dots are used to indicate time derivatives, and 
$z_q(t,\beta):=(x_q(t,\beta),y_q(t,\beta))$.  The value
function of $P_\alpha$, which we denote by
$v_\alpha$, is defined 
\[v_\alpha(p)=\inf\left\{\int_0^{t_p(\beta)}\! \! 
\alpha[x_p(t,\beta)]dt: \, \beta\in
\calu, \, t_p(\beta)<\infty\right\}. \]  For
the case of 
$\alpha(x)=x^2$,
we get  Fuller's Problem (FP), whose value function is denoted $v_{\scr
FP}$.  We prove the following result on the Bellman equation of
$P_\alpha$:

\medbreak

\noindent{\bf Theorem} {\it If $\alpha$ is of class  MK, and if
$w:\R^2\to\R$ is a viscosity
solution of the 
Bellman equation
\begin{equation}
\label{eq}
-x_2 D_1 w(x_1,x_2)+\left\vert D_2 w(x_1,x_2)\right\vert -\alpha(x_1)=0
\end{equation}
on $\R^2\setminus\{0\}$ which is bounded below and null  at
$\vec0$, then
$w(x)=v_\alpha(x)$ for all $x\in\R^2$.}

\medbreak

Notice that if $\alpha$ is also convex, then $v_\alpha$ is convex and
therefore continuous on $\R^2$.  
Therefore, $v_{\scr
FP}$ is the unique continuous 
viscosity solution of the corresponding
Bellman 
equation (\ref{eq}) for Fuller's Problem (with $\alpha(x)=x^2$) which is
null at $0$ and bounded below.  The hypotheses of the theorem are
minimal, since one can find solutions of (\ref{eq}) which are null
at the origin but not bounded below.  For details, see \cite{SM00}.

\ssection{Lemmas}
To prove our theorem, we need the following converse dynamic
programming inequalities 
and generalization of Barb\v{a}lat's Lemma:
\bl
\label{lemma0}
Let $\alpha$ be a function of class MK.  Let $w$ be a viscosity
solution of (\ref{eq}) on $\R^2\setminus\{0\}$ and $B\subseteq
\R^2\setminus\{0\}$  be a bounded open set for which $0\not\in\bar
B$.  Let $q\in B$.  If $\beta\in \calu$ and $0\le
r<\inf\{t\ge 0: z_q(t,\beta)\in \partial B\}$, then 
\begin{equation}
\label{e1}
w(q)\le \int_0^r \alpha[x_q(t,\beta)]dt +w[z_q(r,\beta)].
\end{equation}
Moreover, we have
\begin{equation}
\label{e2}
w(q)\ge \inf_{\beta\in \calu}\left\{\int_0^{\bar t} 
\alpha[x_q(t,\beta)]dt
+w[z_q(\bar t,\beta)]\right\}
\end{equation}
for all $0\le \bar t<\inf\{t\ge 0:  {\rm dist}
(z_q(t,\beta),\partial B)<\delta, \beta\in \calu\}$ and all
$0<\delta<{\rm dist}(q,\partial B)$.\el

\bpr
This is a special case of the proof of Theorem III.2.32 of
\cite{BCD97}.\epr\eproof
\bl
\label{lemma1}
Let $\alpha$ be of  class MK.  
If $\phi:[0,\infty)\to\R$ is differentiable and $\phi'$ is Lipschitz
continuous, and if 
$\int_0^\infty \alpha\left[\phi(s)\right]ds<\infty$,
then 
$\ds\lim_{t\to\infty}\phi(t)=\lim_{t\to\infty}\phi'(t)=0$.\el
\bpr It suffices to check that $\lim_{t\to\infty}\phi'(t)=0$, since then
$\phi$ is Lipschitz, and then the result follows by 
Barb\v{a}lat's Lemma.  We suppose that
 $\limsup_{t
\to\infty}\phi'(t)>\gamma\in(0,1)$ wlog.  (Otherwise, apply the same
argument
to $-\phi$.)
By passing to a subsequence, we can choose $t_k\uparrow +\infty$ for which 
$t_{k+1}-t_k\ge 1$ and $\phi'(t_k)>\gamma$ for all $k$.  
Choose a Lipschitz constant $C>1$ for $\phi'$, pick
$0<\delta<\gamma/(2C)$, and set $I_k=[t_k-\delta, t_k+\delta]$.  Then 
\begin{equation}
\label{effect}
\phi'(t)=\phi'(t)-\phi'(t_k)+\phi'(t_k)\ge 
 -C\, \delta+\gamma\ge\gamma/2\end{equation} for all $t\in I_k$.  
Set
$
v_k=\inf\{|\phi(t)|: t\in I_k\}$ for all $k$, and choose
$s_k\in I_k$ so that $|\phi(s_k)|=v_k$.  

If $\phi(s_k)> 0$, then  (\ref{effect}) gives $s_k=t_k-\delta$, so since
$\phi>0$ on $I_k$, we
get $\phi(t)\ge \gamma[t-(t_k-\delta)]/2$ for all $t\in I_k$, so 
\begin{equation}
\label{summand}
\int_{I_k}\alpha[\phi(s)] ds\; \ge\;
2/\gamma \int_0^{\gamma\delta/4}\alpha(u)du\end{equation} by changing
variables.  
This last inequality remains true if $\phi(s_k)<0$ (in which case
$s_k=t_k+\delta$ and we have 
$-\phi(t)\ge \gamma[t_k+\delta-t]/2$ on
$I_k$) or if 
$\phi(s_k)=0$ (in which case we argue as in the $\phi(s_k)>0$ case on 
$[s_k,t_k+\delta]$ and as in the $\phi(s_k)<0$ case on
$[t_k-\delta,s_k]$).
Since $\delta<1/2$ and $t_{k+1}-t_k\ge 1$ for all $k$, the
$I_k$'s are disjoint, so we sum on $k$ in (\ref{summand}) to
contradict  
$\int_0^\infty\alpha[\phi(s)]ds<\infty$.
\epr\eproof
\ssection{Proof of Theorem}
Let $\alpha$ and $w$ be as in the hypotheses.  The fact that 
$w\le v$ on
$\R^2$ 
follows easily from a repeated application of the inequality 
(\ref{e1}) (cf. \cite{BCD97}).
To prove the reverse inequality, fix $\bar x\in
\R^2$ and $\eps>0$.  We use Zorn's Lemma to find a $\bar \beta\in
\calu$ such that $t_{\bar x}(\bar\beta)<\infty$ and 
\begin{equation}
\label{goal}
w(\bar x)\ge \int_0^{t_{\bar x}(\bar \beta)}\alpha[x_{\bar
x}(t,\bar\beta)]dt-\eps\end{equation}
 and let $\eps\downarrow 0$ to
conclude. 
In what follows, we set $f(x,y,a)=(y,a)$, $\ell(x)=\alpha(x_1)$ for all
$x\in \R^2$, and 
\[E_j(t)\equiv \eps/4 \left[
e^{-(j-1)}-e^{-(t+j-1)}\right]\] for all $t\ge 0$ and $j\in \N$.  
(Our arguments easily generalize to cases where the instantaneous cost
$\ell$ also depends on the control value.)  Consider 

\[
{\cal Z}_1=\left\{\! \! 
\begin{array}{l}
(t,\gamma):  \; \; 0\le t\le 1$, \text{and} $\gamma:
[0,t]\to \R^2 \text{is a}\\[.4em] \!  \! \text{trajectory 
for} \dot z=f(z,\beta ) \text{for \ $\! $some}
\beta\, \in\, {\cal U}\\[.5em] \! \!  \text{for which}  \gamma(0)\, =\, 
\bar x \text{with the property that}\\[.4em]  w(\bar x) \; \ge\; 
\int_0^t \ell(\gamma(s)) ds\; +\; w(\gamma(t))\; -\;  E_1(t)\, .
\end{array}
\! \! \! \! \right\}.
\]
Then ${\cal Z}_1$ is partially ordered by the relation $\sim$ defined by 
$
(t_1,\gamma_1)\sim (t_2, \gamma_2) \iff 
\left(t_1\le t_2 \; \, \&\, \;    
\gamma_2\lceil[0,t_1]\equiv \gamma_1\right)$.  
Moreover, one can use standard arguments to check that every
totally ordered subset of ${\cal Z}_1$ has an upper bound in
${\cal Z}_1$.
(Let $\{(s_j, \mu_j)\}_j\subset {\cal Z}_1$ be totally ordered, 
set $s_o=0$ and $\bar s=\sup_j s_j$, and define 
$\bar \mu:[0,\bar
s)\to\R^N$ by $\bar\mu(p)=\mu_j(p)$
for any $j$ such that $s_j>p$. 
Let
$\beta_j:(s_{j-1}, s_j]\to [-1,+1]$ be a control for the trajectory
$\mu_j\lceil (s_{j-1}, s_j]$, and define
$\beta^\dagger:(0,\infty)\to [-1,+1]$ by $\beta^\dagger(s):=\beta_j(s)$
on $(s_{j-1}, s_j]$ for $j\ge 1$ and $\beta^\dagger(s)=1$ for
$s\ge \bar s$.
Then $\beta^\dagger$ is measurable and 
admits a trajectory $z_{\bar x}(\cdot,
\beta^\dagger)$ on $[0,\infty)$. Since $E_1$ and $w$ are continuous, the
desired upper bound is $(\bar s, z_{\bar x}(\cdot, \beta^\dagger))\in
{\cal
Z}_1$.)

It follows from Zorn's Lemma that ${\cal Z}_1$ contains a maximal
element which we will denote by $(\bar t, \bar \gamma)$. Wlog,  $\bar
\gamma$ does not reach $0$ on $[0,\bar t\,
]$. (Otherwise the requirement (\ref{goal}) 
is satisfied if we use the
input
for $\bar \gamma$). We show that $\bar t=1$. Let $B$ be
an open set containing $\bar \gamma(\bar t\, )=:\hat x$ with $0\not\in\bar
B$.
Suppose $\bar t<1$.  Set $\tau=\inf\left\{t:
\dist(z_{\hat x}(t,\beta),\partial B)\le \dist(\hat
x,\partial B)/2, \beta\in \calu\right\}$. By standard
estimates (cf. \cite{BCD97}, Chapter 3), one checks that 
$\tau>0$.
By Lemma \ref{lemma0}, it follows that there are $t\in (0,1-\bar t\, )$
and
$\beta\in {\cal
U}$ so
that
\begin{eqnarray}
\label{newone}
 w(\hat x)&\ge&\int_0^t
\ell(z_{\hat x}(s,\beta))\, ds\; \; +\; \;  
w[z_{\hat x}(t, \beta)]\nonumber\\&-&E_1(\bar t+t)\; \; +\; \;  
E_1(\bar t\, )\end{eqnarray}
\noindent and so that $z_{\hat x}(\cdot, \beta)$ remains in $B$ on
$[0,t]$. Let
$\tilde \alpha$
denote the control for $\bar\gamma$. Since $(\bar t, \bar
\gamma)\in {\cal Z}_1$, it follows that
\begin{equation}
\label{secondnewone}
 w(\bar x)-w(\hat x)\; \; \ge\; \; \int_0^{\bar t}\ell(z_{\bar
x}(s,\tilde\alpha))\, ds\; -\; E_1(\bar t\, ).
\end{equation}
\noindent Let $\beta^\sharp$ denote the concatenation of 
the input $\tilde \alpha\lceil [0, \bar t]$ followed by the
input $\beta$.  Since $t+\bar t<1$, we can add 
(\ref{newone}) and
(\ref{secondnewone}) to conclude that $(t+t, z_{\bar
x}(\cdot, \beta^\sharp))\in {\cal Z}_1$. This contradicts
the maximality of $(\bar t, \bar \gamma)$, so $\bar t=1$.

We now extend $\bar \gamma$ to get the desired
trajectory.  Set

\[
{\cal Z}_2=\left\{\! \! 
\begin{array}{l} 
(t,\gamma):  \; \; 0\, \le \, t\, \le \, 1$, \text{and} $\gamma:
[0,t]\to \R^2 \text{is}\\[.4em] \!  \! \text{a trajectory 
for} \dot z=f(z,\beta ) \text{for \ $\! $some\ input}\\[.5em]
\beta\, \in\,\calu  \text{which is such that}
\gamma(0)\, =\, 
\bar\gamma(1) \text{and}\\[.35em]  w(\bar \gamma(1)) \; \ge\; 
\int_0^t \ell(\gamma(s)) ds+w(\gamma(t))- E_2(t)\, .
\end{array}\! \! 
\! \! \! \right\}.
\]
and partially order ${\cal Z}_2$ as before.  By
the preceding, we know that ${\cal Z}_2$ contains a
maximal element $(1,\tau_2)$.  Let $u_2:[0,1]\to [-1,+1]$ denote the
input for $\tau_2$.  We can assume wlog that $\tau_2$ does not
reach $0$ on $[0,1]$.  Indeed, if there is an $s_2\in [0,1]$ such
that $\tau_2(s_2)=0$, then we can satisfy the requirement
by concatenating $\tilde \alpha$ followed by $u_2$,   
where $\tilde\alpha$ is the control for $\bar \gamma$ on $[0,1]$.
Let $c_2$ denote the concatenation of $\tilde\alpha\lceil[0,1]$
followed by $u_2\lceil[0,1]$, and let $\gamma_2$ denote the
corresponding concatenated trajectory for $c_2$ from $\bar x$ to
$\tau_2(1)$.

Now reapply the procedure to $\gamma_2(2)$ to get a longer
trajectory that reaches $0$ or runs for  three time units. 
The procedure
is iterated, and it results in a
sequence of terminal points $p_n:=\phi_n(n)$ for trajectories $\phi_n$
which begin at $\bar x$, are defined on $[0,n]$, and wlog do not reach
$0$.  
It follows that for all $n\in\N$, we
would have
\begin{equation}
\label{trips} w(\bar x)\; \; \ge \; \; \int_0^n \ell(\phi_n(s,
c_n))\, ds\; +\; w(\phi_n(n))\; -\;
\frac{\eps}{2}.
\end{equation}
Setting
$\hat \alpha(s)=c_n(s)$ if $n-1<s\le n$, letting $\hat \phi$
denote the corresponding trajectory, and letting $b$ denote a
lower bound for $w$, a passage to the limit as $n\to\infty$ in
(\ref{trips}) gives
$
\int_o^\infty \ell(\hat\phi(s)) ds \le w(\bar x)-
b+\eps/2< \infty$,
since $\hat\phi\equiv \phi_n$ on $(n-1, n]$.   It follows from Lemma
\ref{lemma1}  that
$\lim_{s\to\infty}\hat\phi(s)=0$.
Since the FP dynamics is controllable to the origin and $w$ is
continuous, standard estimates (cf. \cite{BCD97}) guarantee $m\in \N$ and
$\beta_m\in \calu$ for which
$w(\hat\phi(m))\ge -\eps/4$, $t_{p_m}(\beta_m)<\infty$, and 
$\int_0^{t_{p_m}(\beta_m)}\ell(z_{p_m}(s,\beta_m))ds<\eps/4$.  We 
then
satisfy 
(\ref{goal}) by
taking $\bar\beta$ to be $\hat\alpha\lceil[0,m]$
followed by $\beta_m$.\eproof

\medbreak
\noindent{\bf Remark}\ \ One can prove a 
variant of the theorem 
for solutions of (\ref{eq}) on open
sets $\Omega\subseteq \R^2$ with $\{0\}$ replaced by a general closed 
target $\T$.  To do this,
we add the condition 
$\gamma:[0,1]\to\Omega$ in the ${\cal Z}_i$'s and assume 
\[
w(x)<\bar w \; \; \forall x\in \Omega\; \; \; \text{and}\; \;  \lim_{x\to
x_o}w(x)=\bar w
\;  \; \forall x_o\in \partial\Omega
\]
for some $\bar w\in\R\cup\{+\infty\}$ (cf.  \cite{BCD97}).  
The proof of the inequality $w\le v_\alpha$ is  more involved
since one must
consider trajectories which exit $\Omega$ before reaching $\T$.  To cover
cases where $\T\not\subseteq \Omega$, we assume 
$\partial\Omega\subseteq \T$.   For details,
see \cite{SM00}.  
\eproof\medbreak

\vspace{-1.5em}

\begin{thebibliography}{10}

\vspace{-.4em}

\bibitem{BCD97}
Bardi, M., and I. Capuzzo-Dolcetta, {\em Optimal Control and
Viscosity
Solutions of Hamilton-Jacobi-Bellman Equations}, Birkh{\"a}user,
Boston,
1997.\vspace{-.2em}

%\bibitem{BDL97}
%Bardi, Martino, and Francesca Da Lio,  ``On the Bellman equation
%for some unbounded control problems,'' {\em NODEA Nonlinear
%Differential Equations Appl.}, {\bf 4}(1997), pp. 276-285.\vspace{-.2em}

\bibitem{BSP99}
Bardi, Martino, T.E.S. Raghavan, and T. Parthasarathy, Editors, 
{\em
Stochastic and Differential Games: Theory and Numerical Methods},
Birkh\"auser, Boston, 1999.\vspace{-.2em}

%\bibitem{BS91}
%Bardi, Martino, and Pierpaolo Soravia, ``Hamilton-Jacobi equations
%with singular boundary conditions on a free boundary and
%applications to differential games,'' {\em Trans. Amer. Math.
%Soc.},  {\bf 325}(1991), pp. 205-229.\vspace{-.2em}

%\bibitem{meagre}
%Desch, W., H. Longemann, E.P. Ryan, and E.D. Sontag, ``Meagre
%functions and asymptotic behavior of dynamical systems,''
%preprint. (To appear in {\em Nonlinear Analysis})


%\bibitem{DAL}
%Da Lio, F., ``On the Bellman equation for infinite horizon
%problems with unbounded cost functional,'' {\em Appl. Math.
%Optim.}, {\bf 41}(1999), pp. 171-197.


\bibitem{FS92}
Fleming, Wendell H., and Halil M. Soner, {\em Controlled Markov Processes
and
Viscosity
Solutions}, Springer, New York, 1993.\vspace{-.2em}



\bibitem{M99a}
Malisoff, M., ``On the Bellman equation for optimal control problems with
exit times and unbounded cost functionals,'' in 
{\em Proc. 38th IEEE CDC},
 IEEE,
New York, 1999, pp. 23-8.\vspace{-.2em}

\bibitem{M99b}
Malisoff, M., ``A remark on the Bellman equation for optimal
control problems with exit times and noncoercing dynamics,'' in
{\em Proc. 38th IEEE CDC},
 IEEE,
New York, 1999, pp. 877-881.\vspace{-.2em}



%\bibitem{R70}
%Rockafeller, R.T., {\em Convex Analysis}, Princeton University
%Press, Princeton, NJ, 1970.\vspace{-.2em}

\bibitem{S99}
Soravia, P., ``Optimality principles and representation formulas for
viscosity solutions of Hamilton-Jacobi equations I:  Equations of 
unbounded and degenerate control problems without uniqueness,''
{\em Adv. Diff. Eq.}, {\bf 4}(1999), pp. 275-296.  


\bibitem{SM00}
Sussmann, H.J., and M. Malisoff, ``Further results on the Bellman equation
for exit time optimal control problems with nonnegative Lagrangians,''
preprint.\vspace{-.2em}  

\bibitem{ZB94}
Zelikin, M.I., and V.F. Borisov, {\em Theory of Chattering Control
with Applications to Astronautics, Robotics, Economics, and
Engineering},  Birkh{\"a}user, Boston, 1994.

\end{thebibliography}
\end{document}



