%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%  
%            This is a LateX file, containing  the manuscript of
%
%                  THREE CENTURIES OF CURVE MINIMIZATION:
%      FROM THE BRACHISTOCHRONE TO MODERN OPTIMAL CONTROL THEORY
%
%                  by H. J. Sussmann and J. C. Willems
%
%                Date of this version:  GNU GNUember GNUU
%
%             THIS VERSION SUPERSEDES ALL PREVOUS VERSIONS
%
%                    The paper is GNU pages long.
%
%       You should run Latex GNU times on this file. After the GNUth run,
%      there will still be several Underfull \hbox,  Underfull \vbox,  
%      and Overfull \vbox messages (but you should not get any 
%      Overfull \hbox messages!)  Just ignore them and print the .dvi file.
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% AMS-LaTeX 1.2 sample file for book proceedings, based on amsproc.cls.
%
\documentclass[twoside,11pt]{book}
\usepackage{amsfonts,amsthm,amsmath,amssymb,latexsym}
\usepackage{graphics}
\usepackage{times,mathptmx}
%\usepackage{eucal}
\usepackage{epsf}
\setcounter{secnumdepth}{3}
\def\fsh{F_{\hat\Sigma}}
\def\unde{\underline}
\def\axts{{\cal A}_{x,{\cal T}}^{\hat\Sigma}}
\def\vts{{\cal V}_{{\cal T}}^{\hat\Sigma}}
\newcommand{\asjgoes}{\longrightarrow\!\!\!\!\!\!\!\!\!\!\!\!_{_{j\to\infty}}\,\,}
\newcommand{\evset}{{\bf EPS}}
\newcommand{\rset}{{\bf RS}}
\newcommand{\U}          {\mathbb{U}}
\newcommand{\B}          {\mathfrak{B}}
\newcommand{\N}          {\mathfrak{N}}
\newcommand{\Rc}          {\mathbb{R}^\c}
\renewcommand{\P}          {\mathfrak{P}}
\newcommand{\T}          {\mathbb{T}}
\newcommand{\W}          {\mathbb{W}}
\newcommand{\LL}         {\mathbb{L}}
\newcommand{\Rw}      {\mathbb{R}^{\tt w}}
\newcommand{\Bfull}      {\B_{\rm full}}
\newcommand{\rowdim}      {{\rm rowdim}}
\newcommand{\coldim}      {{\rm coldim}}
\newcommand{\rank}      {{\rm rank}}
\newcommand{\Z}         {\mathbb{Z}}
\newcommand{\C}         {\mathbb{C}}
\newcommand{\ba}{\begin{array}}
\newcommand{\ea}{\end{array}}
\newcommand{\Hinf}      {\mathfrak{H}_\infty}
\renewcommand{\c}      {{\tt c}}
\newcommand{\w}      {{\tt w}}
\newcommand{\forheadtitle}{CURVE MINIMIZATION}
\newcounter{gnucount}
\setcounter{gnucount}{0}
\newcommand{\gnu}[1]{\refstepcounter{gnucount}
\medbreak
\begin{center}
\fbox{\begin{minipage}{12cm}
\noindent COMMENT NO. \thegnucount.\ \ #1%\ COMMENT-END
\end{minipage}}\end{center}
\medbreak}

%Uncomment the following  line to run file without printing the gnus
%\renewcommand{\gnu}[1]{}
\input epsf
%\usepackage{makeidx}
\parskip=0.8mm
\def\bb{\mathbb{B}}
\def\brokarrow{\,\hbox to0.8mm{$\i$\hfill}
\raise0.6pt\hbox to6mm{-\hfill-\hfill-\hfill-\hfill}\!\!\!\!>\,}
\newcommand{\idx}{\index}
\def\idrn{{\rm id}_{\rr^n}}
\def\Om{\Omega}
\def\comment#1{}
\def\snell{Snell}
\def\traj{{\rm Traj\,}}
\def\llist#1{
\vspace{-4.5mm}
\begin{center}
$\quad\quad\,\,$\mbox{\begin{minipage}{12.2cm}
#1
\end{minipage}}\end{center}}

\def\lllist#1{
\vspace{-4.5mm}
\begin{center}
$\quad\quad\quad$\mbox{\begin{minipage}{11.9cm}
#1
\end{minipage}}\end{center}}

\newcommand{\compo}{\circ}
\def\transv{{\cap\kern-5.2pt\lower -2.7pt\hbox{$|$}\kern-5.7pt\lower
-8.1pt\hbox{$-$}}} 
\def\baba{|\kern-1.7pt|}
\def\stransv{{\cap\kern-5.8pt\lower -2.7pt\hbox{$\baba$}\kern-6.2pt\lower
-8.1pt\hbox{$-$}}} 
\newcommand{\eps}{\varepsilon}
\def\verylongrightarrow{\raise2.2pt\hbox to 25pt{\hrulefill}\kern-4pt\longrightarrow}
\def\verylongline{\raise2.2pt\hbox to 37pt{\hrulefill}}
\def\ovarrow#1{{\stackrel{#1}{\verylongrightarrow}}}
\def\ovline#1{{\stackrel{#1}{\verylongline}}}
\def\ovlarrow#1{{\stackrel{#1}{\verylongleftarrow}}}
\def\verylongleftarrow{\longleftarrow\kern-4pt\raise2.2pt\hbox to 25pt{\hrulefill}}
\def\longsearrow{\setlength{\unitlength}{0.5cm}
\begin{picture}(1.2,1.2)
\put(-0.2,1.2){\vector(1,-1){2}}
\end{picture}}
\def\longnearrow{\setlength{\unitlength}{0.5cm}
\begin{picture}(1.2,1.2)
\put(-0.2,-1.2){\vector(1,1){2}}
\end{picture}}
\def\longdownarrow{\setlength{\unitlength}{0.25cm}
\begin{picture}(0.6,1)
\put(0.2,1){\vector(0,-1){2}}
\end{picture}}
\def\longuparrow{\setlength{\unitlength}{0.25cm}
\begin{picture}(1.2,1.2)
\put(0.2,-0.5){\vector(0,1){2}}
\end{picture}}
\def\veceps{\vec{\eps}} 
\def\vic{\pmb}
\def\Is{I^{\#}}
\def\Xis{\Xi^{\#}}
\def\xis{\xi^{\#}}
\def\etas{\eta^{\#}}
\def\zetas{\zeta^{\#}}
\def\qs{q^{\#}}
\def\ts{t^{\#}}
\def\taus{\tau^{\#}}
\def\nus{\nu^{\#}}
\def\restr{\,\lceil\,}
\def\corestr{\,\rceil\,}
\def\equaldef{{\buildrel \rm def \over =}}
\def\mybox{$\lower3.5pt\vbox{\hrule height2.6pt depth-2pt\hbox{\vrule width0.6pt\kern1pt\vbox{\kern1pt\phantom{nj}\kern1pt}\kern1.5pt\vrule width0.6pt}\hrule height2.6pt depth-2pt}$\medbreak}
\def\proofendbox{\enspace\hfill \mybox\bigskip}
\def\proofendsq{\hfill $\square$\medbreak}
\def\proofend{\proofendsq}
\def\pmb#1{\setbox0=\hbox{$#1$}%
  \kern-.025em\copy0\kern-\wd0
  \kern.05em\copy0\kern-\wd0
  \kern-.025em\raise.0433em\box0 }
\def\pmbo#1{\setbox0=\hbox{$#1$}%
  \kern-.055em\copy0\kern-\wd0
  \kern.05em\copy0\kern-\wd0
  \kern-.055em\raise.0433em\box0 }
\def\bem{\bf\em }
\def\bemu{{\bem U}}
\def\tto{\longrightarrow\kern-9pt\longrightarrow}
\def\zz{\mathbb{Z}}
\def\qq{\mathbb{Q}}
\def\cc{\mathbb{C}}
\def\rr{\mathbb{R}}
\def\rrinf{\overline{\rr}}
\def\nninf{\overline{\nn}}
\def\rrsmall{{\mysmall I}\kern-.2em{\mysmall R}}
\newcommand{\der}{\frac{d}{dt}}
\newcommand{\R}{\rr}
\newcommand{\Rn}{\rr^{n}}
\newcommand{\Rm}{\rr^{m}}
\newcommand{\Rp}{\rr^{p}}
\newcommand{\ABCD} 
{{\footnotesize{$
{\left[ {\begin{array}{c|c} A&B\\ \hline C&D \end{array}} \right]}$}}}
\newcommand{\Rnn}        {\mathbb{R}^{\tt n \times n}}
\newcommand{\Rpm}        {\mathbb{R}^{\tt p \times m}}
\newcommand{\Rnm}        {\mathbb{R}^{\tt n \times m}}
\newcommand{\Rpn}        {\mathbb{R}^{\tt p \times n}}
\def\nn{\mathbb{N}}
\def\kk{\mathbb{K}}
\def\bmath{\mathbb{B}}
\def\smath{\mathbb{S}}
\def\zmath{\mathbb{Z}}
\newcommand{\nball}{{\bmath}^n}
\def\tub{\mathcal{T}}
\def\tubx{\mathcal{T}^X}
\def\btub{\bar\mathcal{T}}
\def\btubx{\bar\mathcal{T}^X}
\def\linx{{\rm Lin}(X,X)}
\def\linxxp{{\rm Lin}(X,X')}
\def\id{\kern.2em{\rm I}\kern-.56em{\rm 1}}
\def\idnpball{\id_{\partial\bmath_n}}
\def\so{{\rm So}}
\def\ta{{\rm Ta}}
\def\gr{{\rm Gr}}
\def\dom{{\rm domain}}
\def\im{{\rm Im}}
\def\togr{\,\,{\buildrel \rm igr \over \longrightarrow}\,\,}
\def\caF{\mathcal{F}}
%\def\spaco{\ \ \ \ \ \ }
%\newcounter{subsectco}
%\setcounter{subsectco}{0}
\newcommand{\ssection}[1]{\section[$\quad$#1]{#1}}
%\newcommand{\ssubsection}{\section}
%\newcommand{\ssubsubsection}{\subsection}
%\newcommand{\ssubsubsubsection}{\subsubsection}
%\newcommand{\ssubsection}[1]{\refstepcounter{subsectco}\section*{
%\S \thechapter\thesubsectco.\ #1}}
%\newcounter{subsubsubco}
%\setcounter{subsubsubco}{0}
%\newcommand{\ssubsubsubsection}[1]{\refstepcounter{subsubsubco}
%\subsubsection*{\addcontentsline{toc}{subsubsection}{\S \thesubsubsubco.\
%    \ #1}}}
%\newcommand{\ssubsubsubsection}[1]{\refstepcounter{subsubsubco}
%\subsubsection*{\S \thechapter.\thesection.\thesubsubsubco.\ Vaca\ #1}}
%\newcommand{\ssubsubsubsection}[1]{
%\subsubsection*{\S \thechapter.\thesectco.\thesubsubsubco.\ Vaca\ #1}}
\newcommand{\pparagraph}[1]{\addcontentsline{toc}
{subsection}{$\quad$\small{#1}}
\paragraph{{\large #1.}}}
\newcommand{\ben}{\begin{enumerate}}
\newcommand{\een}{\end{enumerate}}
\newtheorem{theorem}{Theorem}[chapter]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{example}[theorem]{Example}
\newtheorem{definition}[theorem]{Definition}
\newcommand{\bfact}{\begin{fact}}
\newcommand{\efact}{\end{fact}}



\newtheorem{xca}[theorem]{Exercise}

%\numberwithin{equation}{section}

%    Absolute value notation
\newcommand{\abs}[1]{\lvert#1\rvert}

%    Blank box placeholder for figures (to avoid requiring any
%    particular graphics capabilities for printing this document).
\newcommand{\blankbox}[2]{%
  \parbox{\columnwidth}{\centering
%    Set fboxsep to 0 so that the actual size of the box will match the
%    given measurements more closely.
    \setlength{\fboxsep}{0pt}%
    \fbox{\raisebox{0pt}[#2]{\hspace{#1}}}%
  }%
}

\setcounter{tocdepth}{3}

\begin{document}
\setcounter{page}{-0}
\thispagestyle{empty}
\begin{center}
{\Huge {\bf Three Centuries\\
of Curve Minimization:\\
from the Brachistochrone\\
to Modern\\
Optimal Control Theory\\
}}

\vspace{1cm}

{\Huge H\'ector J.\ Sussmann}
\medbreak
{\Large  Department of Mathematics - Rutgers University\\
Hill Center -- Busch Campus
110 Frelinghuysen Road 
Piscataway, NJ 08854-8019 -- U.S.A.
\medbreak
{\it sussmann@hilbert.rutgers.edu}}
\bigbreak
{\Huge Jan C. Willems}\\
\medbreak
{\Large Department of Mathematics -- University of Groningen\\
P.~O.~Box 800 -- 9700 AV Groningen\\
The Netherlands\\
\medbreak
{\it J.C.Willems@math.rug.nl}}
\end{center}
\vspace{0.5cm}
\begin{center}\fbox{\fbox{\begin{minipage}{12cm}
\begin{center}
{\Large {\bf THIS IS A DRAFT.}\\
\vskip 0.02cm
{\bf DO NOT QUOTE WITHOUT PERMISSION.}\\
\vskip 0.02cm
{\bf Date of this draft: \today.}}
\end{center}
\end{minipage}}}
\end{center}
\medbreak\noindent AMS Subject classification:
Primary 54C40, 14E20; Secondary 46E25, 20C20. 

\newpage
\thispagestyle{empty}$\qquad$

\newpage\thispagestyle{empty}

\begin{center}
{\Large\bf  ACKNOWLEDGEMENTS}
\end{center}


\medbreak 
\noindent H. Sussmann's research was supported in part by NSF Grant DMS-9803411
and AFOSR Grant 0923. 

A large part of his work was done in the Netherlands, 
while he was ``Johann Bernoulli visiting professor''
at the University of Groningen in May, June and July 1999.
He is immensely grateful to the University for its generous  hospitality and
exciting intellectual atmosphere.

We are grateful to 

\begin{itemize}
\item[$\bullet$] {\bem Andrei Agrachev} for having pointed out to us in 1993 
that the anniversary of Johann Bernoulli's challenge was be celebrated
in three years, and that this was an excellent opportunity 
for reclaiming this event in writing as a significant step 
in the history of optimal control,

\item[$\bullet$] {\bem Tom Archibald}, for his comments on the spurious
solutions of the differential equation derived by Johann Bernoulli in 
for the brachistocrone, 

\item[$\bullet$] {\bem Franco Cardin}, for acquainting us witht
he work of W.~M.~Tulczyjew,

\item[$\bullet$] {\bem Francis H. Clarke}, for pointing out to us in
September 1995 that the existence problem for the 
brachistochrone---which, as will be explained in \S \ref{subsec-existence},
is fairly easy to handle in the optimal control framework---is a delicate 
question in the classcial calculus of variations, due to the noncoercivity 
and the singular denominator of the Lagrangian.

\item[$\bullet$] {\bem Jean-Paul Gauthier} and {\bem Roe Goodman},
for helping us understand the role of the diamond Lie algebra, 

<\item[$\bullet$] {\bem Hans W. Knobloch}, 
for bringing to our attention the passage in C. Ca\-ra\-th\'eo\-dory's
book that we discuss in \S\ref{subsec-carat},

\item[$\bullet$] {\bem Geert Jan Olsder}, for providing helpful information
on Dutch discoveries in the South Pacific and for many other insightful
comments, 

\item[$\bullet$] {\bem Annye K. Sussmann}, for her careful reading of the
manuscript and her invaluable suggestions,  

\item[$\bullet$] {\bem M. Vidyasagar},  for
bringing to our attention an inaccurate statement about Dr. Watson and
Sherlock Holmes that appeared in an earlier draft.
\end{itemize}



\newpage
\thispagestyle{empty}$\qquad$

\newpage\thispagestyle{empty}





\title{Three centuries of curve minimization;
from the brachistochrone to modern optimal control theory}

%    Information for first author
%\author{Author One}
%    Address of record for the research reported here
%\address{Department of Mathematics, Louisiana State University, Baton
%Rouge, Louisiana 70803}
%    Current address
%\curraddr{Department of Mathematics and Statistics,
%Case Western Reserve University, Cleveland, Ohio 43403}
%\email{xyz@math.university.edu}
%    \thanks will become a 1st page footnote.
%\thanks{The first author was supported in part by NSF Grant \#000000.}

%    Information for second author
%\author1{H\'ector J.\ Sussmann}
%    Address of record for the research reported here
%\address{Department of Mathematics,
%Rutgers, the State University of New Jersey, 
%Hill Center---Busch Campus, 
%110 Frelinghuysen Road, 
%Piscataway, NJ 08854-8019. U.S.A.}
%\email{sussmann@hilbert.rutgers.edu}

%    \thanks will become a 1st page footnote.

%\author2{Jan C. Willems}
%\address{Department of Mathematics, University of Groningen, 
%P.~O.~Box 800, 9700 AV Groningen, The Netherlands} 
%\email{J.C.Willems@math.rug.nl}

%    General info

\date{January 1, 1994 and, in revised form, June 22, 1994.}

%\dedicatory{We dedicate this book to ourselves}

\begin{center}
{\Large\bf Abstract}
\end{center}

\medbreak
\noindent On the occasion of the tricentennial of the 1697 issue of
{\em Acta Eruditorum}, containing work on the brachistochrone problem
by Johann Bernoulli, his
brother Jakob, Newton, Leibniz, l'H\^{o}pital, and Tschirnhaus,
we outline how the process that started 300
years ago has led to modern optimal control theory.  We describe the
circuitous path going through the Euler-Lagrange equations, the
Legendre-second order condition, Hamilton's reformulation of the
necessary conditions, and Weierstrass' side condition, to the modern
maximum principle of optimal control. We emphasize the occasions when
opportunities were missed, and the protagonists of our 
story---especially Hamilton and Weierstrass---came close to finding the
modern general version of the necessary conditions, but failed to do so.  We
trace this failure to the fact that the Hamilton equations of motion
were written in terms of a ``Hamiltonian'' which is a function of the
positions and momenta only, a form which obscures the basic facts that
are immediately revealed by using a different ``Hamiltonian,'' which
is a function of the velocities as well as the momenta and the
positions.  We then outline the recent evolution of the necessary conditions
beyond the classical version of the maximum principle, and describe
the connections with nonsmooth analysis and differential geometry.  In
the last part of the paper, we look at the brachistochrone problem and
show how the various steps in the evolution of the necessary
conditions for optimality provide new insights into the original
problem.  In particular, we show how the research program---sketched, for
example, by J.~L.~Synge in 1944---to describe all properties of
interest of a curve optimization problem in terms of ``Hamilton's
principal function,'' when applied to the brachistochrone problem,
naturally leads to a ``differential-geometric analysis,'' in which the
problem is formulated in 4-space, and an interesting Lie-algebraic
structure arises---involving a 4-dimensional Lie algebra (the
``diamond Lie algebra'') and a 3-dimensional distribution---as well
as nontrivial analytic questions, due to the presence of unbounded
controls, and resulting in solutions that contain impulse controls
We conclude that the brachistochrone is a true optimal control problem,
in the sense that only the optimal control formulation fully 
brings out the rich structure associated to what Leibinz called 
a ``splendid problem.''

\gnu{This abstract is a relic from the old days when we 
thought of this as a paper rather than a book. I suppose now that it's
a book the abstract is no longer needed.}



\newpage
\thispagestyle{empty}$\qquad$
 



\tableofcontents


\gnu{We  have to fix the subject classification footnote, by putting 
in the correct classification numbers. (The numbers that are there
were put in for formatting purposes only. They come from a model style
file we got, and have nothing to do with the topic of our book.)}

\newcommand{\hoo}{\raise 1.2pt\hbox{$\stackrel{o}{H}$}}
\newcounter{scholco}
%\newcounter{chipter}
\setcounter{scholco}{0}
\newcounter{apco}
\setcounter{apco}{0}
\newcounter{figurco}
\setcounter{figurco}{0}
\renewcommand{\thescholco}{\Roman{scholco}}
%\def\cchapter#1{
%\ifodd\thepage
%     \newpage
%     \thispagestyle{empty}\null\newpage
%     \setcounter{theorem}{0}\chapter{#1}
%\else
%    \newpage
%     \setcounter{theorem}{0}\chapter{#1}
%\fi}
\newcommand{\cchapter}{\setcounter{theorem}{0}\chapter}
\renewcommand{\thechapter}{\Roman{chapter}}
%\renewcommand{\thechipter}{\Roman{chapter}}
\renewcommand{\thetheorem}{\thechapter.\arabic{theorem}}
%\newcommand{\thessubsection}{\S \Roman{chapter}.\arabic{section}.\arabic{subsection}$\,$}
%\newcommand{\thessection}{\S \Roman{chapter}.\arabic{section}$\,$}
\newcounter{sectco}
\setcounter{sectco}{0}
%\newcommand{\ssection}[1]{\refstepcounter{sectco}\section*{\S \thechapter.\thesectco.\ #1}}
%\newcommand{\ssection}{\chapter}
%\newcommand{\ssubsection}{\section}
%\newcommand{\ssubsubsection}{\subsection}
%\newcommand{\ssubsubsubsection}{\subsubsection}
%\renewcommand{\section}[1]{\section{\thessection #1}}
\newcommand{\scholium}[1]{\refstepcounter{scholco}
\subsubsection*{SCHOLIUM \thescholco.\ \ #1}
\addcontentsline{toc}{section}{{Scholium \thescholco.\
    \ #1}}}
\newcommand{\figurecap}[1]{\refstepcounter{figurco}\bigskip
\begin{center}{\sc Figure \arabic{figurco}}.\ #1
\end{center}\bigskip}
\newcommand{\ssubsectionap}[1]{\refstepcounter{apco}
\subsection*{$\!\!\!\!\!\!\!$A.\arabic{apco}.\ \ #1}
\addcontentsline{toc}{section}{$\qquad$A.\arabic{apco}.\
    \ #1}}

%\section*{This is an unnumbered first-level section head}
%This is an example of an unnumbered first-level heading.

%\specialsection*{This is a Special Section Head}
%This is an example of a special section head%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%\footnote{Here is an example of a footnote. Notice that this footnote
%text is running on so that it can stand as an example of how a footnote
%with separate paragraphs should be written.
%\par
%And here is the beginning of the second paragraph.}%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\def\gua{\hfill H.~J.~SUSSMANN AND J.~C.~WILLEMS}
\def\guo{CURVE MINIMIZATION\hfill}

\pagestyle{myheadings}
\markboth{\gua}{\guo}


\def\restr{\lceil}
\def\pmb#1{\setbox0=\hbox{$#1$}%
  \kern-.025em\copy0\kern-\wd0
  \kern.05em\copy0\kern-\wd0
  \kern-.025em\raise.0144em\box0 
  \kern-.025em\raise.0288em\box0 
  \kern-.025em\raise.0433em\box0 }
%\renewcommand{\thesubsubsection}{\S$\,$\arabic{subsubsection}$\,$}
\def\equaldef{\,{\buildrel  \rm def \over =}\,}
\def\hham{\mathcal{ H}}
\font\mynino=cmr9 scaled\magstephalf
\font\mytenrm=cmr10 scaled\magstephalf
\font\mytensl=cmsl10 scaled\magstephalf
\font\mytenit=cmti10 scaled\magstephalf
\font\forsection=cmbx10 scaled\magstep 0
\font\mynino=cmr9 scaled\magstephalf
\font\mit=cmti9
\font\mysmall=cmr7
\font\mysmallit=cmti7
\font\myverysmall=cmr5
\font\mysmall=cmr7
\font\mynine=cmr9
\font\mysmallit=cmti7
\font\mynineit=cmti9
\font\myninebf=cmbx9
\font\mymagonebf=cmbx10 scaled\magstep 1
\font\mymagtwobf=cmbx10 scaled\magstep 2
\font\mymagthreebf=cmbx10 scaled\magstep 3
\font\mymagfourbf=cmbx10 scaled\magstep 4
\font\mymagfivebf=cmbx10 scaled\magstep 5
\font\mymagonerm=cmr10 scaled\magstep 1
\font\mymagtworm=cmr10 scaled\magstep 2
\font\mymagthreerm=cmr10 scaled\magstep 3
\font\mymagfourrm=cmr10 scaled\magstep 4
\font\mymagfiverm=cmr10 scaled\magstep 5
\font\mymagoneit=cmti10 scaled\magstep 1
\font\mymagtwoit=cmti10 scaled\magstep 2
\font\mymagthreeit=cmti10 scaled\magstep 3
\font\mymagfourit=cmti10 scaled\magstep 4
\font\mymagfiveit=cmti10 scaled\magstep 5
\def\eps{\varepsilon}
\def\smalltyperm{\baselineskip=12.2pt \rm}
\def\smalltypesl{\baselineskip=12.2pt \sl}
\newcommand{\bi}{\begin{itemize}}
\newcommand{\ei}{\end{itemize}}
\newenvironment{Quoterm}{
\bigskip\begin{center}\begin{minipage}{11cm}\smalltyperm\noindent}
{\end{minipage}\end{center}\bigskip}
\newenvironment{Quotesl}{
\bigskip\begin{center}
\begin{minipage}{11cm}\smalltypesl\noindent}{\end{minipage}
\end{center}\bigskip}
\newcommand{\Ft}[1]{\footnote{\mynine\baselineskip 8pt #1}}
\newcommand{\bqrm}{\begin{Quoterm}}
\newcommand{\eqrm}{\end{Quoterm}}
\newcommand{\bqsl}{\begin{Quotesl}}
\newcommand{\eqsl}{\end{Quotesl}}
\parindent 6mm
\newtheorem{ttheorem}{Theorem*}
\newtheorem{etheorem}{Theorem*}
\newtheorem{qtheorem}{Theorem*}
\newtheorem{myquestion}[qtheorem]{Open Problem}
\newenvironment{question}{\begin{myquestion}$\!\!\!$.\ \rm}{\end{myquestion}}
\newcommand{\bques}{\begin{question}}
\newcommand{\eques}{\end{question}}
%\thispagestyle{empty}
\newcommand{\hdleft}{{\rm H. J. Sussmann and J. C. Willems\hfil}}
\newcommand{\hdright}{{\rm \hfil 300 years of optimal control}}
\setcounter{page}{0}


\cchapter{Introduction\label{sec-intro}}

\noindent In the June 1696 issue of the journal 
{\em Acta Eruditorum}, Johann 
\mbox{Bernoulli} challenged mathematicians to 
solve the ``problema novum'' of finding the curve that---following Leibniz'
suggestion---he named the ``brachistochrone''\Ft{Cf. 
Chapter \ref{controbrach} 
below for a precise statement of Bernoulli's problem.}. The journal's
May 1697 issue contains articles on this problem by six 
of the most renowned mathematicians of the time\Ft{Johann Bernoulli, 
Johann's elder brother Jakob, Leibniz, Newton, the marquis de l'H\^{o}pital, 
and Tschirnhaus; cf. \S \ref{subsec-thechal} in Page \pageref{par-sixmaths} 
below.}.   Leibniz, who was one of the six authors,
called  Johann Bernoulli's question  a ``splendid'' problem.
The fact that he and five other thinkers of such unparalleled distinction 
chose to become actively engaged in the search for its solution
shows  that at the time of its formulation the problem
was perceived by the leaders of the 
\mbox{mathematical} community 
as a major challenge, lying at the very frontiers of their research. 

It would not have been easy at that point to predict the future impact 
on science of Johann Bernoulli's challenge. Yet, the six famous authors of the
1697 papers must have had their own reasons\Ft{We will
speculate on these reasons in \S \ref{subsec-specu} (in Page
\pageref{subsec-specu}) below.} to sense that an exceptionally
promising line of inquiry was being opened, and the developments of the 300
years that followed have lent ample support to their intuition.

Today, it is clear that the events of 1696-97 represented a critical
turning point in the history of mathematics, and it is widely\Ft{Though not
unanimously, cf.  {\mit infra}, Chapter \ref{secreasons}, esp.  Remark
\ref{dissent}.} agreed that with them an important new field, later to be
called the ``calculus of variations,'' was born\Ft{The words ``calculus of
variations'' were first used by Euler in a 1760 paper.}. The newborn
subject was destined to become a major area of mathematics. Moreover, its
implications would soon extend to physics, whose very foundations would be
shown in the 18th century---by Euler, Maupertuis, Lagrange, and others---to
depend on principles formulated in the language of the calculus of
variations.

So it is not surprising that the story of the brachistochrone is told in
many books, and that the importance of this curve is stressed 
in most \mbox{histories} of mathematics.  The brachistochrone has 
received honors not commonly 
\mbox{bestowed} on mathematical curves.  For example,
it is \mbox{depicted} in a stained-glass window of the Academy building
(the main venue of the \mbox{university}) in Groningen (cf. page
\pageref{stained-glass}). And it is probably 
the only  \mbox{mathematical} curve 
that has been deemed worthy of a  monument in its honor
(cf. Scholium \ref{monubrach} below, in Page
\pageref{monubrach}).    

With this book we wish to honor, in our own
way, the brachistochrone and the mathematicians who studied it, on the
occasion of the the 300th anniversary of the publication of the two
issues of {\em Acta Eruditorum}.

Our original objective, when we set out to do the thinking and carry
out the research that would eventually lead to this work, was to make
a modest but very personal contribution to the celebration of the
300th \mbox{anniversary} of the events of 1696-97, by paying tribute
to the \mbox{mathematicians} who \mbox{participated} in them and
showing how, in the three centuries that \mbox{followed}, their ideas
have been fruitful to an extent that would have made them proud.  We
wanted to refrain from retelling once more, from a \mbox{slightly}
\mbox{different} \mbox{angle}, a story that has already been told many
times in many \mbox{nearly} 
\mbox{identical} \mbox{versions}.  Rather, we
were determined to take full \mbox{advantage} of our \mbox{specific}
difference within the community of mathematicians, and use our own
\mbox{specialized} mathematical knowledge to look at the story from a
new \mbox{perspective} that, we hoped, might yield truly novel
insights on an old \mbox{subject}. Since for both of us {\em control theory}
is the primary area of 
\mbox{scientific} interest, we found it natural to
choose this field to guide us in our inquiry and provide us with a
theme that would lend distinctness and \mbox{freshness} to our
\mbox{account}. This led us naturally to explore the various ways in
which the 300-year process of discovery that began in 1696-97 is
related to 
\mbox{{\bem optimal control theory}}, the branch of control theory
that deals with curve \mbox{minimization} problems and is most closely
related to the calculus of variations.

\bigbreak
\vbox{
\[
\epsfbox{monument.eps}\label{monument}
\]
\figurecap{\label{monu}H\'ector Sussmann, Jan Willems, and the brachistochrone monument in Groningen}
}

\bigbreak

When we started our investigation we felt that our decision to focus on 
optimal control was justified on the basis of

\bi
\item[$\bullet$] our own experience with optimal control, which had taught 
us that this theory was richer and much more powerful than its 
immediate predecessor the calculus of variations, and had made us confident 
that this superior power would manifest itself if one applied both
theories to an old problem and compared the results;

\medbreak
\item[$\bullet$] the striking coincidence that the word \label{brachmean}
``brachistochrone'' actually means ``shortest time'' (cf. box below),
so that the brachistochrone is the solution of a
{\bem minimum time} problem, i.e., of a genuine {\bem \mbox{optimal}
control} problem in the modern sense of the term;

\medbreak
\item[$\bullet$] the fact that there was at least one example---the
question of the existence of an optimum for the brachistochrone
(cf. \S \ref{subsec-existence} 
in page \pageref{subsec-existence} below)---where it was evident that 
the control approach yielded better results than calculus of
variations methods\Ft{It was  F.~H.~Clarke who brought our attention
the fact that the existence problem for the brachistochrone
was very delicate in the calculus of variations setting.}.
\ei

Although we realized that these  were not sufficient grounds
for making a
sweeping claim about the significance of optimal control---as opposed to
the calculus of variations---for the brachistochrone problem, they seemed 
good enough to us as a point of departure for an inquiry 
that, we hoped, would yield additional evidence strengthening our case.
We were persuaded that we would find good arguments to advocate the view that
{\em the brachistochrone problem is closely related to optimal control
theory for much more substantive reasons 
than the strikingly prescient choice of a word made by Leibniz and Johann
Bernoulli, and the usefulness of optimal control as a technique for proving
an existence result.}   

\medbreak
\begin{center}\label{pagboxbrach}
\fbox{\fbox{
\begin{minipage}{11cm}
{\large
\null\medbreak
\begin{center}
{\bf THE WORD ``BRACHISTOCHRONE''}
\end{center}
\medbreak
``Brachistochrone'' means ``shortest time.'' It is derived 
from the Greek words ``brachistos'' 
($\beta\rho\acute{\alpha}\chi\iota\sigma\tau
o\varsigma$), which means ``shortest,''
and ``chronos'' ($\chi\rho\acute{o}\nu o\varsigma$), which means
``time.'' 

``Brachistos'' is the superlative of 
``brachys'' ($\beta\rho\alpha\chi\acute{\upsilon}\varsigma$),
which means ``short.''   Words  such
as brachylogy, brachycephalic, brachydactylic, brachyuran,
brachypterous, are derived from ``brachys.''

Some authors\Ft{Including ourselves in our earlier paper
\cite{sussjan}!} use the spelling ``brachystochrone,'' 
but we choose the much more common spelling 
``brachistochrone'' to reflect the fact that
the ``y'' (Gr. ``$\upsilon$'') is already changed into 
an ``i'' (Gr. ``$\iota$'') in Greek, when forming the superlative.

It was Leibniz who suggested to Johann Bernoulli that he use
this name for the
``shortest time'' curve that solves his ``splendid'' problem.
$\qquad$}
\end{minipage}}}
\end{center}
\medbreak

Subsequently, our perspective evolved, as we repeatedly came 
\mbox{across} new and even more convincing 
reasons for thinking that our original 
\mbox{decision} to look at the 300-year 
evolution from an optimal control perspective had been 
\mbox{fruitful} \mbox{beyond} our best expectations. 
We realized that not only the 
\mbox{brachistochrone} problem, but also many
other classical questions of the 
\mbox{calculus} of variations, were related
to optimal control in a much more 
\mbox{complex} and interesting way than we
had anticipated at first.  

Moreover, the control point of view threw new light on a number of
episodes in the 300-year history of the calculus of variations.  As we
\mbox{retraced} on our own the zigzagging path that had led from the
\mbox{brachistochrone} to modern optimal control via the calculus of
variations, we recognized that those who had followed it first had
been part of an intellectually 
\mbox{exciting} adventure, full of surprising
twists and interesting cases of 
{\bem missed \mbox{opportunities}}. Significant 
discoveries had been within the reach of giants such as Johann
Bernoulli, Karl Weierstrass, and Constantin Carath\'eodory, but had
failed to be made for reasons that to us, with our turn of the
\mbox{millenium} hindsight and our optimal control perspective, look minor
and perhaps even trivial. It is a story rich in connections with
several \mbox{other} areas of 
\mbox{mathematics}, and full of lessons about the
decisive importance of \mbox{foundational} issues---such as the choice of a
correct formalism, or the \mbox{precise}
\mbox{specification} of the set of
variables on which a function depends---in the 
\mbox{process} of discovery.

We concluded that the march along the path of discovery, as it actually took
place, is related to present-day optimal control control theory
in a great variety of ways:

\bi
\item[$\bullet$] First, there is a simple {\em kinship} relation, 
in the form of a direct line of descent
going from the brachistochrone ancestor to the 
twentieth-century optimal control
offspring, following the calculus of variations path.

\medbreak
\item[$\bullet$] Second, {\em the history throws light on 
the present},  in that the meaning of the 
powerful results of today is much better understood by regarding them as 
the last steps in a long but simple sequence of conceptually natural 
improvements, each one of which corresponds to a point in the 300-year 
path.
	
\medbreak
\item[$\bullet$]  Third, {\em the present throws light on the history.}
Several false steps and  missed \mbox{opportunities} along
the 300-year trek are best understood by looking at 
them from our privileged vantage point, which enables us to see what was 
there all along but was not seen by the \mbox{original}
\mbox{explorers}. 
This is especially true of important ``non-events,'' such as the 
\mbox{explorers'}
failure to make certain discoveries that were clearly 
\mbox{within} their
reach.  There is a perfect analogy with the bizarre 
\mbox{adventure} of
the 17th \mbox{century}
explorer A.~J.~Tasman\Ft{who, as befits our story, was Dutch, 
and in fact a native of the very same province of Groningen
where Johann Bernoulli did his work on the brachistochrone and 
related \mbox{problems}, and came within a whisker of discovering hyperbolic
geometry and the independence of Euclid's parallel postulate}, 
whose achievements---such as the 
\mbox{discovery} of Tasmania, New Zealand, Tonga
and the Fiji islands in 1642---pale in comparison with the much more 
amazing fact that he \mbox{circumnavigated} Australia but failed to discover it
(cf. \mbox{Scholium} \ref{tasremark} in Page \pageref{tasremark}).
The story of Tasman's trip takes on an extra \mbox{dramatic}
\mbox{dimension}
when one follows it while keeping in mind that 
\mbox{Australia} was there 
all \mbox{along}.  In exactly the same way,  the 
work by \mbox{Johann} Bernoulli on the brachistochrone-related optimization
problem that gives rise to  what we would now call the ``Poincar\'e
half-plane,'' the study by
\mbox{Weierstrass} of the ``excess function,''
and the \mbox{reformulation} by Carath\'eodory of Weierstrass's result, 
are seen in a new light when we realize how close these mathematicians 
had been to two \mbox{major} 
discoveries---hyperbolic geometry, the maximum principle of 
\mbox{optimal} control---and try  to \mbox{understand} why they failed to 
notice the presence of these 
\mbox{enormous} ``Australias'' that were plainly there to be found.  

\medbreak
\item[$\bullet$] Fourth, {\em the results of the present enhance the
value of the old problems.} If one ``looks back at the past from a
twentieth-century perspective'' and views the old problems in the
light of the most recent results and techniques, they often yield new
and interesting insights.

\medbreak
\item[$\bullet$]  Fifth, in particular, {\em the brachistochrone problem 
itself acquires new dimensions} when approached from the optimal control
perspective. This perspective  sheds  new light on  
the old problem, and leads to new results that we believe would have pleased
Johann Bernoulli and the other five contributors to the May 1697 {\em Acta
Eruditorum}. We therefore feel justified in claiming, as we will
later on, in Chapter \ref{controbrach},
that {\bem it is only within the optimal control framework 
that one can do full justice to Johann Bernoulli's ``splendid problem.''}
\ei

\medbreak\noindent It became clear to us that there was a lot to the 
story that was not widely known and needed 
to be told, so we decided to tell it, entertainingly if possible, but
supplying enough details to go beyond the purely anecdotal and clarify the
conceptual issues, while at the same trying to reach a wide audience of
mathematicians, pysicists and engineers.  

We now understand that our original decision to tell the story from an
optimal control perspective had been correct for many more reasons
than those of which we had been aware at the beginning of our journey.
The relationship between the 300-year path and modern optimal control
has turned out to be multifaceted, and optimal control has shown
itself to be much more than a twentieth-century derivative by-product
of the calculus of variations. Among other things, modern optimal
control theory becomes a powerful explanatory tool, whose mere
existence---in the Platonic universe of ideas\Ft{Or in a Popperian
``third world'.''}, even before it was discovered by humans---helps us
makes sense of some past events, and renders them more dramatic, by
revealing to us that these events happened {\em because the maximum
principle of optimal control---like Australia---was really there all
the time, waiting to be discovered.}

The links between the brachistochrone and optimal control have proved so
strong, and optimal control has shown such a remarkable ability of to
render meaningful and coherent many facts about the calculus of variations
and its history, that we reached the somewhat radical conclusion 
that the best way to think of the events of 1696-97 was as marking {\em the
birth of optimal control}. In other words: 
\bi
\item[$\bullet$] The formulation and solution of the
brachistochrone problem 
\mbox{was} one of the first steps in a long 
series of results of increasing generality and mathematical 
elegance about the curve minimization problem. 


\medbreak
\item[$\bullet$] With the maximum principle of optimal control theory,
this evolution attains a high point, from which one can 
survey the surroundings and see how the winding  calculus-of-variations
path leading to the top had been all along a part of a meaningful and
coherent landscape.

\medbreak
\item[$\bullet$] It is therefore reasonable to view
the 300-year history of the 
\mbox{calculus} of variations as a process 
of ascent towards the more general theory of optimal control, so that 
the history of optimal control can truly be said to start 300 years ago.
\ei

\gnu{Footnote \ref{foot-iso} must be rewritten.}


\noindent Actually, we eventually reached the conclusion that the
brachistochrone was not the only old curve minimization problem that
had a more natural home in the optimal control theory setting than in
that of the classical calculus of variations. Other old questions,
such as the ones of refraction and reflection of light, the
famous---and ancient---isoperimetric problem\Ft{\label{foot-iso} The
case of the isoperimetric problem is somewhat different from the
others.  The problem does not belong to the category of ``standard
calculus of variations problems,'' but it does belong to a larger
class of calculus of variations problems with constraints, aptly named
``isoperimetric problems.''  So one might surmise that at least in
this case the calculus of variations framework is sufficient to do
full justice to the problem. We will, however, argue that this is not
quite so, but the reasons will be more
subtle. Cf. \S\ref{subsec-isoper} for our discussion of this problem.}
and even the most ancient of all such problems---that of finding the
shortest path joining two points---turn out to lend themselves to a
much smoother treatment by means of an optimal control approach than
by calculus of variations methods\Ft{\label{foot-refra} Although, of
course, they can be solved directly by \textit{ad hoc} methods, as is
well known.  Our point here is not that optimal control theory is
\textit{needed} to solve these problems, but that until the advent of
optimal control theory these problems required special methods and the
main results of the theory were not were not directly applicable to
them. For example, it is an elementary freshman calculus exercise to
derive \snell's law of refraction from Fermat's minimum time
principle, but this problem is \textit{not} covered by the
Euler-Lagrange equation or by the classical ``smooth'' maximum
principle of optimal control.  We feel it is inelegant for a problem
that played such an important role in the birth of the calculus of
variations and optimal control to be left outside the range of
applicability of the main results of these theories, and relegated to
the much less appealing class of ``special problems that can be solved
directly by special {\em ad hoc} methods.''  For another example, the
classical Euler-Lagrange equation is not directly applicable to the
problem of the shortest path joining two points because for that
problem the Lagrangian is not differentiable; one therefore needs an
{\em ad hoc} trick, such as giving a direct geometric argument, or
proving that length minimization is equivalent to energy minimization
and then solving the energy minimization problem instead of the
original one, or even making mathematical mistakes in the analysis, as
in some texbooks. These issues will be explored at length in \S GNU
below.}

\gnu{We must make sure that the issues discussed in Footnote 
\ref{foot-refra} are indeed covered in a later section, and we must
put the number of this section in Footnote \ref{foot-refra}. 
Also; we must add to footnote the 
remark that all these problems are covered by 
HJS's more recent nonsmooth results.}


This had a radicalizing effect on our views, as we concluded
that a good case can be made that optimal control is at least three
thousand years old, and as old as geometry. 


Our collaboration began in
September 1995, when we exchanged ideas on the subject during a conference
in Groningen, in the Netherlands.  At that time, we intended that the
final product of our effort---with our 
\mbox{relatively} modest initial
claims---should appear in print on the occasion of the 300th 
\mbox{anniversary} 
of Johann Bernoulli's 1696 challenge or, in case further
delays were
\mbox{unavoidable}, on the 300th anniversary of the publication of the 1697
articles. And the pursuit of this limited objective met with success, 
for our paper (\cite{sussjan}) appeared on time, in
June 1997. This paper, however, only gave an abrigded version of some parts
of the story---explaining,  for example, why we felt comfortable with the
claim that the events of 1696-97 mark the birth of optimal control theory,
but not presenting the complete argument.  
\mbox{Moreover}, by the time 
our manuscript was completed---and certainly much more so 
by the time the paper appeared---our findings had taken 
us a lot farther, and we were encouraged to go 
far beyond the humble goals of the original project and give 
a more detailed account of our story that would spell out 
our more ambitious claims. 

Thus the writing continued, and the final result is this book, a belated
contribution to what we now feel should really be the celebration of the
300th anniversary of the events of 1696-97. It is clear to us that these
events constituted an important turning point in the history of the old
subject of curve minimization, and can be regarded as either
\bi
\item{} marking the birth of the calculus of variations {\em and optimal
control},  
\ei
or, if one adopts the alternative view that optimal control is in fact much
older,
\bi
\item{} signaling the end of the first period in the history of optimal
control,  which had been 
characterized by the study of special problems solved by ad hoc methods,
and ushering in a new era in which general techniques were 
developed to tackle broad classes of problems.
\ei

As we have already suggested, our retrospective
journey along the path from the brachistochrone to the maximum
principle will be 
\mbox{unabashedly} ``control-centric''\Ft{Readers who hold
views currently in vogue about the social construction of reality and
the impossibility of an unbiased account of anything whatsoever might
suspect that our strong claims about the importance of optimal control
and the brachistochrone are merely reflections of the professional and
nationalistic biases of 
the authors. We gladly plead guilty to most of this charge, except
only for the word ``merely,'' and state for the record that {\em we
are both control theorists, and one of us teaches at Groningen, where
Johann Bernoulli was a professor from 1695 to 1705}. These facts 
almost certainly explain why we became interested in our topic to begin with, 
and why we approcahed it from our particular point of view, but are
of course irrelevant to the merit and validity of our claims.} and
guided by our own turn of the millenium 
\mbox{hindsights}\Ft{In postmodernist parlance, it
will be ``deeply anchored in our own historicity.''}.

In our discussion, the three topics of 
(1)~the brachistochrone 
\mbox{problem}, (2)~\mbox{optimal}
control, and (3)~the 300-year path from the former to the 
\mbox{latter}, 
will be presented in a partly historical, partly logical order, 
\mbox{selected} so as to emphasize the main points outlined above.
In particular, we will wait until Chapter \ref{controbrach}
to embark upon our detailed analysis of the
\mbox{brachistochrone} problem. By then, the reader will have
become acquainted with the main steps in
the 300-year evolution that led from the brachistochrone
\mbox{problem} to
the classical calculus of variations, from there to modern optimal
\mbox{control} and, more recently, to differential-geometric control and the
use of 
\mbox{nonsmooth} analysis techniques. We will then be in a position to
show how these 
\mbox{techniques} yield new insights on the original
300-year old problem of 
\mbox{Johann} Bernoulli.


\gnu{The previous paragraph is incomplete. We need to put in
more stuff 
to have a smooth ending for the chapter!}

\scholium{The brachistochrone monument\label{monubrach} }

In 1994, on the occasion of the 375th anniversary of its foundation, 
the University of Groningen, in the Netherlands, decided to honor 
in various ways the most famous former members  of its faculty.
In the case of 
\mbox{Johann} Bernoulli, who had been a
professor from 1695 to 1705, a decision was made to honor him
by erecting a monument to the brachistochrone, one of 
the most significant discoveries---and one of which he was particularly 
proud---that he made during his Groningen period.  
The unveiling of the brachistochrone monument, located in the  
Zernike complex of  the University, took place in 1996. 
\gnu{EXACT DATE? MONTH, AT LEAST.}
\noindent In September 1995 a conference
was held at the university to commemorate the 300th anniversary
of the appointment of Johann Bernoulli. The 
\mbox{unveiling}
of the monument in 1996 coincided with
the 300th anniversary of the publication of Johann's challenge in
{\em Acta  Eruditorum.}

\scholium{Johann Bernoulli, his family and his time\label{famtime}}

Since Johann Bernoulli is going to be a constant presence 
throughout our narrative, he deserves an early introduction, 
so we will now briefly discuss the person, his family, and
the historical context that  surrounded his life and work.

The Bernoullis were a Protestant family originally from Antwerp in
\mbox{Flanders}.  They fled Antwerp in 1583 to escape religious oppression by
the \mbox{Spanish} rulers and, after spending some time in Frankfurt, finally
settled in Basel, Switzerland, early in the seventeenth century.  In
the Bernoulli family there were eight mathematicians in three
consecutive generations, at least three of whom were outstanding and
became deservedly famous.  Most of them ended up as professors in
Basel, but many spent extensive 
\mbox{periods} in other universities in
Europe.  The most prominent of the Bernoullis were the brothers Jakob
(1654-1705) and Johann (1667-1748), and Johann's son Daniel
(1700-1782), born in Groningen while his father was a professor there.
Jakob Bernoulli made important contributions, in particular to
probability theory (so that, for example, the
``Bernoulli distributions'' are named after him).
Daniel is the discoverer of Bernoulli's law in hydrodynamics, one of
the \mbox{truly} great laws in physics.  ``A majority of the mathematical
members of this family in the second and third generations ... did not
deliberately choose mathematics as a profession but drifted into it in
spite of themselves as a dipsomaniac returns to alcohol.''  (Bell,
\cite{bell}, p.~132). For example, 
\mbox{Johann} started as a doctor of
medicine, and Daniel, ``starting out in law he became professor of
eloquence in Basel before succeding his father in the chair of
mathematics.''  (\cite{bell}, p.~136.)

\vbox{
\[
\epsfbox{blogo.eps}\label{blogopage}
\]
\figurecap{\label{blogo}Bernoulli family logo}
}

At the time that the young Johann Bernoulli came of age, mathematics was
\mbox{undergoing} a revolutionary change. In 1684 Leibniz published his first
article about differential calculus in the {\em Acta Eruditorum}. He
illustrated the power of the {\em Nova Methodus}, as he called 
the new techniques,
through a number of examples for determining maxima and minima in a
much more effective way than had been possible before. Johann and
Jakob Bernoulli were among the first to master Leibniz' {\em Nova
Methodus}, and in 1691 Johann achieved his first success by determining
the catenary, the shape of a hanging chain, by means of the
differential calculus (cf. \S \ref{subsec-catenary}, in page
\pageref{subsec-catenary}). Still in his mid twenties, Johann was
hired by the Marquis de l'H\^{o}pital, one of the leading
mathematicians of his time, to teach him differential calculus. Johann
always claimed that he was the real discoverer of ``L'H\^{o}pital's
rule,'' which appears in L'H\^{o}pital's book. In 1992, documents were
found which brought positive evidence for Johann's claim (cf. Scholium
\ref{schol-lhopi} below).


\medskip

Johann Bernoulli was not an easy person. He often quarreled with
his colleagues,  and had grievances
about  his salary, his
health, and his work. In 1695, shortly after taking up the chair in Groningen,
he complained in a 
\mbox{letter} to Leibniz, who had encouraged him to take the
chair: {\em I have not met any of the practitioners of Algebra, which
you consider present in 
\mbox{Holland}. To the contrary, I have not had the honor to
meet one single
\mbox{person} who would even deserve to be called a ``mediocre
mathematician.''} In the same letter he complained that his teaching took
too much of his time, and that {\em the more progress the students make,
the less progress I make.}  Such 
\mbox{politically} incorrect views were not only
expressed in private letters to his friends.  Johann Bernoulli did not
hesitate to make his views public.  While in 
\mbox{Groningen} he got into serious
difficulties with the local Protestant 
\mbox{theologians} and clergy, who
disapproved of the way new discoveries in the physical sciences
cast doubt on the validity of revealed truth.

\medskip

His disputes with his colleagues were often very open and unrelenting. He
was perhaps the most abrasive contender in the bitter dispute 
between the English,
Newtonian, and the continental, Leibnizian, schools, 
regarding the originality and rigor of the differential calculus. 
He ``was a man of violent likes and dislikes: Leibniz and Euler were his
gods;  Newton he positively hated and greatly underestimated.''
(\cite{bell}, p.~135.)  His rivalry with his
brother Jakob became an embarrassment to the scientific community and when
in 1699 they were both elected to the Paris Academy, it was on the explicit
condition that they promise to cease arguing, a promise that of course was
not kept.  

\gnu{Check date}

\noindent Even more peculiar was Johann's rivalry with his own son Daniel,
whom he criticized for being a Newtonian, 
and of whose success he was very jealous. He once threw 
Daniel out of the house for having won a French Academy of Sciences prize
for which Johannn had also been a candidate, cf. \cite{bell}, p.~134.
Probably with the help of his loyal disciple student Euler
(a student of 
\mbox{Johann} in Basel and a colleague of Daniel in Saint
Petersburg), Johann plagiarized Daniel's path-breaking work on
hydrodynamics (cf. \cite{guillen}). Daniel, 
\mbox{however},
remained dutifully respectful towards his father, although he 
frequently expressed his misgivings to his friend Euler 

Below is a photograph of a glass-stained window of the Academy building
(the main venue of the university) in Groningen. It shows Daniel Bernoulli
sweetly clutching his father's robe, while Johann is holding
a drawing of his brachistochrone. 

\medskip

\gnu{PUT PHOTOGRAPH OF STAINED GLASS HERE.}
 \label{stained-glass} 

\scholium{Tschirnhaus}\label{schol-tschirn}

{\bem Ehrenfried Tschirnhaus} (1651-1708) studied mathematics, philosophy and
medicine at the  University of Leiden, visited various European 
\mbox{countries}, 
met Wallis, Leibniz and Huygens, and did scientific work, 
among other things, on new methods for making porcelain. 
A factory at Meissen 
\mbox{started} production of Tschirnhaus's 
porcelain in 1710,  and the first sales of any consequence of 
porcelain took place at the Leipzig Fair in 1713. 
\mbox{European} governments
competed to obtain his porcelain technique, but Tschirnhaus refused 
to let them have it, and ended up incurring huge debts.  As a 
\mbox{mathematician},
he is most famous for the discovery of the ``Tschirnhaus 
\mbox{transformation},'' 
that can be used to eliminate the term of degree $n-1$ from a polynomial
equation $x^n+a_1x^{n-1}+\cdots+a_{n-1}x+a_n=0$
(by setting $x=y-\frac{a_1}{n}$), and for the detailed
study of the plane curve with equation $3ay^2=x(x-a)^2$, known as 
``Tschirnhaus's cubic'' since being so named by R.~A.~Archibald in 1900.

\scholium{The Marquis de L'H\^opital}\label{schol-lhopi}

{\bem Guillaume $\,$Francois $\,$Antoine $\,$Marquis 
$\,$de $\,$L'H\^opital} $\,$(1661-1704) 
\mbox{became} a mathematician after having to resign as a cavalry
office because of 
\mbox{nearsightedness}. In 1691 he hired 
Johann Bernoulli to teach him the new methods of calculus, which
Johann did with lessons taught in Paris and 
at de l'H\^opital's country house at Oucques, and then later 
by correspondence, when Johann was living in Basel. 
In 1696, de l'H\^opital published his calculus textbook 
\textit{Analyse des infiniment petits pour l'intelligence des lignes
courbes}, in whose preface he stated that he was ``obliged to the 
gentlemen Bernoulli for their many bright ideas, particularly to the
younger Mr. Bernoulli who is now a professor in Groningen.''  The book
contains the famous ``l'H\^opital rule'' for computing the limit of a
quotient when the numerator and the denominator go to zero. After
L'H\^opital's death in 1704, Johann claimed the he was the real author 
of the book, and in particular that L'H\^{o}pital's rule  about limits
was due to him.  He said that he had not disclosed this fact before because
he was bound by his contract with L'H\^opital 
to let the marquis take credit for the discoveries which
Johann may make during this teaching.  His contemporaries tended to
ignore this claim, since Johann was not known to be particularly
generous to others or objective about his own achievements. 
However, in 1922 a copy of the notes made by his nephew Nicolaus Bernoulli
of Johann' course was found in Basel, and this 
brought positive evidence for Johann's claim, since the course turned out
to contain practically all the material found in the book.



\scholium{A. J. Tasman's exploration of the South Pacific\label{tasremark}}

{\em Abel Janszoon Tasman}~(1603-1659), ``the greatest of Dutch
explorers, was  
the discoverer of Tasmania, New Zealand, Tonga and the Fiji Islands''
\mbox{(\cite{brit}, p. 1070)}.

Tasman was born at Lutjegast, in the Dutch province of Groningen, in whose
capital---the city of Groningen---Johann 
Bernoulli would live
\mbox{later} from 1695 to 1705, do his work on the
brachistochrone, and follow in 
\mbox{Tasman's} footsteps by coming close to making, but
failing to make, an earth-shaking discovery for which he had all the tools,
namely, that of a model of non-Euclidean geometry, which would have enabled
him to settle the long-standing question of the independence of Euclid's Fifth
Postulate (cf. \S \ref{subsec-noneuclidean} in Page 
\pageref{subsec-noneuclidean}). 

In August 1642 Tasman sailed from Batavia (now Jakarta, in the
\mbox{island} of Java, in Indonesia) to Mauritius, and then southward
and eastward, \mbox{discovering} Tasmania\Ft{The discovery took place
in late November 1642. The first European to see the island wasate
Tasman, who sighted it from the {\em Heemskerck}, one of the two ships in
his expedition, at 4pm on November 24.  The first European to land in
it was Pieter Jacobsz, a master carpenter who swam ashore from the
Heemskerck and placed the Dutch flag.  The island was named ``Van
Diemen's Land,'' after Anthoony Van Diemen, the Governor-General of
the Dutch East India Company, who had commissioned Tasman's
expedition.  It was visited by French and British explorers, among
them the famous James Cook in 1777, and the infamous William Bligh in
1788 and 1792. (Bligh's 1788 visit, on the HMS Bounty, consisted of a
stop at Adventure Bay on August 2, 1788, to replenish water and
firewood supplies before sailing on September 4 for Tahiti.  The
mutiny occured on April 28, 1789, while the Bounty was headed to the
Caribbean and then back to England, after a long stay in Tahiti.)  Van
Diemen's Land was formally claimed for Britain in 1788. It became a
penal colony for British convicts, dreaded and cursed by prisoners for
its horrendously repressive system (cf. Hughes \cite{hughes}), at a
time when Britain had lost its American colonies and needed a new
outlet for its large number of convicts, among them many political
prisoners, in particular from Ireland. The island was renamed
Tasmania in 1856, as part of an effort to give it a more
appealing image. The convict system was completely abolished
in 1886.}, exploring New Zealand, Tonga and the Fiji Islands, and then
turning northwest, returning to Batavia in June 1643 after sailing
along the coast of New Guinea.

Tasman's life, and in particular his spectacular ten-month voyage, are
fascinating material for a biographer, historian or novelist.  But a
narrative of his life and travels cannot possibly omit mentioning
the amazing fact that {\em he circumnavigated
Australia but failed to discover it}. Australia was not really discovered by
European explorers until about 100 years after Tasman's death.  Before
that date, and even during Tasman's life, there had been some
sightings by Dutch explorers of the Northwest coast of
Australia. Tasman himself explored in 1644 part of the northeast coast
of Australia, from the Gulf of Carpentaria to 22$^o$ S, but it was not
until the late 18th century that the true dimensions of the
island-continent were understood and its systematic exploration
undertaken. The first recorded sighting of a kangaroo by a European
was in 1770.

%\[
%\epsfbox{kangahop.eps}\label{kagaroopage}
%\]
%\figurecap{\label{tasmantrip}A kangaroo}

\[
\epsfbox{tasman1.eps}\label{tasman1page}
\]
\figurecap{\label{tasmantrip}Tasman's 1642 trip}

The Tasman example is quite relevant to the issue of the appropriatenes of
talking about missed opportunities. It is often argued that it is pointless
to pass judgement on the past and discuss events that did not happen but
might, or perhaps should, have happened. We agree that this is so when the
event that did not happen was so remote from the range of realistic
possibilities that it is unimaginable that it might have happened.  We
would not take seriously a biographer who told us that Tasman failed to
discover the planet Pluto, or Maxwell's equations, or the structure of
DNA. But the fact that he missed Australia is clearly a central event in
our perception of the story, to which it adds drama and emotion. It is hard
to reflect on Tasman's journey without feeling that he could and should
have found Australia, and without being tempted to speculate on what would
have happened if he had.

\def\transfu{Another good example of the usefulness of hindsight to throw
light on past scientific research is the history of blood transfusions.
Experiments on transfusions were conducted as far back as the 1660s,
by physicians such as Richard Lower (1631-1691). It was soon noticed 
that in some cases the patient would show a spectacular improvement, while in
others there would be a severe reaction that could even result in death. 
Why this was so remained a mystery until blood types were discovered in
1900.  The mystery is now explained,
and with our twentieth-century hindsight we can make sense of the  
facts observed earlier. In his 1997 novel \textit{An Instance of the
Fingerpost} 
(\cite{pears}), Iain Pears tells the story of a physician 
who tried to help a patient with transfusions in 1663. In the first
transfusion, the patient was given blood from her own daughter, and 
the patient's condition clearly improved. For the second transfusion the
daughter was 
not available, so the physician used his own blood, and the patient got
worse and eventually died. Blood types are not explicitly mentioned, 
but the dramatic effect of the scene when the physician decides to use 
his own blood is certainly enhanced by the fact that {\em the reader is 
aware of a serious danger that the physician knows nothing about}.}


Similarly, in the path that led from the brachistochrone to the 
\mbox{maximum}
principle there were at least two Australias (the already mentioned 
\mbox{maximum}
principle and also, as will be explained in Chapter \ref{secongeo}, the
\mbox{independence} of Euclid's parallel postulate) and at least three 
``Tasmans''---Johann \mbox{Bernoulli}, 
\mbox{Hamilton} and Weierstrass. No account of their
adventures can be truly complete unless it points out their failure to
make momentous discoveries to which they were very 
close\Ft{\transfu}.\proofend

$\qquad$


\ssection{A preview of our main themes\label{sec-preview}}

\noindent We begin with a rough outline of our main themes, which will
be developed in more detail in  subsequent chapters.

\bigskip
\subsection{The setting: calculus of variations vs. optimal control,
and optimal control vs. control}
\label{setting}

As a preliminary to our 300-year journey, and to set the stage for the 
\mbox{discussion} and comparisons that will follow,  we will
describe, in \mbox{Chapter~\ref{secreasons}}, the subject matters of
the calculus of variations and optimal control, 
and \mbox{explain} how the latter
is much more general and naturally leads to the study of 
\mbox{richer} \mbox{mathematical}
structures than the former.  We then digress to point out that {\em
control}, as opposed to {\em optimal control}, is an even larger and
richer area.  The presentation in  Chapter \ref{secreasons} is informal, 
but we provide at the end of the book an appendix with
some truly formal definitions,
for the benefit of those readers who wish to see completely rigorous 
formulations. 

\subsection{Optimal control was born in the years
1696-97, or perhaps even earlier} 
We have already stated that ``brachistochrone'' means ``shortest time'' 
and that, as the name suggests, the brachistochrone is indeed the solution
of a minimum time problem, i.e., of a genuine optimal
control problem.  We will argue that this is not a coincidence, and that 
there are profound reasons why  the brachistochrone problem truly 
belongs to optimal control theory.  These reasons are 
\bi
\item[a.] that the problem involves an interplay between a {\em
nontrivial dynamics} and the minimization of a Lagrangian cost
functional, exactly as in modern optimal control theory (as explained
in Chapter \ref{secreasons}), whereas in the classical calculus of variations
one minimizes functionals over the space of all curves, so no
interesting dynamical constraint is involved other than the one
arising from the minimization, 
%(b)~that some of the most natural and
%interesting generalizations of Bernoulli's problem are to similar questions
%that are completely outside the range of the calculus of variations
%but fit naturally within the optimal control context, 

\medbreak
\item[b.] that the
optimal control approach, applied to the brachistochrone, is superior
to other methods such as that based on the classical calculus of
variations, and thus comes closest to doing full justice---as will be 
shown in Chapter \ref{controbrach}---to  what Leibniz called a 
``splendid'' problem.
\ei

\subsection{The path from the brachistochrone problem to the
\mbox{maximum} principle}
Johann Bernoulli's challenge was the
initial step along a continuous path that led, by successive
incremental improvements, to modern optimal \mbox{control}.  

In Chapter \ref{fromto} we will retrace some of the main steps along this
path, and show how the work on the brachistochrone was %progressively
\mbox{extended} by 
\mbox{Johann} Bernoulli's successors Euler, Lagrange, Legendre,
Hamilton,
\mbox{Weierstrass}, Bolza and Carath\'eodory, who discovered
necessary conditions for a 
\mbox{minimum} that were of increasing strength
and applicable to successively larger classes of curve minimization
problems.  This process led from the study of the brachistochrone to
the Euler-Lagrange \mbox{equations}, and then to 
\mbox{Legendre's} second
order necessary condition, Hamilton's equations, Weierstrass' 
``\mbox{excess}
function'' side conditions, Bolza's analysis of abnormal extremals,
and the \mbox{effort} by  Carath\'eodory to reformulate 
\mbox{Weierstrass'} result in
\mbox{Hamiltonian} form.
Eventually, this resulted---in the 1950s---in the
statement of the 
``\mbox{maximum} principle'' of optimal control theory and
the proof of its first version, stated here in Chapter \ref{themaxprin}.


\subsection{The clarification of the role of the momentum}
As we follow the evolution of the necessary conditions for an optimum,
we will pay particular attention to the role of the ``momentum.''
The classical Euler-Lagrange equations contain the vector
\begin{equation}\label{defmoment}
\pi(t)=\pi_\xi(t)\,\equaldef\,
\frac{\partial L}{\partial\dot x}(\xi(t),\dot\xi(t),t)\,,
\end{equation}
known as the {\em momentum} along the trajectory $\xi(\cdot)$. 
(In modern \mbox{terminology}, $\pi$ is a {\em field of covectors}
along  $\xi(\cdot)$. In control theory, $\pi$ is often 
\mbox{referred} to
as the ``adjoint vector.'')  The momentum also occurs 
in Hamilton's equations and in Weierstrass' condition. As we shall
show, the crucial discovery that Weierstrass and Carath\'eodory
came close to making but missed, was that, once we have the
Weierstrass condition, {\em the momentum no longer  needs
to be defined by (\ref{defmoment})}. One can regard the momentum 
as an ``\mbox{independent}'' 
\mbox{object}, and interpret the necessary conditions
discovered by Euler, 
\mbox{Lagrange} and Weierstrass as asserting the existence
of a field of covectors $\pi$ \mbox{having} certain properties. 
\mbox{Under} the specific technical 
\mbox{assumptions} made by 
\mbox{Weierstrass} throughout 
his work on the calculus of variations, these properties 
\mbox{imply}
in \mbox{particular} that (\ref{defmoment}) holds. But these technical 
assumptions can be \mbox{considerably} 
\mbox{weakened}, and when this is done
the necessary conditions 
\mbox{remain}  valid, as 
\mbox{assertions} about the
existence of a momentum field $\pi$ 
with certain properties, but it no longer follows that $\pi$ is
given by the ``classical momentum condition'' (\ref{defmoment}).


\subsection{The significance of the maximum principle}
The maximum principle was a major achievement that, as we shall
\mbox{explain} in Chapters \ref{fromto} and \ref{themaxprin}, unified,
simplified and clarified a very large part of the 
\mbox{understanding} of
path optimization problems acquired in the three \mbox{preceding}
centuries, by turning a collection of highly technical results with
lengthy and complicated statements into a much simpler and more
\mbox{accessible}
\mbox{formulation}. Moreover, the maximum principle had the
\mbox{additional} 
\mbox{advantage} of applying to a much broader class of problems,
and pointing the way 
\mbox{towards} even newer and more general discoveries.
This makes it a significant turning-point in the march that began in
1696-97, marking the culmination of 300 years of research as well as a
new beginning.  (Some of the work that has arisen as an outgrowth of
the maximum principle, much of which is currently in progress, will be
described in Chapters \ref{frompri}, \ref{nonsmoothcon}, 
\ref{secongeo}, and \ref{beyond}.)

\gnu{JAN: In the ``missed opportunities'' section we have to say
something about Dyson's famous article \cite{dyson}.  I am planning to read
it when I get back from Europe, but if you want you can go ahead and 
read it and write something about it.}

\subsection{Missed opportunities}  Our twentieth-century 
control-centric perspective will enable
us to tell some interesting stories of ``missed opportunities,'' by
identifying points along this path where the actors had all the
knowledge that was needed to make a dramatic discovery and get close
to the maximum principle, but failed to make it for trivial reasons,
such as their having chosen one particular formalism rather than
another that on the surface looked nearly identical, but would have
had profoundly different implications. Specifically, we will argue
that 

\medbreak
\begin{center}
\begin{minipage}{11cm}
{\large{\em The evolution of the calculus of variations reached a
critical fork in the road when Hamilton's equations were written}},
\end{minipage}
\end{center}


\medbreak\noindent  and that 

\medbreak
\begin{center}
\begin{minipage}{11cm}
{\large{\em the way the equations were actually written 
was only one of two possible formulations, and may not have been the
best one}}. 
\end{minipage}
\end{center}

\medbreak\noindent  Precisely, we will show that
\medbreak
\begin{center}
\begin{minipage}{11cm}
{\large {\em If
Hamilton's equations had been written in a slightly
\mbox{different} way---using a Hamiltonian that is a function of all three
sets of variables (positions, velocities and momenta) rather than a
function of the positions and momenta only---then some 
\mbox{important}
discoveries that were not made until the 1950s would almost certainly
have been made much sooner}}. 
\end{minipage}
\end{center}

\medbreak\noindent To make this point, we will explain how a
number of facts from the 
\mbox{classical} calculus of variations, such as the
Euler-Lagrange equations, Legendre's 
\mbox{second} order condition, and the
Weierstrass side conditions, can be seen right away to fit together
into a formulation which is shorter, simpler and more 
\mbox{elegant} than the
conditions stated separately, provided only that the 
``\mbox{correct}''
Hamiltonian formalism is used.  We will then show that 
\mbox{Weierstrass} had
essentially found this formulation, but failed to appreciate what it
\mbox{meant} because he was not using the right Hamiltonian formalism.
The damage done by the use of an inappropriate 
Hamiltonian formalism will become even more evident when
we point out how Carath\'eodory came even 
\mbox{closer}, 
by \mbox{attempting} to write Weierstrass' condition in Hamiltonian form,
but ended up getting a complicated expression that failed to reveal
the simple underlying phenomenon of Hamiltonian maximization. 

Furthermore, we will argue that the ``correct''
(i.e., control-theoretic) 
\mbox{formulation} leads directly to a 
natural conjecture, in the form of 
\mbox{another} statement which is even
simpler, and in addition happens to be much more powerful and general.
We will show that this conjecture actually 
\mbox{turns} out to be essentially
true, and all that is  needed for the statement to 
\mbox{become} \mbox{completely} true 
is one small extra step, namely, the addition of the
``\mbox{abnormal}'' multiplier, whose need had already been understood 
by Bolza 
as far back as 1913. One then gets the full force of the
maximum \mbox{principle} of optimal control theory, which was only discovered
in the 1950s and, as we indicated earlier, is not only {\bem
stronger} but also {\bem simpler} than the combination of its
predecessors.

\subsection{The superior power of the control-theoretic
\mbox{formulation} of Hamilton's equations} Those 
readers who are skeptical about this kind of historical speculation may
choose to regard our ``missed opportunities'' stories as merely 
a \mbox{pedagogical} device,
whose purpose is to show why the ``correct'' (control) 
\mbox{formulation} of
Hamilton's equations is much more powerful than the classical one, even
though both formulations look very similar and are about
\mbox{equally} easy to understand.

This will be demonstrated in Chapters \ref{fromto}, \ref{themaxprin}
and \ref{rela} in three ways:
\bi
\item[1.] In Chapter \ref{fromto} we will give several very simple examples of
minimization problems that at first glance do not appear to lend
themselves to a calculus of variations approach but are easily solved
when Hamilton's equations are reformulated in ``control'' form.
\item[2.] Also in Chapter \ref{fromto}, we will show that the transition from
\mbox{Hamilton's} 
\mbox{equations} to the maximum principle is a very small and
\mbox{completely} \mbox{natural} step, {\em provided that 
the Hamilton equations are written in their
control \mbox{version}}.  We will speculate on the reason for Weierstrass'
Tasman-like failure to discover the simple Hamiltonian 
\mbox{interpretation} of his excess function
\mbox{result} and to find the far-reaching 
\mbox{generalization} that this
\mbox{interpretation} would have suggested. 
\item[] \ \ \ \ Our proposed \mbox{explanation} is as follows:
\begin{itemize}
\item[a.] Weierstrass was working with \mbox{Lagrangians}
that are {\em positively \mbox{homogeneous} of degree one with respect to the
velocities}. 
\item[b.] \mbox{Restricting} oneself to such Lagrangians is something
that can be done without loss of generality, 
because a problem with a more
general Lagrangian is \mbox{always} equivalent---by a change of time
\mbox{parameter}, as will be explained in 
\mbox{\mbox{Chapter} \ref{fromto}---to} one of \mbox{Weierstrass'}
special kind.  
\item[c.]  On the other hand, ``without loss of generality'' is not 
the same as ``without loss of useful insights.'' Actually,
it \mbox{turns} out 
that {\em for the positively \mbox{homogeneous}
\mbox{Lagrangians} used by Weierstrass the reformulation
of the Euler-Lagrange necessary
\mbox{conditions} in \mbox{classical}
\mbox{Hamiltonian} \mbox{form} is {\bem never} possible}, because 
the \mbox{classical} Hamiltonian 
$(q,p,t)\mapsto H(q,p,t)$ can only be defined when, for 
each $(q,t)$,  the map
$$
\dot q\mapsto p=\frac{\partial L}{\partial \dot q}(q,\dot q,t)
$$
defined by the momentum condition can be inverted, so that 
$\dot q$ can be eliminated by expressing
it as a function of $q$, $p$, and $t$.  
\item[d.] Furthermore, the classical Hamiltonian formulation of the
Euler-Lagrange conditions was  the only one known to him. 
\item[e.] It follows that, by choosing to work with his special class
of \mbox{Lagrangians},  and given the limitations of the classical
\mbox{Hamiltonian} formulation, Weierstrass unwittingly placed himself 
precisely in the case where there was no chance that he
would see how his result could be formulated in Hamiltonian form.
\ei
\item[] \ \ \ \ We will show how, if Weierstrass had used
the ``control'' Hamiltonian formulation instead of the classical one,
his side condition would have been seen right away to be no more than
the maximization of the Hamiltonian, and he would 
never have needed to introduce a new entity called the ``excess function.''
\item[] \ \ \ \ Additional supporting evidence for our argument about
\mbox{Weierstrass'} missed 
\mbox{opportunity} will be presented in \S \ref{subsec-carat}, where we 
will show how Carath\'eodory, in his work on the Calculus of
Variations,  came within striking distance 
to discovering the \mbox{Hamiltonian} maximization
\mbox{condition}, but 
failed to find it, because of his insistence on using the classical
Hamiltonian.
\item[3.] In Chapter \ref{rela}, we will look at the proper way 
to derive a \mbox{Hamiltonian} formulation
of relativistic mechanics, a topic that mechanics 
\mbox{books} usually
avoid ``like the plague'' (in the words of H. Goldstein,
the \mbox{author} of one of the leading textbooks on the subject;
cf. his \mbox{statement} quoted in Chapter \ref{rela}), 
or handle superficially and unconvincingly, and often in a noncovariant
way.  We will show that in this case all the 
\mbox{difficulties} disappear if
one uses the control Hamiltonian. This is not surprising, because the
difficulties arise precisely from the very same fact that had stood in
Weierstrass' way before, namely, that the 
\mbox{relativistic} Lagrangians
belong precisely to the class that Weierstrass had used.  
\ei

\subsection{Hamilton's \ value \ function: \ the \ eclipse \ and 
\ \mbox{re-emergence} of
a great idea} Having mentioned Hamilton, we will briefly digress from
our account of the evolution from the Euler-Lagrange equations to the
maximum principle, and discuss an isolated episode whose connection to
the rest of our story will only become apparent later, in
Chapter \ref{valuefun}.  Quoting from a 1944 account by J.~L.~Synge, we will
describe Hamilton's idea of the ``characteristic function'' (or
``principal function'').  According to Synge, the principal function
was ``Hamilton's great central idea,'' but ``Hamilton's plan to
develop all the properties of the extremals from the characteristic
function'' is ``hard to follow,'' so hard in fact that ``few
mathematicians in the ensuing century have given serious
consideration'' to it.  As will be 
pointed out in Chapter~\ref{valuefun}, Synge wrote
this just a few years before
his view of the importance of Hamilton's ``great
central idea'' was completely vindicated. Soon after, the new theory
of {\bem modern optimal control} would be born, and in this theory
Hamilton's function---renamed the ``value funcion''---would 
play precisely the central role for which Synge thought 
it was destined.

This will provide us with a new argument in favor of our control-centered
perspective, because it will show that the control approach reveals
the true meaning of the momentum and makes it possible to carry out 
Hamilton's program as Synge had suggested.

\gnu{On mathematical rigor: we have to check carefully that we really do
all the things promised here in \S\ref{sect-rigor}!}


\subsection{The issue of mathematical rigor}\label{sect-rigor}  The
progression described in Chapter \ref{fromto} will involve successively more
general {\em formulations}, which in all cases will be imprecise and
nonrigorous according to the standards of twentieth-century mathematics.
In Chapters \ref{frompri} and \ref{nonsmoothcon} 
we will explain how these vague statements can be and have been
translated into rigorous theorems.  Our reason for waiting so long to
address this question is that we feel that it is desirable to move as
fast as possible towards the best possible formulation, and 
then---and {\em only then}---take a careful look at it and seek to state
it and prove it rigorously.  Naturally, one could do this at each
intermediate step, and, for example,  discuss in detail
the rigorous proofs
of the Euler-Lagrange equation or the Weierstrass' side condition.  We
will not do so, because all the information that could be derived from
doing it will in any case be contained in the rigorous results arising
from the most general version.

\gnu{Actually, we really should and probably will talk about proofs 
of intermediate results.}

In some cases, however, we will discuss certain issues having to do
with the rigorous justification of intermediate versions. This will be
done to show how the later developments can also throw light on the
true meaning of earlier results, e.g. by showing that these results
can be proved under weaker technical conditions than had originally
been thought necessary. (For example, we will briefly review the
well-known textbook proof of the Euler-Lagrange equation, in order to
show that, as usually presented, it requires much more
differentiability of the Lagrangian than is really needed.  With the
maximum principle, or even with Hamilton's equations in control form,
one can see rather easily that the Lagrangian need not be
differentiable with respect to the velocity.)

By contrast with our formulation of the maximum principle in
\mbox{Chapter~\ref{themaxprin}}, where it will be stated as a {\em principle}
(i.e., not yet a complete theorem, with precise hypotheses and
conclusions), Chapters \ref{frompri}  and \ref{nonsmoothcon} 
will focus on \mbox{various} completely \mbox{rigorous}
{\em versions} of the principle. That is, we will 
\mbox{state} a number of \mbox{rigorous}
mathematical theorems obtained by choosing \mbox{precise} \mbox{technical}
\mbox{hypotheses}, beginning with the first version of the 1962 book 
\cite{pontrybook} by \mbox{Pontryagin}, 
\mbox{Boltyanskii}, Gamkrelidze
and Mischenko---for \mbox{systems} of class $C^1$---and proceeding from there to
the nonsmooth versions---for \mbox{locally}
\mbox{Lipschitz} systems---and to more recent
versions for 
systems with a \mbox{nonlipschitz} and even a discontinuous right-hand side. 

\gnu{Need a box on ``Principles and Versions,'' either here
or in the rigorous chapters.}

\subsection{The geometric connection, and how Johann
Bernoulli missed an opportunity to discover non-Euclidean geometry}
In Chapter \ref{secongeo} we follow the evolution of a 
relationship of central
\mbox{importance} in mathematics, namely, the
multiple connections between the theory of curve
\mbox{minimization}
\mbox{problems} and geometry.  Some of these
go back to many centuries before
1696, as shown by the fact that in ancient civilizations 
it was understood that the straight-line segment is the shortest path 
between two points. Furthermore, their importance and practical usefulness
was noticed, for example, by those who took advantage of this characterization 
of segments by using ``rope-stretching'' in order to build temples 
and altars with perfectly straight lines 
(cf. \S \ref{sect-rope}). It is manifest 
in the study of the isoperimetric problem in classical
Greece, and in \mbox{Johann} Bernoulli's work on the
brachistochrone and related problems, where a non-event of truly
Tasmanian dimensions took place, when Bernoulli studied the light rays
in a plane medium with speed of light proportional to $|y|$, and found
what we would now call the ``Poincar\'e half-plane,'' but failed to
notice that this provides a very simple model for non-Euclidean hyperbolic
geometry.  The linkage shows up again in Lagrange's proof of the
invariance of the Euler-Lagrange equations under arbitrary nonlinear
coordinate changes, which was the first important step in the road to
differential geometry. It then becomes a crucial aspect of the work
of Bernhard  Riemann, who created differential geometry by 
by inverting  the traditional relationship between geometry and curve
minimization, and regarding curve minimization not only 
as a collection of interesting problems to be {\em studied} using
geometry, but as a source that could be used to {\em create new
geometries}.  As a 
preparation for \S \ref{beyond}, which discusses the application of
differential-geometric ideas to control, we include in Chapter \ref{secongeo}
a general discussion of the concepts of covariance and invariance.

\subsection{Differential geometry and control}  In
Chapter \ref{beyond} we move beyond the maximum principle, by showing how the
incorporation of differential-geometric ideas  makes it possible to 
give coordinate-free descriptions of important properties of control systems, 
in a spirit very similar to that of Lagrange's ideas about coordinate 
invariance.    

\gnu{JOB F HJS: Expand and complete this section}

\subsection{Some open problems} In Chapter \ref{threeopen} we bring
the reader to the current frontier of research, by stating three
challenging open problems whose solution, in our view, would result in
significant progress.  Unlike Johann Bernoulli's 1696 challenge, ours
consists of problems that we do not know how to solve and, to our
knowledge, are open.  

\subsection{The brachistochrone} In
Chapter \ref{controbrach} we return to our starting point, and look at Johann
Bernoulli's problem from various perspectives.  We review in
succession (a)~Bernoulli's own rather ingenious solution, based on a
geometrical optics analogy, (b)~the calculus of variations approach,
based on the Euler-Langrange equation, (c)~the optimal control method,
and (d)~the analysis of the problem based on ideas from
differential-geometric control. In each case, we show how the 
new technique adds to the understanding of the problem.

In particular, we will argue in \S \ref{brachprinc} that the brachistochrone
problem, as commonly stated, exhibits some rather peculiar features, and a
proper formulation leads to a problem which is quite singular from the
point of view of general optimal control theory, because it has a
nontrivial constant of motion (the energy) and unbounded controls.  It will
then turn out that the use of differential-geometric analysis is not only
convenient but in fact necessary for analyzing the problem.  We will carry
out this analysis and show how it provides a considerably sharper
understanding of the true underlying geometric structure.  It is 
interesting to note that a very natural way to understand these
peculiarities of the brachistochrone problem is to ask Hamilton's
``central'' question, i.e., to try to find the value function for the
problem. This leads immediately to a four-dimensional state space,
a three-dimensional distribution, the ``diamond Lie algebra,''
and impulse controls, 

We also include three more examples, involving the application of optimal
control to three problems closely related to the brachistochrone.  One of
them is due to M. Hestenes, and can be solved using the classical maximum
principle. The other two require that we use a recent ``nonsmooth'' version
of the maximum principle (Sussmann
\cite{suss1,suss2,suss3,suss4,suss5,suss6,suss7}).  Hestenes' problem is
``an interesting variation of the brachistochrone'' dealing with 
``an idealized rocket ship moving in a vertical plane
(Hestenes \cite{hestenespap}, p. 64).

Hestenes proposed his problem in a effort ``to speculate on what type
of brachistochrone problem the Bernoulli brothers might have
formulated had they lived in modern times.'' In a similar vein, we
propose a second mathematically natural extension of the
brachistochrone problem, namely, the problem of the {\bem reflected
brachistochrone}, in which one looks for the light rays in the whole
$(x,y)$ plane (rather than in a half-plane) with a speed of light
proportional to $\sqrt{|y|}$.

With the third example we travel back in
time even farther, and look at the derivation of {\bem \snell's law of
refraction} from Fermat's minimum time principle.  \snell's law was a
crucial tool in Johann Bernoulli's solution of the brachistochrone problem.
The other two problems are very natural extensions of the original
brachistochrone question, and we have no doubt that Johann Bernoulli, who
was always eagerly looking for new problems to test his methods, would have
found them interesting.   

It turns out that all three problems fall outside the scope of standard
necessary conditions of the calculus of variations.  The Hestenes problem
naturally belongs to ``classical'' optimal control, but the other two do
not, because their right-hand sides are not Lipschitz continuous, and in
the case of \snell{}' law they even fail to be continuous.\Ft{Naturally,
\snell{}' law of refraction can be derived from Fermat's principle by
elementary calculus,  but this is not our point,
as we explained in Foonote \ref{foot-refra}.}
But all three problems are covered by the more recent nonsmooth result.

\subsection{A speculative conclusion}
Based on these examples, we argue that it is very likely that
Johann Bernoulli himself would have appreciated the superiority 
of optimal control methods and would have accepted that {\em optimal control}
is the general technique that he had been after for 
solving curve minimization problems, while the Euler-Lagrange 
equations and other methods of the calculus of variations prior 
to the 1950s were only incomplete and preliminary steps
in the process of discovery of this powerful approach. 


\cchapter{Calculus of variations, optimal control, and 
control\label{secreasons}}

\noindent Our first task will be to present our reasons for questioning the
conventional wisdom that regards optimal control as a twentieth century
newcomer to mathematics and a late offspring of the much older calculus of
variations. 

This will require that we explain in detail how optimal control
differs from the calculus of variations. In particular, we will have 
to give precise definitions of the notions of 
a ``standard calculus of variations problem'' and an
``optimal control problem.''  For this purpose, it will be necessary
to agree on some notational conventions, and to be 
clear about the choices to be made of the ``configuration spaces''
for our problems.

\ssection{Some notational conventions} In  our discussion, it will
be necessary to refer to the work of several authors from the
seventeenth to the nineteenth century.  Every time we do so, we will be
confronted with the question whether it is better to cite
these authors using their own mathematical notations, or to 
stick to a uniform set of notational conventions  of our own.

In $\,$the $\,$interest $\,$of $\,$clarity $\,$and $\,$consistency, 
$\,$we $\,$will $\,$choose $\,$the $\,$latter \mbox{alternative}.
{\bem We will always use our own \mbox{mathematical}
notations and 
terminology rather than those of the various authors under
\mbox{consideration}}. To do otherwise would make our presentation much-
harder to follow, and would render difficult the comparison of the
contributions of different authors, since their notations vary wildly,
and often conflict with twentieth-century standards of mathematical
rigor (e.g. by failing to distinguish between definite and indefinite
integrals). So, for example, 
 
\begin{itemize}
\item{} the letter $L$ will always stand for
the ``Lagrangian,'' 
\item{} the state variables will usually---but not always\Ft{Sometimes
we will use $x$ instead of $q$, or write $q=(x,y)$ or
$q=(x,y,z)$ when $q$ takes values in $\rr^2$ or $\rr^3$.}--be called 
$q$, 
\item{} the independent variable---often called $x$ or $y$ in 
early papers on the subject---will
usually be called $t$, and should be thought of as time,
\item{} we will use dots---and on a few occasions primes, and also 
$\frac{d}{dt}$, when we want to differentiate a long expression---to denote 
differentiation with respect to time (cf.  Equation (\ref{eulagvectoru}) 
below for an example of the use of these notations). Thus,
the derivative of a scalar- or vector-valued
function $\rr\ni t\mapsto\xi(t)\in\rr^n$ will usually be called $\dot\xi$, 
although sometimes we may use $\xi'$ or 
$\frac{d\xi}{dt}$ instead of $\dot\xi$.
\end{itemize}

\medbreak\noindent In addition, we shall adopt the following convention:

{\em 

\begin{itemize}
\item{} {\bem Roman letters} will always refer to the various  
``variables''---such as \mbox{positions}, velocities, momenta, controls, 
and time---related to a \mbox{calculus} of
variations or optimal control problem. {\bem Greek letters} will be used to
denote {\bem functions of time} with values in the spaces of the
corresponding variables.
\end{itemize}
}

\medbreak\noindent Thus, for example, a curve in $q$-space will often
be called $\xi$, and $\dot\xi$ will be its time-derivative. The Lagrangian
will be a function $(q,u,t)\mapsto L(q,u,t)$ of a ``position variable,''
a ``velocity variable'' $u$, and the ``time variable'' $t$. If 
$t\mapsto\xi(t)$ is a curve in $q$-space, then we can plug in
$\xi(t)$ in the $q$-slot and $\dot\xi(t)$ in the $u$-slot, 
thereby obtaining a function $t\mapsto L(\xi(t),\dot\xi(t),t)$ 
of time.  For this reason, calculus of variations books sometimes
use the notation $\dot q$ for the velocity variable, and write
the Lagrangian as a function
$$
(q,\dot q,t)\mapsto L(q,\dot q,t)\,.
$$
We will occasionally
do so as well, but usually we will prefer using the less confusing
name $u$ for the velocity, which makes it clear that 
$u$ is an ``independent variable,'' a fact that may be
less evident to some readers if this variable were to be called
$\dot q$. 

\medbreak 

\ssection{Calculus $\,$of $\,$variations $\,$and $\,$optimal 
$\,$control $\,$problems} The 
calculus of variations deals mainly with optimization problems in which 
one wants to minimize a functional
\begin{equation}\label{thefunctio}
J(\xi)=\int_a^bL(\xi(t),\dot\xi(t),t)\,dt
\end{equation}
of a curve $\xi$.  The integrand of the above expression involves a 
real-valued 
function 
$$
(q,\dot q,t)\mapsto L(q,\dot q,t)
$$ 
of ``position, velocity, and time.''
Nowadays, this function is usually called the {\bem Lagrangian.}  
Optimal control theory is more general, since it considers
situations where the  velocity $\dot q$ is allowed to depend
on position and time, as well as on an extra ``control'' parameter\Ft{The 
letter $u$ is traditionally used to denote the control because 
the Russian word for control is ``upravleniya.''} %GNU Check This
$u$. So in optimal control theory the dynamical law governing
the system's behavior is of the form 
$$
\dot q=f(q,u,t)\,.
$$
It is then natural
to allow the Lagrangian $L$ to depend on $u$ as well. 
\mbox{Moreover}, since
$\dot q$ is determined by $q$, $u$ and $t$, there is no need 
to think of $L$ as a 
\mbox{function} $(q,\dot q,u,t)\mapsto L(q,\dot q,u,t)$,
since one could always rewrite such a function as 
$L(q,f(q,u,t),u,t)$, i.~e., a function of $q,u,t$ only. So the 
Lagrangian in optimal control theory will be a function
$$
(q,u,t)\mapsto L(q,u,t)\,.
$$ 
To make all this more precise, we start by discussing the nature of the
``configuration spaces'' $Q$ where the variable $q$ takes values.

\ssection{Configuration spaces} In
contemporary mathematical writing, it is often customary to represent the 
``position''---or 
\mbox{``configuration,''} or ``state''---of a system, 
at least \mbox{locally}, by a
vector\Ft{We follow the modern convention of using superscripts for
``contravariant indices,'' i.e., components of vectors.}
\begin{equation}\label{vecrep1}
q=(q^1,q^2,\ldots,q^n)
\end{equation}
in $n$-dimensional space $\rr^n$.
The \mbox{velocity} is then also represented by a vector 
\begin{equation}\label{vecrep2}
\dot q=(\dot q^1,\dot q^2,\ldots,\dot q^n)\in\rr^n\,.
\end{equation}
We should think of the members of $\rr^n$ as {\em column vectors},
so that in fact we really should have written
$$
q=\left[\begin{array}{l}
q^1\\q^2\\\vdots\\q^n\end{array}\right]\,,\qquad
\dot q=\left[\begin{array}{l}
\dot q^1\\\dot q^2\\\vdots\\\dot q^n\end{array}\right]\,,
$$
or $q=(q^1,q^2,\ldots,q^n)^{\dagger}$,
$\dot q=(\dot q^1,\dot q^2,\ldots,\dot q^n)^{\dagger}$ 
(where the symbol ``$^\dagger$'' stands for ``transpose'') rather than
(\ref{vecrep1}) and (\ref{vecrep2}).

The {\em momentum} is also a vector, usually represented
as $p$, which should be thought of as a member of the 
{\em dual space} of $\rr^n$, i.e., as a {\em covector},  or
{\em linear \mbox{functional}}
on $\rr^n$. We represent these linear functionals as {\em row 
vectors}, and the 
\mbox{space} of all $n$-dimensional real
row vectors as $\rr_n$, so the momentum is\Ft{Here we
are following standard conventions once again, 
and using subscripts for ``\mbox{covariant} indices,'' i.e., components of 
covectors. ``Covectors'' are dual objects to 
\mbox{``vectors.''}}
$p=(p_1,\ldots,p_n)\in\rr_n$, and the result of applying such
a $p$ as a linear functional on a vector (\ref{vecrep1})
is the ordinary matrix product\Ft{The ``Einstein covention''
prescribes that one should sum over pairs of repeated indices,
one of which is contravariant---i.e., a superscript---
while the other one is covariant---i.e., a subscript. It also 
customary to omit the summation symbol from such sums, and write,
for example, $p_jq^j$ for the summation of (\ref{vecrep3}),
but we will not go that far.}
\begin{equation}\label{vecrep3}
p\cdot q=p_1q^1+p_2q^2+\ldots+p_nq^n=\sum_{j=1}^np_jq^j\,.
\end{equation}
We will also write this product as $p(q)$ (to emphasize
the fact that $p$ is alinear functional), or as an
inner product $\langle p,q\rangle$. 

In our discussion, we will use $Q$ to denote the ``configuration
space,'' that is, the set of all points $q$.  Often, we will simply
think of the points $q\in Q$ as $n$-tuples $q=(q^1,q^2,\ldots,q^n)$,
i.~e., as members of $\rr^n$.  But in all interesting physical or
engineering applications, it would not be accurate to say that a point
$q$ of configuration space really {\bem is} an $n$-tuple of numbers.
The $n$-tuples $(q^1,q^2,\ldots,q^n)$ are only {\em coordinate
representations} that we use to talk about the points, but are not the
points themselves.  

For example, suppose $Q$ is ``physical space.''
Then a point of $Q$ is not {\em really} a triple of numbers, although
it can be {\em represented} as such a triple by choosing a {\em
coordinate system}, i.~e., a bijective map $\kappa:Q\mapsto\rr^3$.
The choice of a coordinate system is not unique, and in general there
is no natural way to single out a canonical $\kappa$. For example, in
classical mechanics\Ft{We are talking here, naturally, of ``classical
mechanics with absolute space,'' in which there is ``absolute rest,''
so physical space-time $S$ is the product $S=Q\times T$ of
three-dimensional ``physical space'' and the one-dimensional time axis
$T$. Classical mechanics really is more complicated, because there is
no absolute rest, so physical space-time $S$ is four-dimensional and
is not endowed with a canonical product decomposition $S=Q\times T$.
There is a natural class of coordinate systems
$\kappa:S\mapsto\rr^3\times\rr$, called {\bem inertial systems}. They
are related to one another by the so-called {\bem Galilean
transformations}, but these transformations do not preserve the
decomposition of $S$ a a product of ``space'' and ``time.''}  one can
single out {\bem linear} coordinate systems (specified by choosing an
origin $O$ of coordinates and three noncoplanar straight lines passing
through $O$, known as the ``coordinate axes''; such systems are
distinguished, for example, by the property that, with respect to
them, the motion of a particle on which no external force is acting
takes place with constant velocity), and {\bem Cartesian} coordinate
systems (that is, linear coordinate systems for which the three
coordinate axes are orthogonal).  But it is also possible to use
other, non-linear, coordinate systems, such as, for example, the well
known spherical and cylindrical coordinates. Such systems often are
not ``valid for the whole space.''  (For example, if $Q$ is a
two-dimensional space, and we represent $Q$ as $\rr^2$ by choosing
Cartesian coordinates, then we can also use {\em polar coordinates},
but the corresponding map $Q\ni(x,y)\to\kappa(x,y)=(r,\theta)$ will be
defined on $Q$ minus a ``cutoff'' $C$---often taken to be the
half-line $C=\{(x,y):y=0,x\le 0\}$---and will map the remaining region
$Q\backslash C$ to the product $]0,+\infty[\times I$, where $I$ is an
open interval of length $2\pi$).

The simplest and most common
examples of configuration spaces are the spaces $\rr^n$
\mbox{themselves}, as well as the line $\mathbb{E}^1$, the plane 
$\mathbb{E}^2$, and the
three-dimensional space $\mathbb{E}^3$ of Euclidean Geometry.  We emphasize
that the spaces
$\mathbb{E}^k$ of Euclidean geometry are {\bem not} the same objects as their
close relatives $\rr^k$. For example, $\rr^3$ has a special point
called ``the origin,'' whereas $\mathbb{E}^3$ does not.  The true mathematical
structure of $\mathbb{E}^3$ is that of an {\bem abstract three-dimensional
Euclidean space}. That is, {\bem 
$\mathbb{E}^3$ is a three-dimensional real
affine space}\Ft{A {\bem real
affine space} is like a real linear space but without a privileged
origin. More precisely, a real affine space is a set $A$ endowed, for
each choice of an ``origin'' $a$ of $A$, with a structure $L_a$ of a
linear space having $a$ as its origin.  The structures $L_a$ have to
satisfy some obvious compatibility conditions. For example, one must 
require that the midpoint relative to the linear structure $L_a$
of the segment joining two points $P_1$ and $P_2$ be independent 
of $a$, that is, that $\frac{1}{2}(P_1+_aP_2)=\frac{1}{2}(P_1+_bP_2)$
for all $a,b\in A$. More generally, the sums
$\alpha_1P_1+_a\ldots+_a\alpha_mP_m$ such that $\sum_{i=1}^m\alpha_k=1$ 
are called {\bem affine combinations}, and one requires 
that all such sums be independent of $a$.} {\bem endowed with a Euclidean 
metric}\Ft{\label{footoeuclid} A {\bem Euclidean metric}
on an affine space $A$ is a way of measuring distances
between points of $A$ (that is, a ``metric'' on $A$, i.e., a map 
$A\times A\ni(a,b)\mapsto d(a,b)\in[0,+\infty[$ that satisfies
$d(a,b)=0\Leftrightarrow a=b$, $d(a,b)=d(b,a)$, 
and \mbox{$d(a,c)\le d(a,b)+d(b,c)$}), that in addition possesses three
properties that relate it to the affine structure of $A$, 
\mbox{namely},
(1)~the identity \mbox{$d(b,r.a+(1-r).b)=rd(b,a)$},
(2)~the ``parallellogram law''
\mbox{$4d(a,\frac{1}{2}b+\frac{1}{2}c)^2+d(b,c)^2=2(d(a,b)^2+d(a,c)^2)$},
and, (3)~the translation invariance \mbox{property}
$d(b+a-a',c+a-a'))=d(b,c)$, which have to be valid
whenever $a,a',b,c\in A$, $r\in\rr$ and $r\ge 0$.}.

{\bem Physical space} $P$ serves as the configuration space for a
point particle. For a long time\Ft{With modern hindsight, one could
perhaps attempt to interpret the Earth-centered physical space of
pre-Copernican science as a {\bem linear} rather than an affine space,
because it had a natural center. The Copernican revolution did not
substantially alter this geometric structure, although it put the Sun
rather the Earth as the center. The truly modern view that the
Universe has no privileged center was expressed, for example, by
Giordano Bruno, who speculated that the universe is infinite, that the
stars we see in the night sky are suns like ours, and that they
probably have planets inhabited by intelligent civilizations. For
espousing these heretical ideas, Bruno was sentenced to die at the
stake by representatives of the best minds of the allegedly
intelligent civilization that inhabits the planet Earth, and was
killed on February 17, 1600. Due to the seriousness of Bruno's
offense, his judges denied him the milder form of execution often used
for other crimes, in which the victim was strangled first so that he
would be spared the extreme suffering of being burnt alive.}  it was
believed that $P$ could be accurately modelled as a three-dimensional
Euclidean space $E$, which could then be identified (in a
non-canonical way) with $\rr^3$ by choosing Cartesian coordinates.
This view has changed with the advent of Einstein's theory of
relativity, although mathematical pioneers such as Gauss had already
envisioned the possibility that the geometry of physical space might
be non-Euclidean.  Here, we will stick to the traditional point
of view of classical mechanics, in which $P$ is regarded
as a three-dimensional Euclidean space.  Similarly, to
study a system of $N$ point particles the natural choice of
configuration space is the product $Q=P^N$ of $N$ copies
of $P$, which can be identified with $\rr^{3N}$ by choosing
Cartesian coordinates on $P$. 

There are, in addition, many  important configuration spaces
that \mbox{cannot} be naturally identified with a space $\rr^n$ or even with 
with an open subset of $\rr^n$. For example, there are many systems
(such as wheels or compass needles) whose natural configuration space consists
of {\em directions} rather than numbers or points of some Euclidean space.
For such systems, the 
\mbox{natural} choice of a configuration space $Q$ is
the unit circle in a two-dimensional 
\mbox{Euclidean} space $E$. 
By choosing Cartesian coordinates on $E$, one can 
\mbox{identify}
this configuration space with $\smath^1$, the unit circle 
in $\rr^2$, given by 
$$
\smath^1=\{(x,y)\in\rr^2:x^2+y^2=1\}\,.
$$
(This identification of $Q$ with $\smath^1$ depends on choosing 
coordinates. In  \mbox{general} $Q$ does not have a natural, 
or distinguished point, whereas $\smath^1$ does. For 
\mbox{example}, 
$\smath^1$ can be regarded as the set of complex numbers $z$
such that \mbox{$|z|=1$}, and complex multiplication 
turns $\smath^1$ into a group, having the \mbox{complex}
\mbox{number}
$z=1$ as identity element.  A better way to view the
\mbox{configuration}
space of a wheel is as a {\em principal space} over $\smath^1$,
cf. Footnote \ref{princispace}.)

A configuration space such as $\smath^1$ is an  example of a 
{\em differentiable \mbox{manifold}} that cannot be represented\Ft{
Precisely: there does not exist a triple $(n,\Omega,\kappa)$ such
that $n\in\nn$, $\Omega$ is an open subset of $\rr^n$, 
and $\kappa:\smath^1\mapsto\Omega$ is a homeomorphism.
The  reason is, simply, that $\smath^1$ is compact whereas
a nonempty open subset of $\rr^n$ cannot be compact unless
$n=0$, in which case $\Omega$ would consist of a single 
point.} as an open subset of $\rr^n$ for any $n$. 


For a second, more sophisticated example of a configuration
space which is naturally a differentiable manifold, 
let us suppose
that we want to study the motion of a rigid body $B$ in 
three-dimensional space. Then  we need 
three coordinates $(x,y,z)$ for the position of the center of mass
$c$ of $B$, and in addition we have to specify the {\bem orientation}
of $B$, by indicating how $B$ has been rotated
with respect to some fixed system $F$ of Cartesian coordinates in space.
This can be done by choosing a Cartesian coordinate system 
$F'$ whose  origin is $c$, that moves rigidly with $B$ (so that,
for instance, the $x$ axis of $F'$ goes from $c$ to some 
fixed point $P_x$ on the boundary of $B$). At each time 
$t$, the $F'$ system will be related to the $F$ system 
by a rotation matrix\Ft{Recall that an $n$-dimensional 
{\bem rotation matrix}, or {\bem orthogonal matrix},
is a real $n\times n$ matrix $R=(r_{ij})_{1\le i,j\le n}$ 
such that $\sum_{j=1}^nr_{ij}r_{kj}=\delta_{ik}$ for 
all $i,k\in\{1,\ldots,n\}$, where $\delta_{ik}=1$ if
$k=i$, and $\delta_{ik}=0$ if $k\neq i$.} $R(t)$. Therefore the 
contribution of the orientation
to the configuration variable $q$ of $B$ is a rotation matrix, i.~e., 
a member $R$ of the three-dimensional orthogonal group
$O(3)$. Hence a natural choice\Ft{\label{princispace}To be precise,
this already involves a previous choice of coordinates
for physical space, as well as the choice of the frame $F$.
To make a truly ``intrinsic'' choice of $Q$ one needs some more
mathematical machinery. Physical space is not really $\rr^3$
but an {\bem abstract three-dimensional Euclidean space} $E$
(cf. Footnote \ref{footoeuclid}). 
And the set $\mathcal{F}$ of ``frames'' should not be 
$O(3)$ itself but a {\bem principal space over 
$O(3)$}, that is, a space $S$ that can be ``canonically identified''
with $O(3)$ after a particular member $F$ of $S$ has been selected,
but which---unlike $O(3)$ itself---does not have a privileged ``origin,'' 
or ``identity element.'' In other words,
$S$ is an object such that, given any two members 
$F_1$, $F_2$ of $S$, there is one and only one rotation matrix 
$R\in O(3)$ such that $F_1\cdot R=F_2$. The mathematically precise notion
is that $S$ is a manifold endowed with a an action 
$S\times O(3)\ni (F,R)\mapsto F\cdot R\in S$ of $O(3)$ which is transitive 
(that is, given any two members $F_1$, $F_2$ of $S$, there is an 
$R\in O(3)$ such that $F_1\cdot R=F_2$) and faithful
(that is, $R$ is unique). So the correct choice of $Q$
is $E\times S$.}  of a configuration space 
for rigid body motion is
$$
Q=\rr^3\times O(3)\,.
$$

\medbreak
\begin{center}
\fbox{\fbox{\begin{minipage}{11cm}
\protect\large 
{\em In our subsequent discussion, we will always deal with 
\mbox{configuration}
spaces that are either (a)~open subsets of $\rr^n$ for some $n$, or 
(b)~differentiable
manifolds. The readers who are not familiar with differentiable
manifolds can limit 
\mbox{themselves} to the case of open subsets 
of $\rr^n$, but those who are used to working  with differentiable
manifolds are advised to pay 
\mbox{attention} to this more general setting,
because the manifold formulation is mathematically more natural 
and elegant.}
\end{minipage}}}\end{center}\medbreak


\begin{remark}{\rm \label{ittook} Considering configuration spaces
of dimensions higher than $3$ is natural in
today's mathematics, but the idea that one might as well write
$(q^1,q^2,\ldots,q^n)$ rather than $x$, or $(x,y)$, or $(x,y,z)$, and
``do everything in $n$-dimensions, with $n$ completely arbitrary,''
was by no means natural to the founders of the subject and took a long
time to evolve. Lagrange, for example, devotes a lot of space in his
{\sl M\'ecanique Analytique} to persuading the reader that a system of
$N$ point particles in $3$-dimensional space can be thought of as a single
point evolving in a space of $3N$ coordinates.}\proofend
\end{remark}

\begin{remark}{\rm \label{ittook-a} One can also consider problems 
in which $q$ is a vector in an infinite-dimensional space, and 
a lot of the current activity deals with such questions. 
However, {\sl in this book we will only
discuss finite-dimensional problems}}.\proofend\end{remark}

\ssection{Spaces of curves}\label{allcurves}  Once we have chosen 
our configuration
space $Q$, the problem of minimizing  the functional $\xi\mapsto J(\xi)$ 
only makes precise sense once we have specified the set 
$\Xi$ of curves $\xi$ in $Q$ within which the minimization is to be carried
out.  

For most interesting problems, the minimization in the space
of all curves, with no restrictions whatsoever, would be devoid of interest.
For  example, the shortest paths in the plane are the single points, 
i.~e., the paths of length zero. To get a truly interesting 
``shortest path'' problem one needs to impose a {\em boundary condition}.
The simplest such condition consists of 
{\em fixing the endpoints} $\bar q$, $\hat q$ of the path,
and looking for the shortest path in $Q$ going from $\bar q$ to $\hat q$.

Moreover, even after we have fixed the endpoints $\bar q$, $\hat q$, it still
does not make sense to work in the space of ``all paths in $Q$ going
from $\bar q$ to $\hat q$.''  Indeed, if we take the word ``path'' to mean
``continuous curve,'' as is often done, then the functional $J$ 
given by (\ref{thefunctio}) will
not have a well defined value for most paths $\xi$, because in general
a continuous path $\xi$ will not have a derivative $\dot\xi(t)$
defined at all values of $t$. So one has to work with a more
restricted class of paths. It then turns out that some of the most obvious
choices are not the right ones.  For example, one may think of taking
$\Xi$ to be the space of all paths $\xi$ in $Q$ going 
from $\bar q$ to $\hat q$ and such that $\dot\xi$ exists everywhere,
or one might limit the paths $\xi$ even further by requiring
in addition that $\dot\xi$ be bounded, or continuous.   These spaces
of paths turn out to be too restrictive, since it is very easy to give
examples of problems for which ``the solution'' 
has points where it is not differentiable. (See, for instance, 
Example \ref{toy2} below, where  the solution has ``corners.'') 

In twentieth century mathematics, it has become clear that one cannot
talk about ``all curves,'' and that one has to work curves in some
suitably chosen function space, selected with great care, such as that
of all absolutely continuous curves, or that of all Lipschitz curves.
Furthermore, it is now understood that the choice of function space
can sometimes be crucial, as can be seen from our discussion of the
Lavrentiev phenomenon in \S\ref{lavrephen} below.

The founders of the calculus of variations did not explicitly
address this question, and usually talked about a minimization
in the class of ``all curves.''  This is understandable, because
the modern mathematical 
\mbox{concept} of a function---which
need not be continuous or even measurable, and {\em a fortiori}
need not be everywhere differentiable, and in  fact might 
even be
{\em nowhere differentiable}\Ft{Weierstrass showed that there
are continuous functions $[0,1]\ni t\mapsto\rr$ that are not
differentiable at any value of $t$.}---was only developed in 
the late nineteenth century. 

Here, we shall adopt the following compromise solution: as long as we
are discussing the formal aspect of the necessary conditions for an optimum,
we will work with an unspecified space $\Xi$ of 
curves, in the classical calculus of variations, or with 
an unspecified set $\mathcal{C}$ of 
trajectory-control pairs, in optimal control. Later, when we 
get to the rigorous statement of the results and their proofs,
we will be careful about the precise choice of $\Xi$ and
$\mathcal{C}$.

\ssection{The conventional wisdom} 
The calculus of variations is
supposed to be approximately three hundred years old, whereas optimal
control theory is said to be a much younger subject, born about four
decades ago in the now extinct Soviet Union, with the work on the
``Pontryagin maximum principle'' by L. S. Pontryagin and his group
(cf. \cite{pontrybook}), or perhaps a few years earlier, with the work of
McShane and Hestenes, cf.  \cite{mcshane,hestenes}.  Moreover, there is a
widespread belief among mathematicians that optimal control was no more
than a minor addition to the classical calculus of variations, essentially
involving the incorporation of inequality constraints.  

\begin{remark}
{\rm The article by L. Markus in \cite{markus} gives a nice account of the
unenthusiastic reaction at the 1958 ICM (International Congress of
\mbox{Mathematicians}) to 
the announcement of the maximum principle by the Soviet
group. Several non-mathematical factors may have contributed to the negative
reaction. Among these, two clearly
stand out: first of all, Pontryagin's personality
and, in particular, his notorious antisemitism, and second, the
impression---undoubtedly enhanced by the obvious overtones of
the word ``control''---that the result was primarily intended for military
applications.} \proofend
\end{remark}

Within the almost unanimous consensus that the calculus of variations was
born approximately three hundred years ago, it is a very broadly 
held view that the birth took place in 1696-97.  For
example, G. A. Bliss wrote in 1925 (cf. \cite{bliss}): 
\bqrm 
It is fair to say that {\mytenit the theory of
the calculus of variations had its beginnings in the interesting
brachistochrone problem of John Bernoulli.}  One should not infer from 
this that no \mbox{problems}
of the calculus of variations were known earlier,
for \ldots Newton had
\mbox{proposed} such a problem and described a
characteristic 
\mbox{property} of its solution.  Furthermore, the brachistochrone
problem itself was definitely in the mind of Galileo in 1630 and 1638
\ldots A \mbox{still} older problem of the calculus of variations is the
isoperimetric \mbox{problem} of the ancient Greeks.  {\mytenit None of these,
however could \mbox{rightly} be 
\mbox{regarded} as the starting point of the theory of
the \mbox{calculus} of \mbox{variations}}, for in the early references 
to them there were no 
\mbox{indications} of other problems of similar type, or of methods of solution
\mbox{possessing} generality of application. [Emphasis ours.]  
\eqrm 
A similar assessment is made by J. Stillwell in his recent textbook
(cf. \cite{stillwell}, p.~174), where it is asserted of 
``James\Ft{that is, Jakob} Bernoulli's solution'' of the 
\mbox{brachistochrone} 
problem---rather than Johann's---that ``it is now considered to be the first
major step in the development of the calculus of variations.''
D.~J.~Struik, in
\cite{struik}, p.~392, says of the articles published in the May 1697
issue of {\em Acta Eruditorum} that ``these papers opened the history of a
new field, the calculus of variations.'' 

\begin{remark}\label{dissent}
{\rm Not all historians of mathematics agree with this view.
For some, it is Newton's earlier (1685)
investigation of the shape of a body with minimal drag 
that marks the birth
the calculus of variations.  For others (for example, M. Kline,
cf. \cite{kline}), this momentous event did not take place until much
later, when Euler, in 1760, officially baptized the newborn by using for
the first time the words ``calculus of variations.''} \proofend
\end{remark}


\ssection{An unconventional alternative}  We will claim that
 
\bigbreak
\begin{center}
\fbox{\fbox{
\begin{minipage}{10.5cm}
{\Large{\em 
\begin{center}
The brachistochrone problem truly belongs 
to optimal control theory, and therefore
optimal control is at least three hundred years old,
and possibly much older.
\end{center}}}
\end{minipage}}}
\end{center}

\bigbreak\noindent This is, of course, a much more controversial thesis
than merely claiming that 1696-97 marks the birth of the calculus of
variations.\Ft{A view closely related to ours is expressed by J. L. Casti,
an author who has written extensively on systems, models, paradigms,
reality, nature, and man. In his remarkable book \cite{casti} on alternate
realities, Casti indeed observes that ``classical optimal control theory
dates back at least to the well-known `brachistochrone' problem.''}  
Since
extraordinary claims must be supported by strong evidence, 
we set out to present our reasons. But first we must give a precise
characterization of the subject matters of the calculus of variations and
optimal control. 


\ssection{Standard calculus of variations problems} The calculus of
variations deals mainly with optimization problems 
\mbox{involving} a
function $(q,\dot q,t)\mapsto L(q,\dot q,t)$ of 
``position, velocity, and time.
Nowadays, it is \mbox{customary} to refer to this function as the {\bem 
\mbox{Lagrangian}.}  
\medbreak\noindent The objective of a typical calculus of variations problem
is to find a curve $t\mapsto\xi(t)$ in $q$-space that minimizes the integral 
$$
I=\int_a^bL(\xi(t),\dot\xi(t),t)dt
$$
of the Lagrangian, among all curves that satisfy some extra 
conditions. As explained in \S\ref{allcurves}, ``all curves''
really means ``all curves in some given space of curves $\Xi$''.

For example, one could specify a time interval
$[a,b]$, an initial point $\bar q$, and a terminal point $\hat q$,
and seek to minimize $I$ in the class of all curves $\xi:[a,b]\to\rr^n$ 
sucht that $\xi(a)=\bar q$ and $\xi(b)=\hat q$. This is a 
``standard calculus of variations problem'':

\medbreak
\begin{equation}\label{standard}
\fbox{\fbox{$\left.\begin{array}{rcl}
{\rm Given\,\,}&\,\,& Q,\,L,\,a,\,b,\,\bar q,\,\hat q,\,\Xi\,,\\
\,\,&\,\,&\,\,\\
{\rm minimize\,\,}&\,\,& I=\int_a^bL(\xi(t),\dot\xi(t),t)dt\,,\\
\,\,&\,\,&\,\,\\
{\rm subject\,\,to\,\,}&\,\,&\xi(a)=\bar q\,,\,\,\,\xi(b)=\hat q\,,\\
\,\,&\,\,&\,\,\\
{\rm and}&\,\,&\xi\in\Xi\,.\end{array}\right.$}}
\end{equation} 
\medbreak\noindent Alternatively, one may prefer to avoid using 
the symbol $\dot q$ as variable, and seek to make it transparently clear 
that $L$ is ``a function of three independent variables''---namely, 
the configuration point $q$, the velocity vector $u$, and time $t$---
by using a letter such as $u$ rather than $\dot q$ as the name
for the velocity variable.  This leads to regarding the Lagrangian
as a function
$$
(q,u,t)\mapsto L(q,u,t)
$$
and rewriting (\ref{standard}) 
in the equivalent form 
\medbreak
\begin{equation}\label{standardu}
\fbox{\fbox{
$\left.\begin{array}{rcl}
{\rm Given\,\,}&\,\,&Q,\, L,\,a,\,b,\,\bar q,\,\hat q,\,\Xi\,,\\\
\,\,&\,\,&\,\,\\
{\rm minimize}&\,\,& I=\int_a^bL(\xi(t),\eta(t),t)dt\,,\\
\,\,&\,\,&\,\,\\
{\rm subject \,\,\,to}&\,\,&\xi(a)=\bar q\,,\,\,\xi(b) =\hat q\,,\\
\,\,&\,\,&\,\,\\
\,\,&\,\,&\dot\xi(t)=\eta(t)\,\,{\rm for}\,\,\,a\le t\le b\,,\\
\,\,&\,\,&\,\,\\
{\rm and}&\,\,&\xi\in\Xi\,.\end{array}\right.$}}
\end{equation}

\medbreak\noindent The vector variable $q$ is the {\em state}, or 
{\em configuration}, of the problem (\ref{standard}) or (\ref{standardu}).
The set $Q$ of all possible values of $q$ is the {\em configuration
space}, or {\em state space}, of the problem. One can
allow $Q$ to be a subset of a Euclidean space
$\rr^n$, or a more general differentiable
manifold, such as a spherical surface.  So we will stipulate that

\bi
\item{} the configuration space $Q$ 
is either $\rr^n$, or an open subset of $\rr^n$, or, more
generally, a smooth manifold;
\item{} if $Q$ is open in $\rr^n$, then the variable $u$---the 
``velocity''---takes values in $\rr^n$, and 
$L$ is a real-valued function on $Q\times\rr^n\times[a,b]$; 
\item{} if $Q$ is a manifold then the values of $u$ are, for 
each $q\in Q$, the members of the tangent space $T_qQ$ of $Q$ at 
$q$---so that in fact the pair $(q,u)$ should be thought of as the name 
of a variable, say $V$, with values in $TQ$, the tangent
bundle of $Q$---and $L$ is a real-valued function on $TQ\times[a,b]$;
\item{} $\Xi$ is the space of all curves $[a,b]\ni t\mapsto
\xi(t)\in Q$ that have some \mbox{extra} regularity property,
such as continuous differentiablity, 
piecewise continuous differentiablity, 
absolute continuity, or Lipschitz continuity.
\ei

%\begin{remark}{\rm \label{funcspace} Naturally, neither (\ref{standard})


\begin{remark}{\rm \label{pointtopoint} A minimization problem of the form
(\ref{standard})---or (\ref{standardu})---is a {\em point-to-point}
calculus of variations problem, 
because the endpoint constraint on the trajectory $\xi(\cdot)$ specifies that
$\xi(\cdot)$ has to start at a given point $\bar q$ and end at another given
point $\hat q$.  One can of course consider, both for the calculus of
variations and for optimal control, more general {\em set-to-set}
problems, in which the endpoint constraints are 
\begin{equation}\label{endpoco}
\xi(a)\in S_1\quad{\rm and}\qquad\xi(b)\in S_2\,,
\end{equation}
where $S_1$, $S_2$ are two given subsets of $Q$. Even more generally,
one can consider ``mixed'' endpoint constraints of the form 
\begin{equation}\label{endpocomixed}
(\xi(a),\xi(b))\in S\,,
\end{equation}
where $S$ is a given subset of $Q\times Q$.  Such more general problems
have been studied since 
the very beginning of the calculus of variations, and give rise to
interesting questions about ``transversality conditions.'' In this book,
however, {\em we will only deal with calculus of variations (and optimal
control) point-to-point problems}, since all the points we wish to make
can already be made in this special situation.  This means that we are
leaving out the whole area of transversality conditions.}\proofend
\label{dynapage}
\end{remark}

\ssection{Isoperimetric problems}\label{sect-in-isop}
In addition to the standard
problems (\ref{standard}) or (\ref{standardu}), the more general class
of ``isoperimetric problems'' was considered from the early days of
the calculus of variations. These are problems of the form

\medbreak
\begin{equation}\label{standardisoper}
\fbox{\fbox{$\left.\begin{array}{rl}
{\rm Given}&Q,\,L^0,\,L^1,\,\ldots\,,\,L^m,\,\ell^1,\,\ldots\,,
\,\ell^m,\,a,\,b,\,\bar q,\,\hat q,\,\Xi\,,\\
\,\,&\,\,\\
{\rm minimize}&\int_a^bL^0(\xi(t),\dot\xi(t),t)\,dt\,,\\
\,\,&\,\,\\
{\rm subject\,\,\,to}&\int_a^bL^1(\xi(t),\dot\xi(t),t)\,dt=\ell^1\,,\\
\,\,&\qquad\qquad
\vdots\\
\,\,&\int_a^bL^m(\xi(t),\dot\xi(t),t)\,dt=\ell^m\,,\\ 
\,\,&\,\,\\
\,\,&\xi(a)=\bar q\,,\,\,\xi(b)=\hat q\,,\\
\,\,&\,\,\\
{\rm and}&\xi\in\Xi\,.\end{array}\right.$}}
\end{equation}

\noindent We will show later---in \S\ref{isoper-are-standard}--- that 
{\em once one makes the transition from the
classical calculus of variations to optimal control, the distinction between
\mbox{``standard''} and ``isoperimetric'' problems disappears}, because the
addition of extra integral constraints to a ``standard optimal control
problem'' $P$ just leads to another ``standard optimal control problem''
$P'$.\Ft{Although {\mit P'} will in general not be a ``standard'' calculus of
variations problem, even if {\mit P} is.} 
\label{pageispercheck}

\ssection{Optimal control problems} The distinctive feature of
classical calculus
of variations problems is that the 
\mbox{minimization} of (\ref{standard}) or
(\ref{standardu}) takes place in the space of ``all
curves'' (but cf. \S\ref{allcurves}), so nothing interesting happens
on the level of the set of curves under consideration, and all the
nontrivial features of the problem arise because of the Lagrangian $L$.
In other words, for calculus of variations problems the 
{\bem kinematics} (that is, the specification of which motions
in $q$-space are permitted, or ``feasible'') is trivial
and uninteresting, amounting to no more than the 
statement that one can move in all possible directions
at all possible speeds, subject only to some regularity 
conditions (such as differentiablity) of the resulting 
path.

{\em Optimal control problems}, by contrast, involve a minimization over a set
$\mathcal{ C}$ of curves which is itself determined by {\bem dynamical
constraints} of a particular kind. Precisely,
we assume that an ``agent,'' or ``controller,'' can choose
freely the value of a parameter $u$ belonging to a ``control set''
$U$. This choice can actually be made separately at each time,
so the controller is in fact selecting a {\em control
function} $I\ni t\mapsto\eta(t)\in U$ on some time interval $I$.
We also assume that the control affects the evolution 
of $q$ in such a way that $q$ obeys a differential equation
\begin{equation}\label{dyna} 
\fbox{$\dot q=f(q,u,t)$}
\end{equation} 
Then $\mathcal{ C}$ would consist of all curves 
$t\mapsto{}\xi(t)$ that satisfy the differential equation 
\begin{equation}\label{dyna-b} 
\dot\xi(t)=f(\xi(t),\eta(t),t)
\end{equation} 
for some choice of the control function $t\mapsto\eta(t)\in U$.  

Even more precisely: it may happen that a member $\xi$ of $\mathcal{C}$
does not uniquely determine the control $\eta$ for which
(\ref{dyna}) holds (cf. Example 
\ref{trivial} below). It is therefore better to take $\mathcal{C}$ 
as a set of {\em trajectory-control pairs} $(\xi(\cdot),\eta(\cdot))$.

\label{pageprecise}
An ordinary differential equation such as (\ref{dyna}) defines
a {\em control system}, or {\em controlled dynamics},
or {\em controlled dynamical system}\Ft{
\label{footprecise} Precise definitions of
the basic control concepts, and a discussion of the issue of the 
adequate choice of the set $\mathcal{C}$ of 
``admissible trajectory-control pairs,'' will be given in the appendix.}.

As will be explained below, controlled dynamical systems of the form
(\ref{dyna}) are interesting 
objects to study even in the absence of a minimization problem.
Thus {\bem control theory is a much richer subject than optimal
control}, which deals with only one of many kinds of important control
problems.     

A standard ``point-to-point''  optimal control problem has the form

\begin{equation}\label{standardcontrol}
\fbox{\fbox{$
\left.\begin{array}{lcl}
{\rm Given}&\,\,\,&Q,\,U,\,f,\,L,\,a,\,b,\,\bar q,\,\hat q,\,\mathcal{C}\,,\\
\,&\,\,\,&\,\\
{\rm minimize}&\,\,\,&I=\int_a^bL(\xi(t),\eta(t),t)dt\,,\\
\,&\,\,\,&\,\\
{\rm subject \,\,\,to}&\,\,\,\,&\xi(a)=\bar q\,,\,\,\xi(b) =\hat q\,,\\
\,&\,\,\,&\,\\
\,\,\,&\,\,\,\,&(\xi,\eta)\in\mathcal{C}\,,\\
\,&\,\,\,&\,\\
{\rm and}&\,\,\,\,&\dot\xi(t)=f(\xi(t),\eta(t),t)\,\,{\rm for}\,\,\,
a\le t\le b\,.
\end{array}\right.$}}
\end{equation}

One can also consider ``variable time'' problems, in which 
$a$ and $b$ are not given, and finding them is part of the problem.
As for the classical calculus of variations, 	the endpoint conditions
can also be more general. All these possibilities are subsumed in
the following general form of an optimal control problem:

\begin{equation}\label{standardcontrol-general}
\fbox{\fbox{$
\left.\begin{array}{lcl}
{\rm Given}&\,\,\,&Q,\,U,\,f,\,L,\,S\,,\,\mathcal{C}\,,\\
\,&\,\,\,&\,\\
{\rm minimize}&\,\,\,&I=\int_a^bL(\xi(t),\eta(t),t)dt\,,\\
\,&\,\,\,&\,\\
{\rm subject \,\,\,to}&\,\,\,\,&(\xi(a),a,\xi(b),b)\in S\,,\\
\,&\,\,\,&\,\\
\,\,\,&\,\,\,\,&(\xi,\eta)\in\mathcal{C}\,,\\
\,&\,\,\,&\,\\
{\rm and}&\,\,\,\,&\dot\xi(t)=f(\xi(t),\eta(t),t)\,\,{\rm for}\,\,\,
a\le t\le b\,.
\end{array}\right.$}}
\end{equation}
Here $S$ is a subset of $Q\times\rr\times Q\times\rr$.

\begin{example}\label{ex-unicycle}{\rm Consider the following 
controlled dynamics in $\rr^3$:
\begin{eqnarray}
\dot x_1&=&u_1\cdot\cos{\theta}\,,\nonumber\\
\dot x_2&=&u_1\cdot\sin{\theta}\,,\qquad|u_1|\le 1\,,\,\,|u_2|\le 1
\label{eq-unicycle}\\
\dot\theta\phantom{_2}&=&u_2\nonumber\,\,.
\end{eqnarray}
This represents a ``vehicle''  such as a {\bem unicycle}. The vehicle 
has a {\em \mbox{position}} $(x_1,x_2)$ 
in the plane $\rr^2$ (for example, the position of its center of
mass) as well as an {\em orientation} $\theta$, which 
is an angle\Ft{Which means that, if we want to be 
completely accurate, we should really take 
the configuration space $Q$ to be $\rr^2\times\mathbb{S}^1$ rather
that $\rr^3$.}, thought of as the 
\mbox{direction}
\mbox{towards} which the vehicle is
pointing. The first two equations of the system
(\ref{eq-unicycle}) express
the fact that the vehicle moves in the direction of the vector
\mbox{${\bf v}=(\cos{\theta},\sin{\theta})$}, with a speed $u_1$  
that can be selected by the driver. The third equation captures 
the fact that the driver can also steer the  vehicle by 
changing its direction. The variables $u_1$ and $u_2$ 
represent the two controls available to the driver, namely,
``speed'' and ``steering.''  

\bigskip
\vbox{
\[
\epsfbox{thecoordinates.eps}
\]
\figurecap{The unicycle\label{thecoor-fig}}
}

The dynamical behavior described by (\ref{eq-unicycle})
is already an object worth studying even before
we associate to it an optimal control problem by asking to optimize
something. For example, we could ask the following
interesting questions.

\medbreak\noindent{\em The complete controllability question}: 
Given any two vehicle configurations 
$\bar q=(\bar x_1,\bar x_2,\bar\theta)$, 
$\hat q=(\hat x_1,\hat x_2,\hat\theta)$, can the driver
always steer the vehicle from $\bar q$ to $\hat q$?

\medbreak\noindent{\em The parking question}: 
Can the driver make the vehicle move sideways,
that is, go from an initial configuration 
$\bar q=(\bar x_1,\bar x_2,\bar\theta)$ to a terminal
configuration $\hat q=(\hat x_2,\hat x_2,\bar\theta)$
such that $(\hat x_1,\hat x_2)=(\bar x_1,\bar x_2)+R(-\sin{\bar\theta},
\cos{\bar\theta})$ for some nonzero number $R$?

\medbreak\noindent{\em The effective complete controllability question}: 
Assuming that the answer to the complete controllability question 
is ``yes,'' can we produce an effective algorithm enabling
the driver to compute, for any given $\bar q$, 
$\hat q$, a control strategy $t\mapsto (u(t),v(t))$ 
that will drive the vehicle from $\bar q$ to $\hat q$?

\medbreak\noindent{\em The effective parking question}: 
How can the driver effectively compute 
a strategy for making the vehicle move sideways?

\bigskip
\vbox{
\[
\epsfbox{sideways.eps}
\]
\figurecap{Moving sideways\label{sideways-fig}}}


\medbreak\noindent In this case, it turns out that the
answers to the above questions are all very simple. For example,
a solution to the effective parking question is as follows: 
to move sideways to the left a distance ${\bf t}$, from a given
initial configuration 1, move to a configuration 2
by making a $90^o$ turn to the left
(that is, using $u=0$, $v=1$ during $\frac{\pi}{2}$ units of time),
then move forward ${\bf t}$ units of length 
at full speed to configuration 3 (that is, use
$u=1$, $v=0$) during ${\bf t}$ units of time),
and then reorient the vehicle into the desired configuration
4 by making a $90^o$ turn to the right.
Once the effective parking question is solved, the effective complete
controllability question becomes trivial, because
the dynamical equations (\ref{eq-unicycle}) already 
tell us how to move in two of the three linearly 
independent directions that exist at each point of $\rr^3$
(namely, forward and rotating), and motion in the third
``sideways'' direction is achieved by our solution 
of the parking problem. 

}\end{example}

\begin{example}\label{ex-unicycle-for}{\rm Consider the following 
controlled dynamics in $\rr^3$:
\begin{eqnarray}
\dot x_1&=&\cos{\theta}\,,\nonumber\\
\dot x_2&=&\sin{\theta}\,,\qquad|u|\le 1\,,
\label{eq-unicycle-for}\\
\dot\theta\phantom{_2}&=&u\nonumber\,\,.
\end{eqnarray}
This represents a vehicle similar to that 
of the preceding example, except that now
the driver is only allowed to steer by changing the orientation, 
the position $(x_1,x_2)$ only varies by forward motion
at a fixed speed. 

The four questions of Example \ref{ex-unicycle} can still
be asked, and the answers are still positive, even though
now it is much harder to control the vehicle, because backwards
motion is not available, and the vehicle is not even 
allowed to ``stay in place while turning,'' as in the first
and third steps of Figure \ref{sideways-fig}.
}\end{example}

\begin{example}\label{opt-vehicle}{\rm For the 
control systems described in Examples \ref{ex-unicycle}
and \ref{ex-unicycle-for}, we could consider the {\bem minimum time
problem}: given two vehicle configurations 
$\bar q=(\bar x_1,\bar x_2,\bar\theta)$, 
$\hat q=(\hat x_1,\hat x_2,\hat\theta)$, 
steer the vehicle from $\bar q$ to $\hat q$ in minimum time.

These are optimal control problems of the form
(\ref{standardcontrol-general}). The dynamical law $f$ is given
by the right-hand sides of the equations of
(\ref{eq-unicycle}) and (\ref{eq-unicycle-for}). The Lagrangian
$L$ is just identically equal to $1$. The set 
$S$ is equal, for given $\bar q$ and $\hat q$, to
$\{\bar q\}\times\rr\times\{\hat q\}\times\rr$, expressing the 
fact that the initial time $a$ and the terminal time $b$ are not
prescribed. (One could also fix the initial time 
to be $0$, and take $S$ to be 
$\{\bar q\}\times\{0\}\times\{\hat q\}\times\rr$.)}
\end{example}

\begin{remark}\label{remdubins}
{\rm The minimum time problem for the system (\ref{eq-unicycle-for}), 
in which the vehicle can only move forwards, is known
as {\bem Dubins' problem}. It was solved by L.~Dubins 
in his 1957 paper \cite{dubins}. The problem for the system 
(\ref{eq-unicycle}), 
in which the vehicle can move forwards and backwards, is known
as the {\bem Reeds-Shepp  problem}. It was solved by J.A.~Reeds
and L.A.~Shepp in their 1990 paper \cite{reedsshepp}.}
\end{remark}


\medbreak
\noindent {\em The purpose of the following two utterly
trivial examples is to illustrate the fact that giving 
a curve $t\mapsto\xi(t)$ in $q$-space that arises from a control
does not suffice to determine the control. It is for this reason
that we have to talk about ``trajectory-control pairs'' 
rather than ``trajectories.''}

\begin{example}\label{trivial} 
{\rm Consider a control system $\Sigma$ defined
by the dynamical \mbox{equation}
\begin{equation}\label{dyna-trivial}
\dot q=uq\,,
\end{equation}
with $q\in\rr$, $u\in\rr$. Then the trajectory $t\mapsto\xi_*(t)$ given by
$$
\xi_*(t)\equiv 0
$$
does not arise from a unique control function.  Indeed, 
$[a,b]\ni t\to\eta(t)\in\rr$ is any function, then $\xi_*$ satisfies
$\dot\xi_*(t)=\eta(t)\xi_*(t)$ for every $t$, so $\xi_*$ is a trajectory of 
$\Sigma$ for the control $\eta$. 
} \proofend
\end{example}
 
\begin{example}\label{trivial-opt} 
{\rm Consider a one-dimensional optimal control problem whose 
dynamical \mbox{behavior} is defined by the same \mbox{equation} 
(\ref{dyna-trivial}) 
as in Example \ref{trivial}. Suppose we wish 
to minimize the integral cost $\int_0^1\eta(t)^2\,dt$,
subject to the endpoint constaints $\xi(0)=\xi(1)=0$
and the dynamical constaint
$\dot\xi(t)=\eta(t)$ for all $t\in[0,1]$. Clearly,
the minimum value of the integral is $0$,
which is achieved by taking $\eta(t)\equiv 0$. 
So the solution of our problem is the trajectory-control
pair $(0,0)$ (that is, $\xi(t)\equiv 0$ and 
$\eta(t)\equiv 0$). It would not be correct, however, to say 
that ``the solution of our problem is the trajectory $\xi$ 
given by $\xi(t)\equiv 0$,'' because this curve 
is also a trajectory for any other control function
$\eta$, and only the choice of 
$\eta(t)\equiv 0$ minimizes the cost.
} \proofend
\end{example}
 
\ssection{Isoperimetric calculus of variations problems
are standard optimal control problems}\label{isoper-are-standard}  

Exactly as we did in \S\ref{sect-in-isop}
for classical calculus of variations problems, 
we could consider enlarging the class of optimal control problems
by adding to it the ``isoperimetric optimal control problems.'' That is,
given a standard optimal control problem (\ref{standardcontrol-general})
and a collection  of $m$ 
\mbox{additional} integral constraints
$I^1=0,\,\ldots\,,I^m=0$, where 
\begin{equation}\label{isocon2-control}
I^j=\int_a^bL^j(\xi(t),\eta(t),t)\,dt\,,
\end{equation}
we could consider the corresponding 
``isoperimetric problem'':

\begin{equation}\label{standardcontroliso}
\!\!\!\fbox{\fbox{$\left.\begin{array}{lcl}
{\rm Given}&\,&Q,\,U,\,
f,\,L^0,\,L^1,\,\ldots\,,\,L^m,\,\ell^1,\,\ldots\,,\,\ell^m,
S,\,\mathcal{C}\,,\\
\,&\,\,\,&\,\\
{\rm minimize}&\,&I=\int_a^bL^0(\xi(t),\eta(t),t)\,dt\,,\\
\,\,&\,\,&\,\,\cr
{\rm subject \,\,\,to}&\,\,\,\,&(\xi(a),a,\xi(b),b)\in S\,,\\
\,&\,\,\,&\,\\
\,\,\,&\,\,\,\,&(\xi,\eta)\in\mathcal{C}\,,\\
\,&\,\,\,&\,\\
\,\,&\,&\dot\xi(t)=f(\xi(t),\eta(t),t)\,\,{\rm for}\,\,\,a\le t\le b\,,\\
\,\,&\,&\,\,\\
{\rm and}&\,&\int_a^bL^j(\xi(t),\eta(t),t)\,dt=
\ell^j\,,\,\,{\rm for}\,\,j=1,\ldots,m\,.\end{array}\right.$}}
\end{equation}

\bigbreak\noindent It turns out, however, that there is no need
to introduce ``isoperimetric optimal control problems'' as a class
separate from that of ``standard optimal control problems.''  In fact,
{\em ``isoperimetric optimal control problems'' are 
again standard optimal control problems.}

To see this, suppose we are given a problem (\ref{standardcontroliso}).
We can then augment the state space $Q$ of (\ref{standardcontrol}) 
by adding a new $m$-vector $r=(r^1,\ldots,r^m)$
of state variables, subject to the dynamical laws
\begin{equation}
\dot r^j=L^j(q,u,t)\,,\,\,\,\,\,j=1,\ldots,m\,.
\end{equation}
If we let
\begin{equation}
Q^*=Q\times\rr^m\,,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,
L^*(q,r,u,t)=L^0(q,u,t)\,,
\end{equation}
\begin{equation}
q^*=\left[\begin{array}{l}
q\cr r^1\\
\vdots\cr r^m
\end{array}\right],\,\,\,
f^*(q^*,u,t)=\left[\begin{array}{l}
f(q,u,t)\\
L^1(q,u,t)\\
\vdots\\
L^m(q,u,t)
\end{array}\right]\,,
\end{equation}
and 
\begin{equation}
S^*=\left\{\begin{array}{l}
\left(\begin{array}{l}
\left[\begin{array}{l}\bar q\\\bar r\end{array}\right],\,
a\,,\left[\begin{array}{l}\hat q\\\hat r\end{array}\right],b\end{array}\right):
(\bar q,a,\hat q,b)\in S,\,\bar r=0,\,\hat r=
\left[\begin{array}{l}\ell^1\\\vdots\\\ell^m\end{array}\right]\end{array}
\right\}\,,
\end{equation}
then we see that (\ref{standardcontroliso}) is equivalent to 
a standard optimal control problem (\ref{standardcontrol-general}),
with $Q$, $q$, $f$, $L$, $S$, replaced by their starred versions.   

In particular, 

\medbreak

\begin{center}
\fbox{\fbox{{
\begin{minipage}{10cm}
{\large{\em Isoperimetric problems arising from standard  classical 
calculus of variations or from standard optimal 
\mbox{control} problems
are standard optimal control problems. On the other hand,
isoperimetric problems arising from \mbox{standard}  classical 
calculus of variations  problems need not be 
standard  classical calculus of variations \mbox{problems}.}}
\end{minipage}}}}
\end{center}


\ssection{Optimal control is much more general and richer than the
calculus of variations} An optimal control problem has at least
{\bem two} ingredients that give the 
\mbox{situation} interesting
structure, namely, 
\bi
\item{} the {\bem controlled dynamics} $f$,
\item{} the {\bem Lagrangian} $L$, which gives rise to the
{\bem cost functional} $I$ to be minimized.
\ei

This makes it richer than the calculus of variations, where  the dynamics
$f$ is trivial---since $f(q,u,t)=u$---and the only source 
of interesting structure is the Lagrangian $L$.


In particular, optimal control theory contains, at the opposite extreme
from the classical
calculus of variations, problems where the ``Lagrangian'' $L$ is
\mbox{identically}
equal to $1$, i.e., completely trivial, and all the interesting
action occurs because of the dynamics $f$.  Such problems, in which it is
\mbox{desired} to minimize time---i.e., the integral $I$ of
(\ref{standardcontrol-general}) with
$L\equiv 1$---are called {\it minimum-time problems}.  It
is in these problems that the difference between optimal control and the
\mbox{calculus} of variations is most clearly seen, and it is no accident that
these were the problems that propelled the development of optimal control
in the early 1960s, and that time-optimal control is prominently
represented in today's research and in modern optimal control textbooks.

\bigskip
\vbox{
\[
\epsfbox{gor.eps}
\]
\figurecap{\label{gor-fig}}
}


\ssection{To a controlled dynamics there is associated an interesting
Lie-algebra structure} More recently, it has become apparent that, as
R. Hermann (cf. \cite{hermann}) and others (cf. Brockett
\cite{brockett}, Hermes \cite{hermes}, Krener \cite{krenerchow}, Lobry
\cite{lobry}, Jurdjevic-Sussmann \cite{sussju,jususs}) 
had argued since the 1960s and early 70s, a controlled
dynamics such as (\ref{dyna}) is, really, a {\em family of vector
fields}, and such an object has to be studied using the machinery of
differential geometry, which allows us to associate an {\em algebraic
structure} to a family of vector fields, by looking at their Lie
brackets.  {\em This rich and interesting algebraic structure,
arising from a controlled dynamics even in the absence of an
optimization problem, is completely absent from the classical calculus
of variations.}

\ssection{Control is much more general than optimal control} 
In modern terms, one can think of a controlled dynamical system 
(\ref{dyna}) as defining a 
generalization of the concept of a ``continuous-time dynamical system,'' in
which the future evolution of the system state $q$ depends not only on the
current state but also on the action of a ``controlling agent,'' or
``controller,'' who chooses the control function $t\mapsto\eta(t)$.  One
might think that a system defined by (\ref{dyna}) is no more than a
``dynamical system with parameters,'' i.e., an object not significantly
different from an ordinary dynamical system, in the sense that, once the 
function $t\mapsto\eta(t)$ is chosen (\ref{dyna}) reduces to an ordinary 
dynamical system. This, however, would miss the
profound conceptual difference that exists between the study of
dynamical systems and the kind of problems that interest control theorists.

For example, control theorists want to understand 
the natural questions of {\bem reachability}
and {\bem controllability}: for what pairs $q_1$, $q_2$ is it possible to
choose $t\mapsto u(t)$ so as to transfer the state $q$ of (\ref{dyna}) from
$q_1$ to $q_2$? For what systems $f$ can this be done for all pairs
$q_1,q_2$? Given a starting point $\hat q$, what does the set of all
$q$ such that $\hat q$ can be transferred to $q$ look like?

And, most importantly, they are 
primarily interested in {\bem design questions}.

\ssection{Design questions} \label{subsec-design}

{\em Design problems} are problems 
in which the objective is to ``choose the control law $u$''
so as to satisfy some requirements, or ``specifications.''

One possible design question is that of choosing the
function $t\mapsto\eta(t)$ so as to minimize an integral $I$ as before, and
the study of this particular question is what {\bem optimal control theory} 
is about.  But this is just one example of a large number of 
possible design questions that are important for the engineer.
Usually, more information is available at each time $t$ than 
just the value of $t$, and one wants to {\em feed}
all this information {\em back} into the control, so that 
the control law will in fact take the form 
$u=u(t,\iota)$, where $\iota$ represents the other information
available at time $t$.


\ssection{Feedback} \label{subsec-feed} 

{\em Feedback control laws} are laws in which the control is chosen
as a ``function'' of the state of the system, or of  that part 
of the state that is known through measurements.

The simplest form of feedback is ``static state feedback,''
in which $u$ is chosen as a function of the state $q$.
(When you 
drive your car, you are applying a control {\mit u}, consisting of the
pressure exerted by your foot on the brake, the pressure on the gas
pedal, the motion of the steering wheel, and other parameters.  After
you have driven from a point {\mit A} to another point
{\mit B}, you can look back at your
trajectory, and observe that the control {\mit u} that you have
applied was a certain function of time. This, however, does not mean
that the control function $t\mapsto\eta(t)$
was specified by you {\mit a priori}, and the motion
of the car was then determined by solving (\ref{dyna}). Your driving
strategy involved choosing {\mit u} as function of the {\myninebf
state} of the car (for example, ``turn left when you get the corner of
Broadway and 57th,'' or ``slow down when
your distance to $B$ is smaller than a certain
threshold and your speed is larger than a certain value'').  
This is {\myninebf feedback}. A state
feedback law, when applied with a given initial condition, will give
rise {\mit a posteriori} to an open-loop control. Conceptually, hwever,
a state feedback is a very different kind of  object.

We will explain below that the general concept of a feedback control is 
much broader than this, but before we do that we present
a trivial example of state feedback.

\begin{example}{\rm 
Consider the dynamical behavior described by the equation
\begin{equation}
\dot q=u\,,\qquad q\in \rr^n\,.
\end{equation}
Here the vector $u$ is arbitrary but subject to a bound $||u||\le c$,
where $c>0$ is a constant.  Suppose you 
want to reach the origin in minimum time. The obvious solution, starting at
a point $\bar q$, is to take 
\begin{equation}\label{sol1}
\eta(t)=-c\frac{\bar q}{||\bar q||}\,. 
\end{equation}
Now think of a boat in a still lake, which can move in any direction with
speed not 
greater than $c$, and suppose that your objective is to get to some 
point---say a small island---no matter where you may have started.  
An obvious physical
solution would be to tie a rope to a pole located at the target point, and
pull.  Mathematically, this corresponds to choosing
\begin{equation}\label{sol2}
u=\omega(q)=-c\frac{q}{||q||}\,.
\end{equation}
There is an enormous difference between the solutions given by (\ref{sol1})
and (\ref{sol2}). In
the former case, you compute the function $t\mapsto\eta(t)$ once and for all,
and then follow it.  If you made a small error in your initial estimation
of $\bar q$, or if there is a small error in the dynamics---e.g., if the
motion is really governed by 
$$
\dot q=u+v(q)\,,
$$
where $v$ is a small
perturbation term, corresponding, for example, to the fact that the lake is
not quite still---then you will miss the target.  With the feedback
solution (\ref{sol2}) you will not.  Whether or not the feedback solution 
``makes sense''---i.e., can be implemented---for a particular problem, is an
engineering question, whose answer depends on such issues as whether there
really is a pole in the target island, and enough rope is 
available.}  \proofend
\end{example}
As we can see from this simple example, a control design problem may have
different kinds of solutions, whose relevance depends on specific
engineering aspects of the situation being modeled.

\begin{example}{\rm 
For a mathematically richer example, consider the equation 
\begin{equation}\label{frictless}
\ddot x+ax=u\,,
\end{equation}
which governs the motion of 
a frictionless pendulum under the action of an external force $u$.  (The
coefficient $a$ is a parameter, and we assume that
$a>0$.) Suppose our objective is ``to drive $x$ and $\dot x$ to $0$.''
We can
rewrite this as a 2-dimensional system 
$$
\dot q=f(q,u)\,,
$$
by letting
$$
q=\left[\begin{array}{l}q^1\\q^2\end{array}\right]=
\left[\begin{array}{l}x\\ \dot x\end{array}\right]\,,
$$
and 
$$
f(q,u)=\left[\begin{array}{l}q^2\\u-aq^1\end{array}\right]\,,
$$
so the equations of motion
read: 
$$
\left\{\begin{array}{lcl}\dot q^1&=&q^2\,,\\
\dot q^2&=&u-aq^1\,.\end{array}\right.
$$
It is easy---and completely
uninteresting and useless---to see that 

\medskip
\begin{center}
\begin{minipage}{11.3cm}
{\em given any initial condition
$(\bar q^1,\bar q^2)$, there exists a control function 
$t\mapsto\eta(t)$ that gives rise to a
solution $t\mapsto\xi(t)$ such that $\xi(t)\to 0$ as $t\to+\infty$.}
\end{minipage}
\end{center}

\medskip\noindent  (PROOF: Let 
$[0,\infty\,[\,\ni t\mapsto x(t)\in\rr$ be any function of class $C^2$
that goes to $0$ together with its first 
derivative, and satisfies $x(0)=\bar q^1$, $\bar x(0)=\bar q^2$.
Then define $\eta(t)=\ddot x(t)+ax(t)$.)

Much more interesting is the question of choosing $u$ as a {\em feedback
control law}, that is, a ``function'' of the state variables $q^1,q^2$
(i.e., $x$ and $\dot x$) so as to {\em stabilize} the system, i.e., to
guarantee that all the solutions go to $0$.  One obvious way to do that is
to {\em create a friction force}, for example by taking 
$$
u=-c\dot x\,,\quad{\rm  with}\quad c>0\,.
$$
This solution is realistic, as long as one can ``have access'' to the
velocity $\dot x$.  (For example, a possible physical implementation of the
control law would be to add friction by just dipping the pendulum in a
container full of water.) } \proofend
\end{example}

\begin{example}{\rm Let us consider the system of the previous
example, but supposing now
that only the position $x$ is available to us, i.e., we want
$u$ to ``depend on $x$ only.''  How can we stabilize the pendulum?  We
recommend that the reader try answering this question as an exercise.
The answer will be given below in Scholium \ref{schol-solution}, but 
we provide here a  long hint. 

First, a {\bf WARNING}. Almost certainly, your first guess will be to look
for a function  
$$
u=u(x)
$$
of the position $x$ such that all the solutions of the resulting equation
$$
\ddot x+ax=u(x)
$$
go to zero. But if you try this then you will not get very far,  
because {\em such a function cannot exist.}  The reason is quite simple:
define the {\em energy} $E(x,\dot x)$ by 
$$
E(x,\dot x)=\dot x^2+ax^2-2U(x)\,,
$$
where 
$$
U(x)=\int_0^xu(y)\,dy\,.
$$  
Then {\em $E(x,\dot x)$ is constant along all trajectories}, because 
its derivative along a trajectory $t\mapsto x(t)$ is given by 
\begin{eqnarray*}
\frac{d\,\,}{dt}E(x(t),\dot x(t))
&=&2\dot x(t)\ddot x(t)+2ax(t)\dot x(t)-2u(x(t))\dot x(t)\\
&=&2\dot x(t)\Big(\ddot x(t)+ax(t)-u(x(t))\Big)\\
&=&0\,.
\end{eqnarray*}
It follows that, if you start with an initial condition 
$q_{in}=(x(0),\dot x(0))$ such that $E(q_{in})\neq 0$, then 
$x(t)$ and $\dot x(t)$ cannot go to zero. 

This would seem to suggest that the answer to our question is 
that what we were asked to do cannot be done, i.e., that 
``the frictionless pendulum described by equation (\ref{frictless})
cannot be stabilized by means of a feedback control law that depends 
on the position $x$ alone.''   This is quite true {\em if by

\medskip

\begin{center}
\begin{minipage}{11cm}
\noindent (I)$\quad$ a feedback control law that depends on the position
$x$ alone 
\end{minipage}
\end{center}

\medskip\noindent we mean 

\medskip

\begin{center}
\begin{minipage}{11cm}
\noindent (II)$\quad$ a law of the form $u=u(x)$. 
\end{minipage}
\end{center}
}

\medskip\noindent But, if you read carefully the question as it was
actually stated, you will notice that we never said that 
(I) is supposed to mean (II). When we talked about $u$
``depending on $x$ only,'' we did {\em not} say that this meant
``choosing $u=u(x)$.''\Ft{Actually, we did not say what (I) meant 
at all. This is in fact a typical example of the problems 
that arise every day when ``applied mathematicans'' and users 
of mathematics---in this case engineers---try to  communicate.
Mathematicians will often hurry and attach to a not very precise 
question what they think is ``its obvious mathematical meaning,''
without realizing that the only reason why this meaning seems obvious to
them is that it was the first interpretation they could think of. 
They will then go back to the users with an impossibility theorem,
showing that what the users wanted cannot be done. The users will then 
think it over again, and point out that what they wanted 
should not have been interpreted that way anyhow.  At this point,
the mathematicians may go back to the drawing board and conclude that,
with a reasonable reinterpretation of the question's meaning, there is an
answer after all. Or perhaps the users will give up all hope of getting help
from mathematicians, and solve the problem by themselves. And, in either
case, each side will blame the other for the misunderstanding.}   
There are many other
enginnering ways to implement a relation of ``dependence of $u$ on $x$.''
For example, in a dynamic setting $u$ might be a {\em functional},
depending on past values of $x$ as well as present ones. That is,
at any time $t$ the quantity $u$ might depend on the whole collection
$\{x(s):s\le t\}$ of values of $x$ at all times prior to $t$.

This observation is the key to solving our the problem. We have 
to allow $u$ at time $t$ to be a function not just of $x$ at time $t$ but
also of {\em past values} of $x$. A nice way to do that is to have a
functional relation
$$
u=h(x,y)\,,
$$
where $y$ is a new variable that obeys some dynamical equation 
$$
\dot y=g(x,y)\,.
$$
One then looks for functions $g(x,y)$, $h(x,y)$, such that, for
the full system 
\begin{eqnarray*}
\dot x&=&w\,\\
\dot w&=&u-ax\,,\\
\dot y&=&g(x,y)\,,\\
u&=&h(x,y)\,,
\end{eqnarray*}
that is,
\begin{equation}\label{fullsys}\left\{\begin{array}{lcl}
\dot x&=&w\,\\
\dot w&=&h(x,y)-ax\,,\\
\dot y&=&g(x,y)\,,
\end{array}\right.
\end{equation}
all the trajectories converge to $0$ as $t\to+\infty$.
(Notice that we have introduced a new variable $w$ to stand
for $\dot x$, so that (\ref{fullsys}) becomes a system of first-order
differential equations.  The precise technical requirement
imposed on the system (\ref{fullsys}) is that $0$ be a
globally asymptotically stable equilibrium.)

One should think of $y$ as the state variable of a
``controller'' or ``regulator'' system that we, the controllers,
are running, for example by solving the equation $\dot y(t)=g(x(t),y(t))$
on line  with a digital computer,  or by feeding $x$ as a physical input
---e.g., a voltage---into some physical device such as an electric circuit 
which evolves according to $\dot y(t)=g(x(t),y(t))$.   The value 
of $x(t)$ is supposed to be available to us at each time $t$, 
and $y(t)$ is something we can compute or measure. The equation
$u=h(x,y)$ produces our control $u$ as a function of
$x$ and the new variable $y$. Naturally, we do not allow $u$ to depend 
on $w$, because $w$---i.e., $\dot x$---is not available. 

\def\myli{\lower17pt\vbox{\hbox{\vrule height 38.5pt width 0.3pt}}}

In engineering terms, if we think of the dynamical law given by 
(\ref{frictless}) as defining an ``input-to-output'' relation
$\Sigma$, with ``input'' $u$, ``output'' $x$, and internal---or
``state''---variables $x$ and $\dot x$, and use the notation
\begin{equation}
\ovarrow{u}
\fbox{\begin{minipage}{0.4cm}$\Sigma$\end{minipage}}
\ovarrow{x}
\end{equation}
to represent such a correspondence, 
then we are asking to connect this to another regulator system 
\begin{equation}
\quad\ovarrow{x}
\fbox{\begin{minipage}{0.4cm}$\mathcal{R}$\end{minipage}}
\ovarrow{u}\,\,\,\,,
\end{equation}
where we will feed $x$ as input, and obtain the desired control $u$
as output. The new system $\mathcal{R}$ should have 
an internal, or ``state''  variable $y$ (which can be a scalar or a vector),
a dynamical law $\dot y=g(x,y)$, and output given by $u=h(x,y)$.  The
desired objective is that the full ``closed-loop system'' 
\begin{equation}
\myli\!\!\!\!\!\left.\begin{array}{rcl}
\,\,&\,\,&\,\,\\
\ovarrow{u}&
\!\!\!\!\!\!\fbox{\begin{minipage}{0.4cm}$\Sigma$\end{minipage}}&
\!\!\!\!\!\!\kern-0.7pt{\stackrel{x}{{\raise2.2pt\hbox to 39.7pt{\hrulefill}}}}
\\
\,\,&\,\,&\,\,\\\,\,&\,\,&\,\,\\
\!{\stackrel{u}{{\raise2.2pt\hbox to 39pt{\hrulefill}}}}&
\!\!\!\!\!\!\fbox{\begin{minipage}{0.4cm}$\mathcal{R}$\end{minipage}}&
\!\!\!\!\!\!\ovlarrow{x}\\
\,\,&\,\,&\,\,\end{array}\right.\!\!\!\!\!\myli
\end{equation}
be such that all the trajectories go to zero.


One possible choice of $g$ and $h$ is given by 
\begin{eqnarray*}
h(x,y)&=&y\,,\\
g(x,y)&=&ax-y\,.
\end{eqnarray*}
This will give 
$$
\dot u=\dot y=ax-y=-\dot w=-\ddot x\,,
$$
i.e., 
$$
u=-\dot x+{\rm constant}\,,
$$
which is close to what we need, since we know that if we had 
$u=-\dot x$ then the equation $\ddot x+ax=u$ reduces to
$\ddot x+\dot x+ax=0$, all whose solutions $t\mapsto x(t)$ 
decay exponentially as $t\to+\infty$. 

But we have not yet truly solved the problem! To do so, we would 
have to get rid of the constant! The readers should try to 
do this, and if they conclude that the question is hard, then they may
turn to Scholium  \ref{schol-solution}.

For those readers who can answer this question easily but are 
not control theorists, we propose a few 
harder questions:

\bi
\item[1.] Suppose you know that $a>0$ but other than that you
do not know the value of $a$.  How would you then design a feedback
control to stabilize the system for all $a>0$? 
\ei
More generally, 
\bi
\item[2.] How would you design a stabilizing feedback if
you don't know that $a>0$? 
\ei
Even worse,  
\bi
\item[3.] How would you design a stabilizing feedback if all 
you know is that $x$ obeys an equation \label{stat-adaptive}
$$
\ddot x+b\dot x+ax=u\,,
$$
but you don't know anything about the values of $a$ and $b$, not even their 
signs?     
\ei
These questions are examples of {\bem adaptive control} problems.
We refer the reader to textbooks such as 
\cite{canudas,landau,sastry} for details.}\proofend
\end{example}

\gnu{JAN:  At this point, it would be great if we could include
a description of the behavioral approach, and how it
differs from and improves upon the ``inputs-and-outputs''
approach.    Naturally, there is no obvious upper 
bound for how long this part could be. (Anywhere from 1
to 100 pages will do.) It could take the form of an
entirely new chapter, if you want. The only constraint is that whatever 
you write should blend well with the stuff we already have.}


\ssection{The brachistochrone problem truly belongs to 
optimal control}  We have already pointed out that
``brachistochrone'' means ``shortest time'' (cf. Page \pageref{brachmean}),
and sortest time problems naturally belong to
optimal control.  This is an interesting, though somewhat fortuitous,
coincidence.  However, our claim the brachistochrone problem truly
belongs to control is based on much more serious 
considerations. We will show in Chapter \ref{controbrach} that 
\bi
\item[1.] the best solutions of Bernoulli's problem are those 
based on optimal control,
\item[2.] in its most natural formulation, the problem is defined by
an interesting dynamical law $\dot x=f(x,u)$ in four dimensions, which
gives rise to a highly nontrivial Lie-algebraic structure (involving
the ``diamond Lie algebra''), while its Lagrangian $L$ is $\equiv 1$,
i.e., completely trivial and uninteresting.  \ei



\scholium{Solution of the stabilization problem of \S \ref{subsec-feed}}
\label{schol-solution}

The regulator system $\mathcal{R}$ is taken to be of the form 
\begin{equation}\label{thesys-a}
\left\{\begin{array}{lcl}
\dot y_1&=&y_2+\alpha_1(x-y_1)\,,\\
\dot y_2&=&u-ay_1+\alpha_2(x-y_1)\,,\\
u&=&ay_1-y_1-y_2\,.
\end{array}\right.
\end{equation}
The interpretation of this is that $y_1$ and $y_2$ are 
estimated values of $x$ and $\dot x\,\,$\Ft{Never mind that $x$ is directly 
observed, so there really is no need to estimate it.}.  These estimated
valued are subject to the same evolution equations as 
$x$ and $\dot x$, that is, $\dot y_1=y_2$ and $\dot y_2=u-ay_1$---except 
that at each time we compare our observed value of $x$ 
with that of $y_1$, and make ``error corrections'' by adding to 
$dy_i$ and extra term $\alpha_i(x-y_1)dt$. A good stabilizing 
control law would be 
\begin{equation}\label{cannot}
u=ax-x-\dot x\,,
\end{equation}
which would transform our equation (\ref{frictless}) into the obviously 
stable dynamics given by 
\begin{equation}\label{will-ess}
\ddot x+\dot x+x=0\,.
\end{equation}
The law (\ref{cannot}) cannot, however, be implemented, because
$\dot x$ is not directly available, so we use instead 
the law 
\begin{equation}\label{cannot-a}
u=a y_1-y_1-y_2\,,
\end{equation}
in which $x$ and $\dot x$ are replaced by our estimates for their values. 

The intuition is then that, if our error-correcting mechanism works, 
then the estimation errors $e_1$, $e_2$  given by 
$$
e_1=y_1-x\,,\quad e_2=y_2-\dot x\,,
$$
will go to zero as $t\to+\infty$. Then $u$ will be close to the 
value given by (\ref{cannot}), so $x$ will essentially evolve according 
to (\ref{will-ess}).   

To be able to turn this into a rigorous argument, we have to make an
adequate choice of  the  
coefficients $\alpha_1$, $\alpha_2$. Clearly, the equation
$\dot y_1=y_2+\alpha_1(x-y_1)$ implies 
$$
\dot e_1=\dot y_1-\dot x=y_2-\dot x+\alpha_1e_1\,,
$$
so 
\begin{equation}\label{forexo}
\dot e_1=e_2+\alpha_1e_1\,.
\end{equation}
Also, 
\begin{eqnarray*}
\dot e_2&=&\dot y_2-\ddot x\\
&=&u-ay_1+\alpha_2(x-y_1)-(u-ax)\\
&=&a(x-y_1)+\alpha_2(x-y_1)\\
&=&(a+\alpha_2)e_1\,.
\end{eqnarray*}
Therefore
$$
\ddot e_1=\dot e_2+\alpha_1\dot e_1=(a+\alpha_2)e_1+
\alpha_1\dot e_1\,,
$$
so 
$$
\ddot e_1-\alpha_1\dot e_1-(a+\alpha_2)e_1=0\,.
$$
Therefore we can choose 
$$
\alpha_1=-1\,,\qquad\alpha_2=-1-a\,,
$$
and guarantee that $e_1$ satisfies
\begin{equation}\label{forexo-a}
\ddot e_1+\dot e_1+e_1=0\,,
\end{equation}
so $e_1$ and $\dot e_1$ decay exponentially.  With this choice of the
$\alpha_i$ it also 
follows---for example, because of (\ref{forexo}) and
(\ref{forexo-a})---that $e_2$ and  
$\dot e_2$ decay exponentially. If we now rewrite (\ref{thesys-a}) as
\begin{equation}
\left\{\begin{array}{lcl}
\dot y_1&=&y_2+\alpha_1 e_1\,,\\
\dot y_2&=&-y_1-y_2+\alpha_2 e_1\,,\\
\end{array}\right.
\end{equation}
we see that 
$$
\ddot y_1=\dot y_2+\alpha_1\dot e_1=
-y_1-y_2+\alpha_1\dot e_1+\alpha_2 e_1\,,
$$
that is,
$$
\ddot y_1+y_1+y_2+\alpha_1\dot e_1+\alpha_2 e_1=0\,.
$$
But $\dot y_1=y_2+\alpha_1 e_1$, so 
$$
\ddot y_1+\dot y_1+y_1+y_2+\alpha_1(\dot e_1-e_1)+\alpha_2 e_1=0\,.
$$
Since $e_1$ and $\dot e_1$ decay exponentially, and the equation 
$\ddot y+\dot y+y=0$ is exponentially stable, we see that 
$y_1$ and $\dot y_1$ decay exponentially.  Since $x=y_1+e_1$ and 
$\dot x=\dot y_1+\dot e_1$.  we see that
$x$ and $\dot x$ decay exponentially as well. \proofend

\gnu{JAN: I have written down in full detail the
solution of the adaptive control problem stated as No. 3 in page 
\pageref{stat-adaptive}. Do you think we should put it in? It's fairly 
long, and I see reasons for having it (it's far from obvious 
for mathematicians who don't know about control) and reasons
against (it would be straying too far from our main subject).
Wha do you think?}


\cchapter{Adaptive stabilization}

\noindent The plant we want to stabilize is given by
$$
\ddot x + c\dot x+ kx = u \,.
$$
Here the parameters $c$ and $k$ are unknown, are we are only allowed to use
position measurements.

The adaptive stabilizer is going to be a $6$-dimensional system with state
vector 
$$
Z=(X,U,\xi,\eta,\gamma,\kappa)\,.
$$
This system will obey a
dynamical law $\dot Z=F(Z,x)$, driven by $x$ as an input, and will produce
as an output a function $u=\omega(Z,x)$, which will then be fed back 
to the plant, yielding an $8$-dimensional Plant+Regulator
closed-loop system 
\begin{eqnarray*}
\dot x&=&y\,,\\
\dot y&=&-kx-cy+\omega(Z,x)\,,\\
\dot Z&=&F(Z,x)\,.
\end{eqnarray*}
We want to choose $F$ and $\omega$ so that for every initial condition 
the trajectories of the closed-loop system stay bounded for all 
positive times, and the $x$ and $y$ components go to zero. We will actually
get all the cooordinates of the vector $(x,y,X,U,\xi,\eta,\gamma-c,\kappa-k)$ 
to go to zero, except for the last two, which will just stay
bounded but might not go to zero.   

You should think of 

\medbreak\ \ \ \ \ $X$ as a filtered version of $x$, 

\ \ \ \ \ $U$ as a filtered version of $u$, 

\ \ \ \ \ $\xi$ as an estimate for $X$, 

\ \ \ \ \ $\eta$ as an estimate for $\dot X$, 

\ \ \ \ \ $\gamma$ as an estimate for $c$, 

\ \ \ \ \ $\kappa$ as an estimate for $k$. 

\medbreak\noindent The dynamical equations for the adaptive stabilizer 
are:
\begin{eqnarray*}
\dot X&=&x-X\,,\\
\dot U&=&\omega(Z,x)-U\,,\\
\dot\xi&=&\eta+\lambda(Z,x)(X-\xi)\,,\\
\dot\eta&=&-\kappa\xi-\gamma\eta+\sigma(Z,x)(X-\xi)+U\,,\\
\dot\gamma&=&\varphi(Z,x)\,,\\
\dot\kappa&=&\psi(Z,x)\,.\\
\end{eqnarray*}
The five functions $\omega$, $\lambda$, $\sigma$, $\varphi$, $\psi$
will be chosen later. 

We now look at how the ``estimation error'' $E=X-\xi$ evolves
along a trajectory $t\to\Xi(t)$ of the full closed-loop system. We have 
$$
\dot\xi=\eta+\lambda E
$$ and 
$$
\dot\eta+\kappa\xi+\gamma\eta=\sigma E+U\,.
$$
So 
$$
\dot\eta+\kappa\xi+\gamma(\dot\xi-\lambda E)=\sigma E+U\,,
$$
and then 
$$
\dot\eta+\gamma\dot\xi+\kappa\xi=(\gamma\lambda +\sigma)E+U\,.
$$
Also
$$
\ddot\xi=\dot\eta+\dot\lambda E+\lambda\dot E\,,
$$
so 
$$
\dot\eta=\ddot\xi-(\dot\lambda E+\lambda\dot E)\,,
$$
and then
$$
\ddot\xi-(\dot\lambda E+\lambda\dot E)+
\gamma\dot\xi+\kappa\xi=(\gamma\lambda +\sigma)E+U\,,
$$
so that 
$$
\ddot\xi+\gamma\dot\xi+\kappa\xi=(\gamma\lambda +\sigma+\dot\lambda)E+
\lambda\dot E+U\,.
$$
On the other hand, $x$ satisfies 
$$
\ddot x+c\dot x+kx=\omega\,,
$$
i.e. $Ax=\omega$, where $A$ is the differential operator $D^2+cD+k$.
So, if we write $x=\dot X+X$, i.e. $x=BX$---where $B$ is the differential
operator $D+1$---we find
$$
BAX=ABX=Ax=\omega\,.
$$
So, if we let $W=AX$, we have $BW=\omega$, that is 
$$
\dot W=\omega-W\,.
$$
If we compare this with $\dot U=\omega-U$, we see that the difference 
$H=W-U$ satisfies  $\dot H=-H$. So $H$ will decay exponentially 
as $t\uparrow+\infty$. We now have $AX=W=U+H$, i.e.
$$
\ddot X+c\dot X+kX=U+H\,.
$$
Then 
$$
(\ddot X+c\dot X+kX)-(\ddot\xi+\gamma\dot\xi+\kappa\xi)
=U+H-(\gamma\lambda +\sigma+\dot\lambda)E-\lambda\dot E-U\,,
$$
so that
$$
\ddot E+\gamma\dot E+\kappa E+(c-\gamma)\dot X+(k-\kappa)X
=U+H-(\gamma\lambda +\sigma+\dot\lambda)E
-\lambda\dot E-U\,.
$$
Therefore
$$
\ddot E+\gamma\dot E+\kappa E
=H-(\gamma\lambda +\sigma+\dot\lambda)E
-\lambda\dot E+(\gamma-c)\dot X+(\kappa-k)X
\,.
$$
We now choose 
$$
\lambda=1-\gamma\,,
$$
and
$$
\sigma=1-\kappa-(\gamma\lambda+\dot\lambda)\,.
$$
(Notice that $\dot\lambda=-\dot\gamma$, which is a perfectly well defined
function of $Z$ and $x$ thanks to the dynamical equations.) 

With these choices, the equation for $E$ becomes 
$$
\ddot E+\dot E+E=H+(\gamma-c)\dot X+(\kappa-k)X\,.
$$
We now define 
$$
V=E^2+E\dot E+\dot E^2+{1\over 2}\Big(
(\gamma-c)^2+(\kappa-k)^2\Big)\,, 
$$
and observe that $V$ is a function of $x$ and $Z$ (that is, a function
of the full state $(x,y,Z)$ of the closed-loop system 
which does NOT depend on $y$), because $E=X-\xi$, $\dot X=x-X$, and 
$$
\dot E=\dot X-\dot\xi=x-X-\eta-\lambda(Z,x).(X-\xi)=
x-X-\eta-(1-\gamma).(X-\xi)\,.
$$
Moreover, 
$$
E^2+E\dot E+\dot E^2=\Big(E+{\dot E\over 2}\Big)^2+{3\over 4}\dot E^2\,,
$$
so $V$ is $\ge 0$ everywhere, and $V=0$ iff $E=\dot E=\gamma-c=\kappa-k=0$.

\medbreak\noindent NOTE: The most obvious ``Liapunov function''
for the $E$, $\dot E$ part would be $E^2+\dot E ^2$. However, the
derivative of this expression along trajectories is
$2\dot E(E+\ddot E)$, i.e. $-2\dot E^2+$extra stuff.   This is not good
enough because, even after we take care of the extra stuff, we will only be
controlling the behavior at $\infty$ of $\dot E$, and we will not 
be able to deal with $E$ itself.  That's why I am using 
$E^2+E\dot E+\dot E^2$ instead. The derivative now is 
$-(E^2+E\dot E+\dot E^2)+$extra stuff, and 
$E^2+E\dot E+\dot E^2$ is strictly positive definite in $E$ and $\dot E$. 

\medbreak Differentiation of $V$ along trajectories yields
$$
\dot V=2E\dot E+\dot E^2+\tilde E\ddot
E+\dot\gamma(\gamma-c)+\dot\kappa(\kappa-k)\,,   
$$
where 
$$
\tilde E=E+2\dot E\,.
$$
But
$$
\ddot E=-E-\dot E+H+(\gamma-c)\dot X+(\kappa-k)X\,.
$$
So 
$$\dot V=-E^2-E\dot E-\dot E^2+\tilde E\Big(H+(\gamma-c)\dot
X+(\kappa-k)X\Big)+\dot\gamma(\gamma-c)+\dot\kappa(\kappa-k)\,.
$$
Therefore
$$\dot V=-E^2-E\dot E-\dot E^2+\tilde E H+(\gamma-c)(\tilde E \dot
X+\dot\gamma)+(\kappa-k)(\tilde E X+\dot\kappa)\,.
$$
We now choose
$$\varphi=-\tilde E \dot X\,,\qquad\qquad\psi=-\tilde E X\,.
$$
Notice that $\varphi$ and $\psi$ are indeed functions of $Z$ and
$x$ (and in particular do NOT involve the derivative $\dot x$!). 

So now we have
$$
\dot V=-E^2-E\dot E-\dot E^2+\tilde E H
=-{\tilde E^2\over 4}-{3E^2\over 4}+\tilde E H\,.
$$
This almost says that $V$ is decreasing along trajectories.   To make this
rigorous, we use 
$$
\tilde E H\le{\tilde E^2\over 8}+2H^2\,,
$$
and conclude that 
$$
\dot V\le-{\tilde E^2\over 8}-{3\dot E^2\over 4}+2H^2\,.
$$
Since $H$ decays exponentially along each trajectory, $V$ has a finite 
limit as $t\uparrow\infty$,
and then $E$, $\dot E$, $\gamma$, and $\kappa$ stay bounded
as $t\uparrow\infty$. Moreover, if $t\to\Xi(t)$ is a trajectory of the full
closed-loop system, then
$$
\int_0^T{\cal E}(\Xi(t))dt\le V(\Xi(0))-V(\Xi(T))+2\int_0^TH(t)^2dt\,,
$$
where
$$
{\cal E}={\tilde E^2\over 8}+{3\dot E^2\over 4}\,.
$$
So the function $t\to{\cal E}(\Xi(t))$ is integrable on $[0,\infty\,[\,$, and
this implies that $t\to E(\Xi(t))$ and $t\to\dot E(\Xi(t))$ are
square-integrable on $[0,\infty\,[\,$.  It then follows that 
$E(\Xi(t))$ goes to $0$ as $t\to+\infty$. 

\medbreak
So far, the boundedness of $V$ only implies the boundedness of four 
of the eight variables of the closed-loop system. That is fine, because 
we have not yet said anything about how to choose our feedback $\omega$ so
as to stabilize the system. 


The idea now would be to use the ``certainty equivalence'' stabilizer  
$$
U= (\gamma-1)\eta+(\kappa-1)\xi\,,
$$ 
which has the property that, if $\gamma$ and $\kappa$ actually did have the
``right values'' $c$, $k$, and $\xi$, $\eta$ were indeed $X$ and $\dot X$,
then you would get the equation
$$
\ddot X + \dot X+X = H\,,
$$
which is clearly stable.   We cannot, however, choose $U$ this way, because
$U$ is a state variable. But we can choose $u$, because the function
$\omega$ is still at our disposal.  

To get $U= (\gamma-1)\eta+(\kappa-1)\xi$  we should have 
$$
\dot U=\dot\gamma\eta+(\gamma-1)\dot\eta+\dot\kappa\xi
+(\kappa-1)\dot\xi\,.
$$ 
This would yield
$$
\dot U+U=\dot\gamma\eta+(\gamma-1)(\eta+\dot\eta)+\dot\kappa\xi
+(\kappa-1)(\xi+\dot\xi)\,.
$$ 
So we define 
$$
\omega(Z,x)=
\dot\gamma\eta+(\gamma-1)(\eta+\dot\eta)+\dot\kappa\xi+(\kappa-1)(\xi+\dot\xi)\,,
$$
and observe that this is acceptable because $\xi$, $\eta$, $\gamma$,
$\kappa$ and their derivatives are just functions of $Z$ and $x$. 

Now, if we define 
$$
W= (\gamma-1)\eta+(\kappa-1)\xi\,,
$$ 
we see that $\dot W+W=\omega(Z,x)$. Since $\dot U+U=\omega(Z,x)$
by our dynamical laws, the difference $G=U-W$ satisfies
$\dot G=-G$, so $G$ decays exponentially along trajectories. 

So we now have 
$$
U= (\gamma-1)\eta+(\kappa-1)\xi+G\,,
$$ 
which is essentially the feedback we wanted, up to an exponentially
decaying term. 

The two equations for $\xi$ and $\eta$ now become
\begin{eqnarray*}
\dot\xi&=&eta+\lambda(Z,x)E\,,\\
\dot\eta&=&\xi-\eta+\sigma(Z,x).E+G\,.
\end{eqnarray*}
Therefore
\begin{eqnarray*}
\ddot\xi&=&\dot\eta+\dot\lambda E+\lambda\dot E\,,\\
&=&-\xi-\eta+\sigma E+G+\dot\lambda E+\lambda\dot E\,,\\
&=&\xi-\dot\xi+\lambda E+\sigma E+G+\dot\lambda E+\lambda\dot E\,,
\end{eqnarray*}
so that 
$$
\ddot\xi+\dot\xi+\xi=
(\lambda+\sigma+\dot\lambda)E+G+\lambda\dot E\,.
$$
Our choice of $\lambda$ and $\sigma$ is such that
$$
\sigma+\dot\lambda=1-\kappa-\gamma\lambda\,.
$$
So 
$$
\lambda+\sigma+\dot\lambda=1-\kappa-(\gamma-1)\lambda\,,
$$
and the equation for $\xi$ becomes
$$
\ddot\xi+\dot\xi+\xi=(1-\kappa-(\gamma-1)\lambda)E+G+\lambda\dot E\,.
$$
Since $\lambda=1-\gamma$, this says that
$$
\ddot\xi+\dot\xi+\xi=(1-\kappa+(\gamma-1)^2)E+G+(1-\gamma)\dot E\,.
$$

Now we are ready to conclude the stabilization proof. Let us work along a
fixed trajectory. We know that 
$E$ and $\dot E$ are square-integrable, and $G$ decays exponentially. 
So $\xi$ and $\dot\xi$ are bounded (because $s^2+s+1$ is stable). 
We also know that $E$ and $\dot E$ are bounded.  So 
$X$ and $\dot X$ are bounded, because $X=\xi+E$.    If we now go back to
the equation 
$$
\ddot E+\dot E+E=H+(\gamma-c)\dot X+(\kappa-k)X\,,
$$
and recall that $H$ decays exponentially and $\gamma$ and $\kappa$
are bounded, we can conclude 
that $\ddot E$ is bounded. Since we know that $\dot E$ is
square-integrable, we can now conclude that $\dot E$ goes to $0$. 

Now use again the equation
$\ddot\xi+\dot\xi+\xi=(\kappa-1+(\gamma-1)^2)E+G+(1-\gamma)\dot E$.
Since $G$ decays exponentially, $\gamma$ and $\kappa$
are bounded, and $E$ and $\dot E$ go to $0$, we conclude that $\xi$ and
$\dot\xi$ go to zero, and then the equation implies that 
$\ddot\xi$ goes to zero.  Since $X=\xi+E$, and 
$\xi$, $\dot\xi$, $E$ and $\dot E$ go to $0$, we can 
conclude that $X$ and $\dot X$ go to zero. Going back to 
$\ddot E+\dot E+E=H+(\gamma-c)\dot X+(\kappa-k)X$, we can now 
infer that $\ddot E\to 0$. Since $\ddot\xi\to 0$, we get 
$\ddot X\to 0$.   Now $x=\dot X+X$, and $y=\dot x=\ddot X+\dot X$.
So $x$ and $y$ go to zero. Finally, since 
$\ddot X+c\dot X+kX=U+H$, and we already know that 
$X$, $\dot X$ and $\ddot X$ go to $0$, we see that $U\to 0$ as well. 

\medbreak
\centerline{\hbox to 5cm{\hrulefill}}

\medbreak\noindent For completeness, we now write the 
complete system of equations for the adaptive stabilizer and the
closed-loop system.

\medbreak\begin{center}
\fbox{\fbox{
\begin{minipage}{12cm}
\begin{center}
{\bf FINAL FORM OF THE CLOSED-LOOP SYSTEM}
\end{center}

\begin{eqnarray*}
\dot x&=&y\,,\\
\dot y&=&-kx-cy-2X-U+\xi-2\eta+3X\gamma+2X\kappa+U\gamma-3\xi\gamma\\
&\,\,&+2\eta\gamma+\eta\kappa\\
&\,\,&+4xX\xi+3xX\eta+x\xi\eta
-5xX^2-x\xi^2+2x\eta^2-2x^2\eta+2x^2X-2x^2\xi\\
&\,\,&-2X\xi^2+X\xi\eta-2X\eta^2-2X\gamma\kappa-2X\gamma^2-X^2\xi-X^2\eta\\
&\,\,&+3X^3+2\xi\gamma^2+\xi\gamma\kappa-\eta\gamma^2\\
&\,\,&-10xX\xi\gamma+7xX^2\gamma+3x\xi^2\gamma\\
&\,\,&-2x^2X\gamma+2x^2\xi\gamma+3X\xi^2\gamma+X\gamma^3\\
&\,\,&+4X^2\xi\gamma-2X^2\xi\gamma+4X^2\xi\gamma\\
&\,\,&+2X^2\eta\gamma-2X^2\eta\gamma-3X^3\gamma-2X^3\gamma-\xi\gamma^3\\
&\,\,&+4xX\xi\gamma^2-2xX^2\gamma^2-2x\xi^2\gamma^2-4X^2\xi\gamma^2
+2X^3\gamma^2\\
\dot X&=&x-X\,,\\
\dot U&=&-2X-2U+\xi-2\eta+3X\gamma+2X\kappa+U\gamma-3\xi\gamma+
2\eta\gamma+\eta\kappa\\
&\,\,&+4xX\xi+3xX\eta+x\xi\eta
-5xX^2-x\xi^2+2x\eta^2-2x^2\eta+2x^2X-2x^2\xi\\
&\,\,&-2X\xi^2+X\xi\eta-2X\eta^2-2X\gamma\kappa-2X\gamma^2-X^2\xi-X^2\eta\\
&\,\,&+3X^3+2\xi\gamma^2+\xi\gamma\kappa-\eta\gamma^2\\
&\,\,&-10xX\xi\gamma+7xX^2\gamma+3x\xi^2\gamma\\
&\,\,&-2x^2X\gamma+2x^2\xi\gamma+3X\xi^2\gamma+X\gamma^3\\
&\,\,&4X^2\xi\gamma-2X^2\xi\gamma+4X^2\xi\gamma\\
&\,\,&+2X^2\eta\gamma-2X^2\eta\gamma-3X^3\gamma-2X^3\gamma-\xi\gamma^3\\
&\,\,&+4xX\xi\gamma^2-2xX^2\gamma^2-2x\xi^2\gamma^2
-4X^2\xi\gamma^2+2X^3\gamma^2\\
\dot\xi&=&X-\xi+\eta-X\gamma+\xi\gamma\,,\\
\dot\eta&=&+X-\xi-X\kappa-X\gamma+\xi\gamma-\eta\gamma\\
&\,\,&-6xX\xi+2xX\eta+5xX^2-2x\xi\eta+x\xi^2-2x^2X+2x^2\xi\\
&\,\,&+2X\xi\eta+X\xi^2+X\gamma^2+4X^2\xi-2X^2\eta-3X^3-\xi\gamma^2\\
&\,\,&+4xX\xi\gamma-2xX^2\gamma-2x\xi^2\gamma-4X^2\xi\gamma+2X^3\gamma\,,\\
\dot\gamma&=&5xX-x\xi+2x\eta-2x^2+X\xi-2X\eta-3X^2-2xX\gamma+2x\xi\gamma\\
&\,\,&-2X\xi\gamma+2X^2\gamma\,,\\
\dot\kappa&=&-2xX-X\xi+2X\eta+3X^2+2X\xi\gamma-2X^2\gamma\,.
\end{eqnarray*}
\end{minipage}}}
\end{center}

The adaptive stabilizer is the $6$-dimensional system described by the last
six equations, with state variable $Z=(X,U,\xi,\eta,\gamma,\kappa)$. 

The feedback control law is $u=\omega(x,Z)$, where 
$\omega(Z,x)$ is the expression that, according to the right-hand side of
the second equation, is equal to $\dot y+cy+kx$. 

\cchapter{The behavioral approach}

Control is a multifaceted subject. The aim of this chapter is to put
the problems discussed in this book in a broad control and systems
theory perspective. We will discuss a number of main themes, in
particular, open loop and feedback control, and the various
mathematical views of the notion of a dynamical system that have
emerged from control and from the modelling of open systems.

The main subject of this book is \textit{trajectory optimization. } It
is about how to transfer the state of a dynamical system from one
point to another. As we have explained, this is a subject that has
roots going all the way back to antiquity.

A second aspect of control is as a process of decision making. Assume that continuous observations are made on a dynamical systems, and on the basis of these observations, actions are taken to guide the system, for example to steer the state of a system along a particular path. This idea of choosing control variables on the basis of measurements, is referred to as \textit{feedback control}. This way of feedback decision making is, of course, very much part of daily life. It guides the actions of individuals, of societies, of economic agents, etc. It is also very much part of technology. Feedback control devices have also been constructed since antiquity, in water wheels, to control the speed of windmills, temperature controlling devices, etc. Feedback control mechanisms became widespread during the age of industrialization in the seventeenth century, starting with various controllers, as the Watt regulator, that were used to control the speed and pressure in steam engines. However, with the advent of electronics, starting with the feedback amplifier, and certainly with the present day computers, feedback control is an increasingly important part of industrial processes. A good source for further reading about the history of feedback control is \cite{Mayr, Bennett}.   

A third way of viewing control is as design of subsystems aimed at achieving good performance of the overall system. These subsystems may or may not act as feedback devices. Examples of such passive controllers that do not act as feedback processors are mechanical dampers, pressure valves, heat fins, etc. Such devices are, of course, part of our daily environment. We will refer to this more general way of viewing control as \textit{behavioral control,} or \textit{control by interconnection}.

In this chapter we will briefly explain these various aspects of control. However, these developments have found a fall-out in various mathematical concepts, which form the core of an area of applied mathematics called \textit{Systems Theory}. We will often find it necessary and advantageous to put the developments in this broader perspective.

\section{Open loop versus feedback control} \label{statefb}

We start by sketching the important distinction between what is called
\textit{open loop control} and what is called \textit{feedback
control}. We will do this first in the context of state
feedback. Later on more general situations will be discussed.
Consider a system described by the first order differential equation 
\begin{equation} \label{i/s} 
\dot{x}=F(x,u)\,. 
\end{equation}
In (\ref{i/s}), $x \in \Rn, q \in \Rm,$ and $F:\Rn\times\Rm\rightarrow\Rn$.  
 
The variable $x$ is called the \textit{state} of the system (in a
mechanical system, for example, the state consists of the vector of
the positions and velocities in a multi-body system), $u$ is an
external input variable that is imposed by the environment (for
example, the input could be external forces that act on the multi-body
system), and $F$ is the \textit{vector field;} $F(x,u)$ tells how the
state $x$ of the system will involve when the system is in state $x$
and input $u$ is applied to it. Note that it may be more accurate to
call $F$ a \textit{family} of vector fields, since for each $u$,
$F(\cdot,u)$ defines a vector field in the usual sense of the
term. Throughout this chapter we will assume that we are dealing with
systems in which the time-signals take their values in finite
dimensional real vector spaces. More general situations in which the
variables, for example the state $x$ in (\ref{i/s}), belong to a
manifold, or, for example the input $u$ in (\ref{i/s}), satisfy
inequality constraints, are very meaningful in applications, but will
not be considered in this chapter, since they would unnecessarily
encumber the exposition.

For the purposes of this and the following sections, $u$ is a variable
that can be chosen in order to control the system, and will therefore
often be called the \textit{control (input).}  It is in the way in
which $u$ is generated that the rather subtle distinction between open
loop and feedback control emerges.

The simplest way of choosing the input is through a function $\omega:
\R \rightarrow \Rm$, resulting in the input signal $u=\omega(t)$ at
time $t$. Thus, if $\omega$ is the control input that is chosen, the
state of the system will evolve according to
\begin{equation}\label{openloop} 
\der x(t) = F(x(t),\omega(t)), 
\end{equation}
with a right hand side which, due to the presence of $\omega(t)$, is
time-varying. Under reasonable conditions of $F$ and $\omega$, there
will be a unique solution $x:\R\rightarrow\Rn$ to (\ref{openloop})
emerging form any initial state $x(t_0)=x_0$. This leads to
\textit{open loop control}. Different choices of $\omega(\cdot)$ lead
to different paths $x(\cdot)$. Some resulting trajectories $x(\cdot)$
may be bad, some may be good, some may be optimal (in some
well-defined sense). Many problems in trajectory optimization come
down to choosing the optimal $\omega(\cdot)$. The discussion of
mathematical techniques for finding these optimal $\omega(\cdot)$'s
form the core of this book.

However, in many applications the control input is not chosen as a
function of time directly, but as a function of the state. That is,
through a function $\alpha: \Rn \rightarrow \Rm.$ This function
$\alpha$ tells that if the state variable equals $x$, then the control
input will be $u=\alpha(x)$. The system state is then be governed by
the differential equation
\begin{equation}\label{feedback} 
\der x(t)=F(x(t),\alpha(t))\,.
\end{equation}
Under reasonable conditions on $F$ and $\alpha$, there will again be a
unique solution $x:\R\rightarrow\Rn$ to (\ref{feedback}) emerging form
any initial state $x(t_0)=x_0$. This leads to \textit{feedback
control}. Different choices of $\alpha$ lead to different paths
$x(\cdot)$. Some resulting trajectories $x(\cdot)$ may be bad, some
may be good, some may be optimal (in a well-defined sense). Many
control problems in feedback control come down to choosing the optimal
$\alpha(\cdot)$.

Note, very importantly, that for a given feedback law $\alpha: \Rn \rightarrow \Rm$, the trajectory $x(\cdot)$ that emerges from $x(t_0)=x_0$, can also be generated by applying the open loop control law $\omega(t)=\alpha(x(t))$. Thus the distinction between open loop and feedback control does not lie in the fact that different trajectories are actually generated, but by the  way in which these trajectories are generated. In open loop control, the controller is \textit{clock driven:} the clock time $t\in\R$ determines that the control input $u=\omega(t)\in\Rm$ will be used at time $t$. In state feedback  control, the controller is \textit{state driven:} the state variables $x\in\Rn$ determine that the control input $u=\alpha(x)$ will be used when the system is in state $x$.  

We now explain one of the main features for using feedback control:
robustness.  Assume that system (\ref{i/s}) is in state $x(t_0)=x_0$
at time $t_0$. Now consider for example the problem of transferring
the state $x_0$ at time $t_0$ to a desired state $x_{\rm final}$ in
minimum time over all open loop controls
$\omega:\R\rightarrow\mathbb{U}$, and over all feedback controls
$\alpha:\Rn\rightarrow\mathbb{U}$, with $\mathbb{U} \subset \Rm$ some
give constraint set (e.g., $\mathbb{U}=[-1,+1]^{\tt m}$). Of course,
in order for such optimal controls to exist, $\mathbb{U}$ will
typically have to be a compact subset of $\Rm$, but the precise nature
of $\mathbb{U}$ is not important for the present exposition.  Assume
that we have computed both the `optimal' open loop
$\omega^\ast:\R\rightarrow\mathbb{U}$, and the `optimal' state
feedback control law $\alpha^\ast:\Rn\rightarrow\mathbb{U}$.  The
resulting trajectories, i.e., the solutions of
\[  \der x'(t)=F(x'(t),\omega^\ast(t))),~~~ x'(t_0)=x_0, \] 
and 
\[ \der x''(t)=F(x''(t),\alpha^\ast(x''(t)),~~~ x''(t_0)=x_0, \] 

are thus both ``optimal.''  It is easy to see that this must lead to
$x'=x''$, which we will denote by $x^\ast$. Also,
$\omega^\ast(t)=\alpha^\ast(x^\ast(t))$.

So far, it looks like that there is no difference between using an
open loop or a feedback law. The difference emerges if we consider
disturbances, which leads to the main practical advantage of feedback
versus open loop control.  Assume that for some reason, the trajectory
$x^\ast$ becomes disturbed during the execution of the optimal
path. Say that a disturbance causes the system state at some time $t_1
\in (t_0, t_{\rm final})$ to jump from the expected state
$x^\ast(t_1)$ to the unexpected $x(t_1)=x_1$. Then the feedback
control will execute the trajectory
\[ \der x(t)=F(x(t),\alpha^\ast(x(t)),~~~ x(t_1)=x_1,\]
which, if $\alpha^\ast$ has been computed well, may now execute the
optimal state transfer from $x_1$ to $x_{\rm final}$. However, the
open loop trajectory, generated by
\[  \der x(t)=F(x(t),\omega^\ast(t))),~~~ x(t_1)=x_1, \]
unaware that a disturbance has happened, will in general not even
execute the desired transfer.
This shows an important form of \textit{robustness} of feedback
control, robustness against disturbances. Feedback also gives
robustness against variations in the system parameters. Two main
examples of such controllers are feedback amplifiers and
PID-controllers. See \cite{Book} for a discussion of these
controllers.

\section{Output feedback}

The use of state feedback instead of open loop control comes at a
price: indeed, it requires that all the state variables are
continuously measured, whereas open loop control only requires
knowledge of clock time.

In a typical application, the state dimension ${\tt n}$ is large. For
a multi-body system, the state dimension equals twice the number of
degrees of freedom, and measuring the state would require measuring
the position and velocity of each of the components.  For a flexible
system, for example, the state space even becomes in principle an
infinite dimensional space.  So, it is very desirable to consider
situations in which only part of the state, or, more generally, a
function of the state is measured. This leads to the following
generalization of (\ref{i/s}):
\begin{equation}\label{i/s/o} 
\dot{x}=F(x,u),~~~y=H(x,u)\,.
\end{equation}
Here $x\in\Rn,u\in\Rm,y\in\Rp$, and $F:\Rn\times\Rm\rightarrow\Rn,
H:\Rn,\Rm\rightarrow\Rp$. The `physical' meaning of $y$ is that it is
the output of the sensors, and $H$ expresses the law by which the
state and output generates the sensor outputs. Usually ${\tt m},{\tt
p}<<{\tt n}$.  The system (\ref{i/s/o}) is driven by the input $u$,
which causes by the vector field $F(x,u)$ the evolution of the state
$x$. The \textit{read-out map} $H(x,u)$ then gives the output. The
system (\ref{i/s/o}) is called an \textit{input/state/output system.}

An \textit{output feedback control law} is now also an
input/state/output system in its own right, but one that acts in the
opposite `direction' as (\ref{i/s/o}), i.e.
\begin{equation}\label{outputfeedback}
\dot{z}=K(z,y);~~~u=L(z,y):
\end{equation}
it takes the outputs of (\ref{i/s/o}) as its inputs, and yields the
inputs to (\ref{i/s/o}) as its outputs. In applications, the
controller is often implemented as a microprocessor based computing
device that accepts the sensor outputs and returns the actuator
inputs. An \textit{actuator} is typically a \textit{transducer} that
translates electronic signals into forces, currents, or similar
physical variables that are used to control a system.

Present-day control theory is completely centered around the signal
flow graph shown in figure \ref{intelligentcontrol}.  The system to be
controlled (called the \textit{plant} in the engineering literature)
has four terminals (supporting variables which are typically
vector-valued).  There are two input terminals, one for the control,
one for the disturbances (or other exogenous variables, as set-points,
reference signals, tracking inputs, etc.)  and there are two output
terminals, one for the measurements, and one for the to-be-controlled
variables.  By using feed-through terms in the plant equations this
configuration accommodates, by incorporating these variables in the
measurements, for the possible availability to the controller of
set-point settings, reference signals, or disturbance measurements for
feed-forward control, and, by incorporating the control input in the
to-be-controlled outputs, for penalizing excessive control action.
The control inputs are generated by means of actuators and the
measurements are made available through sensors. 



\begin{figure}[hbt]
  \centerline {\resizebox{7cm}{!}{ \input{feedbackfig.pstex_t}
      }}\caption{ Intelligent control \label{intelligentcontrol}}
\end{figure}

Usually, the
dynamics of the actuators and of the sensors are considered to be part
of the plant.  On the other hand, for the \textit{feedback
controller}, an input/output system in its own right, the inputs are
the measured variables, and the outputs the control variables. We call
this structure \textit{intelligent control}, since it consists of a
controller that uses observations to make decisions, which we view as
one of the essential features of thinking.In intelligent control, the
controller is thought of as a micro-processor type device which is
driven by the sensor outputs and which produces the actuator inputs
through an algorithm involving a combination of feedback,
identification, and adaptation.  Also, often loops expressing model
uncertainty are incorporated in the above as well.  Of course, many
variations, refinements, and special cases of this structure are of
interest, but the basic idea is that of supervisor reacting in an
intelligent way to observed events and measured signals. The aim of
feedback control is to design, for a given plant, the feedback
controller such that the closed loop system meets certain
specifications. For example, stability, disturbance attenuation,
optimality, robustness (for example, preservation of stability
robustness against parameter variations), etc.

In equations, this leads to
\begin{equation}\label{fulli/s/o}
\dot{x}=F(x,u,d),~~~y=H(x,u,d),~~~f=G(x,u,d)\,.
\end{equation}
Here $x\in\Rn,u\in\Rm,u\in\R^{\tt d},y\in\Rp, f\in\R^{\tt f}$, and
$F:\Rn\times\Rm\times\R^{\tt d} \rightarrow\Rn, H:\Rn\times\R^{\tt
d}\Rm\rightarrow\Rp, G:\Rn\times\R^{\tt d}\Rm\rightarrow\R^{\tt f}.$
Together with a controller
\begin{equation}\label{outputfeedback1} 
\dot{z}=K(z,y),~~~u=L(z,y)\,,
\end{equation}
this leads to the \textit{closed loop system} obtained by combining
(\ref{fulli/s/o}) with (\ref{outputfeedback1}). In the closed loop
system, $d$ should be considered the input, and $f$ the output. Note
that $u$ and $y$ become internal variables in the closed loop system.

\section{Input/output systems}

We have seen in the previous section how feedback control leads to
input/output structures.  The input/output framework can be seen as a
\textit{cause/effect} structure, with the inputs as the variables
which act as causes for the evolution of the system variables, and the
outputs the variables that are the effects.  Note that `feedback'
results a blurring of cause and effect, since what is the cause in the
plant, is the effect in the controller, and vice-versa.

In the present section, we discuss briefly input/output
systems. Since, we do no need to distinguish between different types
of inputs and outputs, we discuss 
\begin{equation}\label{i/s/o1}
\dot{x}=F(x,u),~~~y=H(x,u)\,.
\end{equation} 
The study of such systems is a very well-developed area of research
whose development strongly emerged since the 1960's, in the context of
Control Theory, and which resulted in an independent field called
\textit{Mathematical Systems Theory.}

The main aim of (\ref{i/s/o1}) is to describe the relation between
input and output trajectories. We use the term relation, since usually
(\ref{i/s/o1}) does not determine a mapping from the input trajectory
to the output trajectory. Indeed, the output trajectory depends not
only on the input trajectory, but also on the initial state. The state
usually plays an ancillary role: it merely serves to specify the
input/ouput relation. Questions that spring up are: \textit{When do
two systems as (\ref{i/s/o1}) define the same input/output relation?
When and how can a given input/output relation be expressed in the
form (\ref{i/s/o1})? When do we want to call (\ref{i/s/o1}) a stable
system?} Etc., etc.


The theory is particularly rich, which explicit, effectively
computable results, when the maps $F$ and $H$ in (\ref{i/s/o1}) are
linear, leading to the system
\begin{equation}\label{ABCD} 
\dot{x}=Ax+Bu,~~~y=Cx+Du\,,
\end{equation} 
with $A\in\Rnn, B\in\Rnm, C\in\Rpn, $ and $D\in \Rpm.$ This system is
commonly denoted as \ABCD. In this case, and assuming that the initial
state is zero at $t=0$, i.e., $x(0)=0$, it is possible to obtain the
input/output relation as the convolution integral
\[ \label{conv} y(t)=Du(t) + \int_0^t H(t-t')u(t')\,dt,\]
with $H:\R_{+}\rightarrow \Rpm$ given by $$H(t)=Ce^{At}B.$$ $D\delta +
H$ is called the \textit{impulse response matrix}. Equivalently,
assuming again $x(0)=0$, the input/output relation can also be
expressed in terms of the Laplace transforms $\hat{u}$ of the input
trajectory and $\hat{y}$ of the output trajectory as \[
\label{transfer} \hat{u}=G\hat{y},\] with $$G(s)=D+C(Is-A)^{-1}B.$$
The matrix of rational functions $G\in\R(s)^{{\tt p}\times{\tt m}}$ is
called the \textit{transfer function} of \ABCD.

These input/output representations lead to some interesting inverse
problems that go under the name \textit{realization theory: Given an
impulse response $H:\R_{+}\rightarrow\R^{{\tt p}\times{\tt m}}$, or a
transfer function $G \in \R(s)^{{\tt p}\times{\tt m}},$ does there
exists a `realization' \ABCD with the given impulse response or
transfer function? To what extent is it unique? What is the relation
between two realizations? Etc., etc.} The study of the systems \ABCD
is very deep and varied. This is in sharp contrast with the rather
limited range of issues involved in the study of the analogous flows
\[\dot{x}=Ax.\]

The system (\ref{ABCD}) is said to be \textit{input/output stable} if
$H\in\mathfrak{L}_1(\R_{+},\Rpm)$, equivalently if $G$ has all its
poles in the open left half of the complex plane. This is the case if
$A$ has all its eigenvalues in the open left half of the complex
case. When \ABCD is state controllable (see section
\ref{controllability}) and state observable (see section
\ref{observability}) this `if' is in fact `if and only if'. Two
important norms for systems \ABCD are the $\mathcal{H}_2$-norm and the
$\mathcal{H}_\infty$-norm (the $\mathcal{H}$ refers to Hardy
spaces). They are most easily defined in terms of the impulse response
or the transfer function. The $\mathcal{H}_2$-norm is finite if and
only if $D=0$ and the system is stable, and equals
\begin{eqnarray*} 
||
\left[ {\begin{array}{c|c} A&B\\ \hline C&0 \end{array}} \right]
||_{\mathcal{H}_2} &= & \sqrt{\int_0^\infty{\rm trace}(H^\top(t)H(t))\,dt}  \\
 &= & \sqrt{\frac{1}{2\pi} \int_0^\infty{\rm trace}(G^\top(-i\omega)
G(i\omega))\,d\omega}.
\end{eqnarray*}
The $\mathcal{H}_\infty$-norm is finite if and only if the system is
stable and equals
\begin{eqnarray*} 
||
\left[ {\begin{array}{c|c} A&B\\ \hline C&D \end{array}} \right]
||_{\mathcal{H}_\infty} &= & \sup_{\omega\in\R} ||G(i\omega)||,
\end{eqnarray*}
with $||G(i\omega)||$ the Euclidean induced matrix norm, i.e. square
root of the maximal eigenvalue of $G^\top(-i\omega)G(i\omega)$.  These
system norms measure the gain of the system, the degree to which the
input is amplified by the system. Especially the
$\mathcal{H}_\infty$-norm is quite natural, since it is the induced
norm of (\ref{conv}) viewed as a mapping from
$\mathfrak{L}_2(\R_{+}(\R,\Rm)$ to $\mathfrak{L}_2(\R_{+}(\R,\Rp)$. In
the next section, we will see how these norms enter in the design of
output feedback controllers.

\section{Optimal output feedback control}

Consider again the plant 
\begin{equation}\label{fulli/s/o1} 
\dot{x}=F(x,u,d),~~~y=H(x,d),~~~f=G(x,u,d)\,.
\end{equation}
Note that we have assumed that $H$ does not involve $u$ directly. This
will allow us to write explicit equations for the closed loop
system. The controller
\begin{equation}\label{outputfeedback2} 
\dot{z}=K(z,y),~~~u=L(z,y)\,,
\end{equation}
leads to the closed loop system
\begin{equation} \label{closedloop} 
\dot{\begin{bmatrix}x\\z \end{bmatrix}}= 
\begin{bmatrix}F(x,L(z,H(x,d)))\\K(z,H(x,d)) \end{bmatrix},
~~~f=G(x,d,L(z,H(x,d)))\,.
\end{equation}
This is a system with input $d$, output $f$ and state $(x,z)$. The
problem of optimal feedback control is to find, for a given functions
$F,H,G$, defining the plant (\ref{fulli/s/o1}), functions $K,L$,
defining the controller (\ref{outputfeedback2}), such that it is
optimal in some well-defined sense. The performance criterion, of
course, usually refers to the closed loop system
(\ref{closedloop}). Note that finding the optimal $\dim(z)$ is part of
the problem.

The most important and explicit results in the area of optimal
feedback control have been obtained for linear time-invariant systems
\[ \dot{x}=Ax+Bu+Gd,~~~ y=Cx+Dd,~~~ f=Hx+Ju+Qd,\]
with $A,B,G,C,D,H,J,Q $ real matrices of suitable dimension.
The two main problems that have been solved are the minimization of
the $\mathcal{H}_2$- and the $\mathcal{H}_\infty$-norm of the closed
loop system viewed as a system with the disturbances $d$ as input, and
the to-be-controlled output $f$ as output.  Actually, in both cases,
it is possible to prove that the optimal controller is also a finite
dimensional linear time-invariant system. Note however that linearity
was actually built in in our definition of the $\mathcal{H}_2$ and the
$\mathcal{H}_\infty$-norm, but these definitions are readily
generalized to more general systems.

The optimal feedback controller in the $\mathcal{H}_2$ case has the
same order as the plant $\dim(z)=\dim(x)$. The computation of the
optimal gains $(K,L,M,N)$ of the optimal feedback controller
\[ \dot{z}=Kz+Ly,~~~u=Mz+Ny \] 
involves the solution of two uncoupled Riccati equations
\cite{TSH}. The optimal $\mathcal{H}_2$ controller can also be given a
stochastic interpretation in the case that the disturbance input $d$
is modelled as a vector of stationary gaussian stochastic processes,
and the expected value of the norm squared of the to-be-controlled
output is to be minimized. It was, in fact, under this form that this
problem was first appeared and was solved in the 1960's, under the
name of the LQG (linear-quadratic-gaussian) problem. This problem has
in fact the well-known Kalman filter as a special case.

The optimal feedback controller in the $\mathcal{H}_\infty$ case is
not obtained directly, but first conditions are derived for the
existence of a feedback controller such that the
$\mathcal{H}_\infty$-norm of the closed loop system is less that some
number $\gamma^2$. Then by iterating on $\gamma$, the optimal
controller is obtained. Checking the existence of a feedback
controller such that the $\mathcal{H}_\infty$-norm of the closed loop
system is less than $\gamma^2$ involves solving two Riccati equations,
and checking a coupling condition. If the required controller exists,
then its state dimension is again the same as that of the plant,
$\dim(z)=\dim(x)$, and the gains $(K,L,M,N)$ of the optimal feedback
controller
\[ \dot{z}=Kz+Ly,~~~u=Mz+Ny \] 
follow from the solution of two Riccati equations. This result has
something spectacular about it, and was first obtained in
\cite{DGKF}. It is also covered in \cite{TSH}. The ramifications of
the $\mathcal{H}_\infty$ problem dominated control theory in the
1990's.

\section{Open versus closed systems} 

Control theory, together with signal processing, are typical areas of
applied mathematics that study \textit{'open'} systems, i.e. dynamical
systems whose evolution does not only depend on the initial conditions
and the system laws, but also on the environment. In the input/output
setting, this influence of the environment is captured by the inputs.
Dynamical systems whose evolution depend on the environment do not
only occur in control and signal processing. Indeed, they are the
rule, not the exception.  Many of the basic laws encountered in
physics and elsewhere are basically open systems. Newton's second law
gives the relation between position and force, but it does not explain
the force. Modelling the force is not part of Newton's second law. It
should be viewed as imposed by the environment. In Maxwell's
equations, relating the electric field, magnetic field, current
density, and charge density, there are also free variables. In
economic models, it may be natural to view things as consumer demand,
or exchange rates as exogenous inputs. The study of open systems is
indeed a much more cogent framework in which to cast the study of
dynamical systems than the rather restricted framework of flows (on
manifolds),
$$ \dot{x} = F(x), $$ 
which has carried the day in mathematics as a formalization of dynamics.

The question remains what a suitable mathematical framework is for the
study of open systems. We have argued that flows are not, but are
input/output systems the panacea?  In signal processing, and in
control using actuators and sensors, the input/output framework is
very natural indeed. However, most physical and economic systems do
not behave as signal processors, and this has led to the need for a
broader framework, behavioral systems.

\section{Behavioral systems}  

In order to capture clearly the notion of a behavioral system, we
start this section with some introductory remarks about the nature of
a mathematical models. We view a model as follows.  Assume that we
have a phenomenon (that is, a set of outcomes) that we try to model.
Nature (that is, the reality that governs this phenomenon) can produce
certain outcomes.  The totality of these possible outcomes
({\it{before}} we have modelled the phenomenon) forms a set $\U,$
called the {\it{universum}}.  A {\it{mathematical model}} restricts
the outcomes that a model declares possible to a subset $\B$ of $\U$:
according to the model, outcomes in $\B$ are possible, those not in
$\B$ are not possible. This subset $\B$ is called the {\it{ behavior}}
of the model.

As an example, assume that we wish to model the relation between the
temperature $T$, the pressure $P$, and the volume $V$ of a gas in a
container. Before we have modelled this phenomenon, every
$(T,P,V)\in\R_{+}^3=\mathbb{U}$ is possible. After we have modelled
(for example for an ideal gas), we obtain the behavior
$\B=\{(T,P,V)\in \R_{+}^3 \mid \frac{PV}{T} = \text{ a constant}\}.$

In the study of dynamical systems we are, more specifically,
interested in situations where the outcomes of the phenomenon are
signals, i.e., maps with independent variables (time for the case at
hand) and dependent variables (the space where the time-signals that
are being modelled take on their values).  In this case the universum
is therefore the collection of all maps from the set of independent
variables to the set of dependent variables.  It is hence convenient
to distinguish these sets explicitly in the notation: $\T$ (suggesting
`time') for the set of independent, and $\W$ for the set of dependent
variables.  Whence we define a {\it{dynamical system}} as a triple
$$\Sigma =(\W,\T,\B)$$ with $\T \subseteq \R$, and $\B$, the behavior,
a subset of $\W^\T$, ($\W^\T$ is the standard mathematical notation
for the set of all maps from $\T$ to $ \W$). In the behavioral
framework, therefore, a dynamical system is simply identified with a
family of trajectories.

For example, {\it{Newton's second law}} imposes a restriction that
relates the position $\vec{q} $ of a point mass and the force
$\vec{F}$ acting on it.  This relation is $\vec{F}= m
\frac{d^2}{dt^2}\vec{q}$, with $m$ the mass.  This is a dynamical
system with $\T=\R, \W=\R^3 \times \R^3$ (typical elements of $\B$ are
maps $w=(\vec{q},\vec{F}):\R \rightarrow \R^3 \times \R^3$) and
behavior $\B$ consisting of all maps $t \in \R \mapsto
(\vec{q},\vec{F})(t) \in \R^3 \times \R^3 $ that satisfy the
differential equation $\vec{F}= m \frac{d^2}{dt^2}\vec{q}$.

First principles laws in physics always state that some outcomes can
happen (those satisfying the model equations) while others cannot
happen (those violating the model equations).  This is a far distance
from specifying a system as being driven by free inputs which together
with an initial state specifies the other variables, the outputs.  The
behavioral framework treats a model for what it is: an exclusion law.

\section{Latent variables}

In the basic equations describing mathematical models, very often
other variables appear in addition to those whose behavior the model
aims at describing. The origin of these auxiliary variables varies
from case to case. They may be state variables (as in automata and
input/state/output systems); they may be potentials (as in the
well-known expressions for the solutions of Maxwell's equations); they
may be interconnection variables that express how the subsystems of an
interconnected system interact. It is important to incorporate these
auxiliary variables in our basic modeling language \textit{ab initio},
and to distinguish clearly between the variables whose behavior the
model aims at, and the auxiliary variables introduced in the modeling
process. We call the former \textit{manifest} variables, and the
latter \textit{latent} variables.

A {\it{mathematical model with latent variables}} is defined as a
triple $(\W,\LL,\Bfull)$ with $\W$ the universum of manifest
variables, $\LL$ the universum of latent variables, and $\Bfull
\subseteq \W\times\LL$ the \textit{full behavior}.  It induces the
{\it{manifest model}} $(\W,\B)$, with $$\B=\{ w \in \W ~| \mbox{ there
exists } \ell \in \LL \text{ such that} (w,\ell) \in \Bfull \}. $$ A
{\it{dynamical system with latent variables}} is defined completely
analogously as $\Sigma_{\rm latent}=(\T,\W,\LL,\Bfull) $ with $\Bfull
\subseteq (\W \times \LL)^\T$.  It induces the manifest dynamical
system $\Sigma=(\T,\W,\B)$, with \[ \B=\{w:\T\rightarrow\W\mid \text{
there exists } \ell:\T\rightarrow\LL \text{ such that }
(w,\ell)\in\Bfull\}. \]

The notion of a system with latent variables is usually the natural
end-point of a modeling process, for example by tearing and zooming,
and hence a very natural starting point for the analysis and synthesis
of systems.  We shall see that latent variables enter also very
forcefully in representation questions.

Examples:

1. In modeling the port behavior of an {electrical circuit}, the
manifest variables are the voltage, or vector of voltages, $V$ across
the port(s) and the current, or vector of currents, $I$ into the
circuit through the port(s). However, it is usually not possible to
come up directly with a model (say in the form of a differential
equation) that involves only the port variables $(V,I)$.  In order to
model the port behavior, we usually need to look at the internal
structure of the circuit, and introduce the currents through, and the
voltages across, the internal branches as latent variables. Using
Kirchhoff's laws and the constitutive equations of the elements in the
branches, this readily leads to a latent variable model for the port
variables.  Similar situations occur in other systems, for example in
mechanical systems with subcomponents, and, more generally, in any
type of interconnected system.  \newcommand{\Cinf}
{\mathfrak{C}^\infty}

2. Assume that we want to model the relation between the temperatures
and the heat flows radiated at the ends of a heat conducting bar of
length $1$. The bar is assumed to be isolated, except at the
ends. Denote the manifest variables by $T_0,T_1,q_0,q_1$. These are
the temperatures and heat flows at both ends. In order to obtain a
model, it is convenient to introduce the temperature distribution
$T(x,t), 0\leq x \leq 1,$ in the bar as latent variables. The full
behavior is then described by the partial differential equation
$$
\frac{\partial}{\partial t} T =\frac{\partial^2}{\partial x^2} T 
$$
combined with the boundary conditions
$$
T(0,t)=T_0(t), T(1,t)=T_1(t), \frac{\partial}{\partial x}T(0,t)= -q_0(t),
\frac{\partial}{\partial x}T(1,t)=q_1(t)\,.
$$
This defines a latent variable system with $\T=\R, \W=\R^4, \LL=
\R^{[0,1]}$, and $\Bfull$ consisting of all maps
$((T_0,T_1,q_0,q_1),T): \R \rightarrow \R^4 \times \R^{[0,1]})$
such that the above PDE and boundary conditions are satisfied.


3. In a state model for a dynamical system, the input/output behavior
is specified through a system of differential equations as
$$ \dot{x} = F(x,u),~~~ y = H(x,u). $$
This defines a latent variable system 
$(\R,\U \times \mathbb{Y},\mathbb{X},\Bfull)$ 
with $\Bfull$ all trajectories 
$ ((\omega,\eta),\xi):\R\rightarrow (\U \times \mathbb{Y}) \times \mathbb{X}$ 
that satisfy 
$$
\der \xi(t)=F(\xi(t),\omega(t)), ~~~ \eta(t)=H(\xi(t),\omega(t)\,,
$$ 
for all $t \in \R.$ The manifest behavior is the input/output
behavior, that is all trajectories $ (\omega,\eta):\R \rightarrow (\U
\times \mathbb{Y})$ that are 'supported' (in the sense made apparent
by the full behavior) by a state trajectory $\xi: \R \rightarrow
\mathbb{X}$.

Situations in which models use latent variables either for
mathematical reasons or in order to express the behavioral constraints
abound: internal voltages and currents in electrical circuits, momenta
in mechanics, chemical potentials, entropy and internal energy in
thermodynamics, prices in economics in order to explain production,
state variables, the wave function in quantum mechanics in order to
explain observables, the basic probability space $\Omega$ in
probability, etc.

In conclusion, the behavior, combined with manifest and latent
variables forms a very attractive framework for discussing
mathematical models in general, and dynamical systems in
particular. They have the standard structures studied traditionally
(flows, input/output systems, input/state/output systems, etc.) as
special cases. But, more importantly, they formalize what is usually
the end result of a modelling procedure, and that can definitely not
be said from any of these special cases.

\section{Differential systems}

The `ideology' that underlies the behavioral approach is the belief
that in a model of a dynamical (physical) phenomenon, it is the
behavior $\B$, i.e., a set of trajectories $w: \T \rightarrow \W$,
that is the central object of study.  However, as we have seen, in
first principles modeling, also latent variables enter \textit{ab
initio}. But, the set $\B$ or $\Bfull$ of trajectories must be
specified somehow, and it is here that differential (and difference)
equations enter the scene. Of course, there are important examples
where the behavior is specified in other ways (for example, in
Kepler's laws for planetary motion), but differential equations are
certainly the most prevalent specification of the behavior of a
dynamical system encountered in applications.  For $\T=\R$, and in the
case of systems without latent variables, $\B$ then consists of the
solutions of a system of differential equations as
$$F_1(w(t),\der w(t),\ldots,\frac{d^N}{dt^N}w(t))= F_2(w(t),\der
w(t),\ldots,\frac{d^N}{dt^N}w(t)).$$
We call these \textit{differential} systems.  

Of particular interest (at least in control, signal processing,
circuit theory, etc.) are systems with a signal space that is a
finite-dimensional vector space and behavior described by linear
constant-coefficient differential equations.  A \textit{linear
time-invariant differential system} is a dynamical system $\Sigma =
(\R,\W,\B)$, with $\W$ a finite-dimensional (real) vector space, say
$\Rw$, whose behavior consists of the solutions of
$$
R(\frac{d}{dt})w = 0, $$
with $R$ a real polynomial matrix of suitable dimensions.  Of course,
the number of columns of $R$ equals the dimension of $\W$. The number
of rows of $R$, which represents the number of equations, is
arbitrary.  In fact, when the row dimension of $R$ is less than its
column dimension, as is usually the case, $R(\frac{d}{dt})w=0$ is what
is sometimes called an {\em under-determined} system of differential
equations which is typical for models in which the influence of the
environment is taken into account. Indeed, in general, some of the
components of $w$ are left free by the differential equation
$R(\der)w=0$. They are specified by the environment. The initial
conditions are in general insufficient to determine the solution
$w:\R\rightarrow\Rw$.

The precise definition of what we consider a solution of
$R(\frac{d}{dt})w=0$ is an issue that we will slide over. But, to be
precise, we take it to be the set of $\Cinf$ solutions. Then the
behavior defined by $R(\frac{d}{dt})w=0$ equals \[ \B= \{w \in
\Cinf(\R,\Rw) \mid R(\frac{d}{dt})w=0\}.\] Hence $\B=\ker(R(\der)),$
with $\ker(R(\der))$ viewed as a map from $\Cinf(\R,\Rw) $ to
$\Cinf(\R,\R^{\rowdim(R)}) $.  We hence call $R(\frac{d}{dt})w=0$ a
\textit{kernel representation} of the associated dynamical
system. There are many other useful system representations, and we
shall meet some later on in this chapter.

The dynamical system $\Sigma=(\R,\Rw,\ker(R(\der))$ is \textit{linear} (the behavior is a linear space), \textit{time-invariant}  (the behavior is invariant under time-shifts) and \textit{differential} (the behavior is the solution set of a set of differential equations). We denote this set of dynamical systems by $\mathfrak{L}^{\tt w}$, when the signal space is $\Rw$, i.e., the number of manifest variables is ${\tt w}$. Denote further $\mathfrak{L}^\bullet = \cup_{{\tt w}\in \mathbb{Z}_+}\mathfrak{L}^\w$.  

A differential system  with latent variables, is described by
\begin{multline*}
        F_1(w(t),\der w(t),\ldots,\frac{d^N}{dt^N}w(t),\ell(t),
\der\ell(t),\ldots, \frac{d^N}{dt^N}\ell(t)) \\ 
= F_2(w(t),\der w(t),\ldots,\frac{d^N}{dt^N}w(t),\ell(t),\der\ell(t),\ldots,
\frac{d^N}{dt^N}\ell(t)), 
\end{multline*}
relating the (vector of) manifest variables $w$ to the (vector of)
latent variables $\ell$. In the linear time-invariant case this
becomes
$$
R(\frac{d}{dt})w = M(\frac{d}{dt})\ell, 
$$
with $R$ and $M$ real polynomial matrices of suitable
dimension. Define the \textit{manifest} behavior of this system as
$$
\{w \in \Cinf(\R,\Rw)
\mid \text{ there exists } \ell \in \Cinf(\R,\R^{\ell}) \mbox{ such that } 
R(\frac{d}{dt})w = M(\frac{d}{dt})\ell \}. 
$$
 
The question occurs whether $\B$ itself can be described by a linear
constant coefficient differential equation. In other words: \textit{Is
the projection of the set of solutions of a system of linear
time-invariant differential equations itself the set of solutions of a
system of linear constant coefficient equations?}  This is the case
indeed.  In fact: For any real polynomial matrices $(R,M)$ with
$\rowdim(R)=\rowdim(M)$, there exists a real polynomial matrix $R'$
such that the manifest behavior of $R(\frac{d}{dt})w =
M(\frac{d}{dt})\ell$ has the kernel representation $R'(\der)w=0$.

The above result is called the \textit{elimination theorem} for linear
differential systems. Its relevance in object-oriented modeling is as
follows. As can already be seen from simple electrical circuits, a
model obtained in the usual way involves very many variables and
equations, among them many algebraic ones. The elimination theorem
tells us that the latent variables may be eliminated and (in the case
of linear time-invariant differential systems) that the number of
equations can be reduced to no more than the number of manifest
variables. Of course, the order of the differential equation goes up
in the elimination process.

The elimination theorem also holds for linear constant coefficient
partial differential equations \cite{pillai}, but, we emphasize, not
for nonlinear differential systems. In other words, the fact that the
components of an interconnected system are all described by smooth
nonlinear differential equations is no guarantee for the manifest
behavior to be described by a differential equation. In fact, only in
exceptional cases will the manifest behavior of an interconnected
system be described by a system of differential equations involving
manifest variables only.

In the next three sections, we will show how some classical notions
from mathematical systems theory become generalized to behavioral
systems.

\section{Controllability} \label{controllability}

An important property in the analysis and synthesis of dynamical
systems is controllability.  Controllability refers to be ability of
transferring a system from one mode of operation to another. By
viewing the first mode of operation as undesired and the second one as
desirable, the relevance to control and other areas of applications
becomes clear.  However, the notion of controllability has found many
other applications, for example in stochastics, in the theory of state
representations, in model reduction, etc.

The concept of controllability has
originally been introduced in the context of state space systems. The classical
definition runs as follows.  The system described by the controlled
vector-field
$$\dot{x} =F(x,u)\,,$$ 
with $u\in\R^{\tt m}$, $x\in\R^{\tt n}$, and $F: \R^{\tt
n}\times\R^{\tt m}\rightarrow\R^{\tt n}$ is said to be
\textit{controllable} if $\text{for all } a,b \in \R^{\tt n}$ there
exists $T \geq 0$ and $\omega:[0,T] \rightarrow \R^{\tt m},$ such that
the solution to $$\der x(t) =F(x(t),\omega(t))$$ with initial $x(0)=a$
yields $x(T)=b$.  One of the elementary results of system theory
states that the finite-dimensional linear system $$\dot{x} = Ax + Bu$$
is controllable if and only if the composite matrix $$[B~~~AB~~~A^2B~~
\cdots ~~A^{\dim(x)-1}B]$$ has linearly independent rows.  Various
generalizations of this result to time-varying, to nonlinear (with
controllability conditions involving Lie brackets), and to
infinite-dimensional systems exist.

A disadvantage of this classical notion of controllability is that it
refers to a particular representation of a system, notably to a state
space representation.  Thus a system may be uncontrollable either for
the intrinsic reason that the control has insufficient influence on
the system variables, or because the state has been chosen in an
inefficient way. It is clearly not good to confuse these reasons.  It
is obviously desirable to have a notion of controllability that can
handle other system representations than state representations.  In
the context of behavioral systems, a definition of controllability has
been put forward that involves the system variables directly, and that
can be applied to any system representation.

Let $\Sigma=(\T,\W,\B)$ be a dynamical system with $\T=\R$ or $\Z$,
and assume that is time-invariant, that is $\B$ is invariant under
time shift, is said to be \textit{controllable} if for all $w_1,w_2
\in \B$ there exists $T\in \T$, $T \geq 0$ and $w \in \B$ such that
$w(t)=w_1(t)$ for $t<0$ and $w(t)=w_2(t-T)$ for $t \geq T$. Thus
controllability refers to the ability to steer the system from any one
trajectory in the behavior to any other one, allowing some time-delay.

Two questions that occur are the following: \textit{What conditions on the
parameters of a system representation imply controllability? Do
controllable systems admit a particular representation in which
controllability becomes apparent?} For linear time-invariant differential
systems, these questions can be answered very explicitly, as follows.
Let $\Sigma = (\R, \Rw,\B)$ be a linear time-invariant differential system. 
The following are equivalent:
\begin{enumerate}
\item  $\Sigma=(\R,\Rw,\B)\in\mathfrak{L}^\w$ is controllable;
\item The polynomial matrix $R$ in a kernel representation 
$R(\frac{d}{dt})w = 0$ of $\B$
 satisfies $\rank(R(\lambda)) = {\rm rank}(R)$ for all $\lambda \in \C$;
\item The behavior $\B$ is the image of a linear constant-coefficient
differential operator, that is, there exists a real polynomial matrix
$M $ of suitable dimension such that $\B=\{w \in \Cinf(\R,\Rw) \mid w
= M(\frac{d}{dt})\ell \text{ for some } \ell \in
\Cinf(\R,\R^{\coldim(\ell)}) \}$.
\end{enumerate}

There exist effective algorithms for verifying controllability of a
system starting from the coefficients of the polynomial matrix $R$ in
a kernel (or a latent variable) representation of $\Sigma$, but we
will not enter into these algorithmic aspects.

A point of the above result that is worth emphasizing is that
controllable systems admit a representation as the manifest behavior
of the latent variable system of the special form $$
w=M(\frac{d}{dt})\ell.  $$ We call this an \textit{image}
representation of the $w$-behavior. It follows from the elimination
theorem that every system in image representation can be brought in
kernel representation. But not every system in kernel representation
can be brought in image representation: it is precisely the
controllable ones for which this is possible.

The controllability issue has been pursued for many other classes of
systems.  In particular (more difficult to prove) generalizations have
been derived for differential-delay, for linear \textit{n-D} systems,
for linear constant coefficient \textit{PDE}'s, and for limited
classes of nonlinear differential systems.  Systems in an image
representation, and close relatives of it, have received much
attention recently for nonlinear differential-algebraic systems, where
they are referred to as \textit{flat} systems.

\section{Observability} \label{observability}

A second notion that has become central in systems theory is that of
observability.  The notion of observability is always introduced hand
in hand with controllability. In the context of the input/state/output
system
$$
\dot{x} =F(x,u),~~~ y=H(x,u)
$$ 
it refers to the possibility of deducing, using the laws of the
system, the state from observation of the input and the output
trajectories. For the linear system \[\dot{x}=Ax+Bu,~~~y=Cx+Du,\]
state observability holds if and only if the composite matrix
$$[C^\top~~~A^\top C^\top~~~(A^\top)^2C^\top~~ \cdots
~~(A^\top)^{\dim(x)-1}C^\top]$$ has linearly independent rows.

The definition that is used in the behavioral context is more general
in that the variables that are observed and the variables that need to
be deduced from the observed variables are kept general.

Let $\Sigma=(\T,\W,\B)$ be a dynamical system, and assume that $\W$ is
a produt space, $\W=\W_1 \times \W_2$. Hence elements of $\B$ are
pairs $(w_1,w_2):\T\rightarrow \W_1 \times \W_2$. Then $w_1$ is said
to be \textit{observable} from $w_2$ in $\Sigma$ if $(w_1,w'_2) \in
\B$ and $(w_1,w''_2) \in \B$ imply $w'_2 = w''_2$. Observability thus
refers to the possibility of deducing the trajectory $w_1$ from
observation of $w_2$ and from the laws of the system ($\B$ is assumed
to be known).

The theory of observability runs parallel to that of controllability. 
Let $\Sigma = (\R, \R^{\w_1}\times \R^{\w_2},\B)$ be a linear time-invariant
differential system. The following are equivalent:
\begin{enumerate}
\item  In $\Sigma = (\R, \R^{\w_1}\times \R^{\w_2},\B)$, $w_2$ is observable 
from $w_1$;
\item The polynomial matrix $R_2$ in a kernel representation
$R_1(\frac{d}{dt})w_1 = R_2(\frac{d}{dt})w_2 $ of $\B$ satisfies
$\rank(R_2(\lambda)) = \dim(w_2)$ for all $\lambda \in \C$;
\item there exists a set of differential equations satisfied by
(`consequences of') the behavior, of the form $ w_1= R'_2(\der)w_2$,
which puts observability into evidence.
\end{enumerate}

\section{State representations}

As shown in the previous sections, there are many ways to represent a
system, as kernels, as images, with latent variables, etc.  The fact
that systems are often modelled with latent variables and this
obviously induces a multitude of descriptions of a given manifest
behavior.  One very important system representation is the state space
representation, as (\ref{i/s/o}). It is the introduction of this class
of system representations, combined with optimal control, that
propelled and greatly enhanced the mathematical and applications scope
of systems and control theory in the early 1960's. The state property
is a consequence of the fact that this is a first order differential
equation. In this section, we will formalize the state concept.  We
will now explain how the state can be viewed in the context of latent
variable systems in which the latent variable has the additional
property that it separates the past from the future.

Let $\Sigma=(\T,\W,\LL,\Bfull)$ be a latent variable dynamical system.
Then this system is said to be a {\it{state system}} if $\Bfull$ has
the following property: if $(w_1,\ell_1) \in \Bfull$, $(w_2,\ell_2)
\in \Bfull$ and $\ell_1(t_0)=\ell_2(t_0)$, then the concatenation of
$(w_1,\ell_1)$ and $(w_2,\ell_2)$ at $t_0$ must also belong to
$\Bfull$.  Concatenation is defined as
\[ 
  (w_1,\ell_1) \wedge_{t_0} (w_2,\ell_2)(t)= \left\{ \ba{ccc}
    (w_1,\ell_1)(t) & for & t < t_0 \\
    (w_2,\ell_2)(t) & for & t \geq t_0 \ea \right.  
\] 
This property must hold for all such $(w_1,\ell_1)$, $(w_2,\ell_2) \in
\B$, and all $t_0 \in \T$.  Henceforth, we will use the notation $x$
instead of $\ell$ when a latent variable representation has, or is
expected to have, the state property.

It is possible to prove, under some mild regularity conditions, that a
differential system is a state space system if and only if $\Bfull$
can be described by a differential equation that is \textit{first
order} in $x$ and \textit{zero-th order} in $w$, i.e., a differential
equation of the form
$$
F_1(w,x,\dot{x})=F_2(w,x,\dot{x})\,.  
$$
In particular, for linear time-invariant differential systems,
this yields the representation
$$
Kw+Lx+M\der x=0\,,
$$
with $K$, $L$, $M$ matrices of suitable dimensions. 

It can be shown that for every linear time-invariant differential
dynamical system (and hence, by the elimination theorem, for every
system described by linear constant-coefficient differential equations
with latent variables) it is possible to define latent variables
$x=X(\der)w$ through the polynomial matrix $X$ ($X(\der)$ is then
called a \textit{state map}) such that the full behavior defined by
$$\{(w,x) \in \Cinf(\R,\R^{\w+\dim(x)}) \mid R(\der)w=0, x=X(\der)w
\}$$ satisfies the state property. In other words, such that the
variables $w,x$ are related by a linear constant coefficient
differential equation that is \textit{first order} in $x$ and
\textit{zero-th order} in $w$.  Moreover, if the dimension of $x$
(equivalently, the number of rows of $X$) is as small as possible
(under the constraint that the relation between $x$ and $w$ is a state
system), then this first order state equation can actually be written
as
\[
\frac{d}{dt}x=Ax+Bw_1, ~~~w_2=Cx+D w_1\,,
\] 
with $(w_1,w_2)$ a component-wise partition of $w$, i.e., $(w_1,w_2) =
w$ up to a permutation of the components.

This shows how the popular input/state/output representation emerges
as a special system representation that involves (i) a suitable,
component-wise, partition of the manifest variables, and (ii) a
suitable choice of the latent variables to represent the behavior in
state form. Hence such an input/state/output representation displays
both the partition of its manifest variables in its free (input)
components and its bound (output) components, and also its memory (as
parameterized by the state variables). The usefulness of such
representations for the analysis and synthesis is uncontested.
However, as a \textit{starting point} for practical and theoretical
work, it is not suitable. It requires a great deal of manipulations
and computation to bring a first principles model into this form. For
example, first principles models invariably contain algebraic
equations, and more latent variables that only the state variables.

\section{Control as interconnection}

The behavioral point of view is quite compelling  as an
approach for modelling (open) dynamical systems.  It is reasonable
that when modelling a physical dynamical system it makes no sense to prejudice
oneself (as one would be forced to do in an input/output setting)
as to which variables should be viewed as inputs and which variables
should be viewed as outputs.  We have argued this point extensively
throughout the previous sections. This is not to say, by
any means, that there are no situations where the input/output
structure is natural. Quite to the contrary. In fact, whenever logic
devices are involved, in information processing, the input/output
structure is often a must.

On first sight, the behavioral approach seems less apt in the context
of control.  Firstly, there is something very natural in viewing
control variables as inputs and measured variables as outputs.
Control then becomes decision making on the basis of observations.
When subsequently a controller is regarded as a feedback processor,
one ends up with the feeling that the input/output structure is in
fact an essential feature of control.  Secondly, since, as we have
seen, it is possible to prove that every linear time-invariant
differential system admits a component-wise input/output
representation, one gets the impression that the input/output
framework can be adopted without second thoughts, that nothing is lost
by taking it as a universal starting point. However, this partition in
inputs and outputs may not be possible for smooth nonlinear
differential systems.

The intelligent control paradigm embodied in figure
\ref{intelligentcontrol} has been universally prevalent ever since the
beginning of the subject, from our interpretation of the Watt
regulator, Black's feedback amplifier, and Wiener's cybernetics, to
the ideas underlying modern adaptive and robust control.  It is indeed
a deep and very appealing paradigm, which will undoubtedly gain in
relevance and impact as logic devices become ever more prevalent,
reliable, and inexpensive.  This paradigm has a number of features
which are important for considerations which will follow. Some of
these are:
\begin{enumerate}
\item There is an asymmetry between the plant and the controller: it
  remains apparent what part of the system is the plant and what part
  is the controller. This asymmetry disappears to some extent in the
  closed loop.
\item The intelligent control paradigm tells us to be wary of errors
  in the        measurements.  Thus it is considered as being
  ill-advised to differentiate measurements, presumably, because this
  will lead to noise amplification.
\item The plant and the controller are dynamical systems which can be
  interconnected at any moment in time.  If for one reason or another
  the feedback controller temporarily fails to receive an input
  signal, then the control input can be set to a default value, and
  later on the controller can resume its action.
\end{enumerate}

However, in many practical control problems, for example, the damper
of an automobile, pressure control valves, heat radiation fins, and
similar passive controllers, simply do not function as signal
processors.  The signal-flow-graph interpretation of figure
\ref{intelligentcontrol} is untenable.  In these examples, there is no
asymmetry between plant and controller, there are no sensors, no
actuators, no measurements, and the state of the plant needs to be
prepared before the controller can be put in place.

In this section, we will sketch a broader framework for control. A
framework that incorporates the above examples, and in which the
feedback control appears as a special case.  The solution which we
propose is the following.  We will keep the distinction between the
plant and the controller.  However, we will abandon the intelligent
control signal flow graph as a paradigm for control.  Instead, we will
view {\em interconnection of a controller to a plant} as the central
paradigm for control theory.  However, we by no means claim that the
intelligent control paradigm is without merits.  To the contrary, it
is extremely useful and important. Claiming that the input/output
framework is {\em not always} the suitable framework to approach a
problem does not mean that one claims that it is {\em never} the
suitable framework.  However, a good universal framework for control
should be able to deal both with interconnection, with designing
subsystems, and with intelligent control.  \textit{The behavioral
framework does, the intelligent control framework does not.}

\begin{figure}[hbt]
\centerline{
\resizebox{10cm}{!}{
\input{interconnectcontrol.pstex_t}
}}
\caption{
Control as interconnection
}
\label{interconnectcontrol}
\end{figure}

The nature of control that we would like to
transmit is illustrated in the system configuration
depicted in figure \ref{interconnectcontrol}.  In the top part of the
figure, there are two systems, shown as black-boxes with terminals. It
is through their terminals that systems interact with their
environment.  The black-box imposes relations on the variables that
`live' on its terminals. These relations are formalized by the
behavior of the system in the black-box.  The system to the left in
figure \ref{interconnectcontrol} is called the \textit{plant}, the 
one to the right the
\textit{controller}. The terminals of the plant consist of
\textit{to-be-controlled variables} $w$, and \textit{control
  variables} $c$.  The controller has only terminals with the control
variables $c$.   Before
interconnection, the variables $w$ and $c$ of the plant have to
satisfy the laws imposed by the plant behavior. After
interconnection, the variables $c$ also have to satisfy the laws
imposed by the controller. Thus, after interconnection, the
restrictions imposed on the variables $c$ by the controller will be
transmitted to the variables $w$.  Choosing the black-box to the right
so that the variables $w$ have a desirable behavior in the
interconnected black-box is, in our view, the basic problem of
control. 

This leads to the following mathematical formulation.  The {\it{plant}} and the
{\it{controller}} are both dynamical systems 
$\Sigma_{\rm plant}=(\T,\W \times \C,\B_{\rm plant})$ 
and $\Sigma_{\rm controller}=(\T,\C,\B_{\rm controller})$ where $\W$
denotes the signal space of the to-be-controlled variables, $\C$ denotes the
signal space of control variables, and both systems are assumed to have the
same time axis $\T$.  The interconnection of $\Sigma_{\rm plant}$ and 
$\Sigma_{\rm controller}$ leads to the system 
$\Sigma_{\rm latent}=(\T,\W,\C,\Bfull)$ which is a system with latent
variables ($\C$) and full behavior defined by
\begin{eqnarray*}
 \Bfull=\{(w,c):\T \rightarrow \W \times \C \mid
(w,c) \in \B_{\rm plant} \mbox{ and }  c \in \B_{\rm controller} \} \nn
  \end{eqnarray*}
 The manifest system defined by $\Sigma_{\rm latent}$ is the
 {\it{controlled system}} and is hence defined as
 $\Sigma_{\rm controlled}=(\T,\W,\B_{\rm controlled})$
 with
  
 \begin{multline*}
 \B_{\rm controlled}=\{w:\T \rightarrow \W  \mid
 \mbox{ there exists } c:\T \rightarrow \C  \mbox{ such that} \nn \\
 (w,c) \in \B_{\rm plant} \mbox{ and }  c \in \B_{\rm controller}
\} \nn
 \end{multline*}
 

The notion of a controller put forward by the above view considers
interconnection as the basic idea of control.  The special controllers
that consist of sensor-outputs to actuator-inputs signal processors
emerge as (very important) special cases.  However, our view of
control as the design of suitable subsystems greatly enhances the
applicability of control. It views control as integrated subsystem
design.

A question that arises in this context is the following.  Assume that
$\Sigma_{\rm plant}$ is given.  What systems $\Sigma_{\rm controlled}$
can be obtained by suitably choosing $\Sigma_{\rm controller}$?  This
question can be answered very explicitly, at least for linear
time-invariant differential systems.
 
Consider the plant given by 
$\Sigma_{\rm plant} =(\R, \Rw \times\R^\c, \B_{\rm plant})\in \LL^{\w+\c}$ 
with $\B_{\rm plant}$ described by a system of linear constant
differential equations $R(\der)w=R(\der)c$.  Let $\Sigma_{\rm
controller}=(\R,\R^\c,\B_{\rm controller}) \in \mathfrak{L}^\c$, and
assume that $\B_{\rm controller}$ is similarly described by a system
of linear constant coefficient differential equations $C(\der)c=0$.
Then, by elimination theorem 
$\Sigma_{\rm controlled} =(\R,\Rw,\B_{\rm controlled}) $ has also a 
behavior that is described by a system of linear constant coefficient
differential equations.  It turns out that the behaviors $\B_{\rm
controlled}$ that can be obtained this way can be characterized very
nicely.

Define therefore two behaviors, $\P$ and $\N$; $\P$ is called the
{\it{realizable (plant) behavior}} and $\N$ the {\it{hidden
behavior}}.  They are defined as follows: $\P$ is the manifest
behavior of the plant, with $c$ viewed as a latent variable, i.e.,
\[
\P = \{w : \R \rightarrow \Rw \mid \mbox{ there exists } c: \R\rightarrow 
\Rc \mbox{ such that } (w,c) \in \Bfull \}, 
\]
and $\N$ is defined as 
\[ 
\N = \{ w:\R \rightarrow \Rw \mid (w,0)\in \B_{\rm plant} \}.  
\]

Hence $\N$ is the behavior of the plant variables that are compatible
with the control variables equal to zero.  Thus, whatever be the
controller, only signal in $\P$ will be possible, and the signals in
$\N$ will remain possible.
 
We say that $\B_{\rm controller}$ {\it{implements}} $\B_{\rm
controlled}$ if there exists a controller such that the controlled
behavior after interconnecting the controller with behavior $\B_{\rm
controller}$ to the plant, yields $\B_{\rm controlled}$ as the
controlled behavior.  The controller implementability problem asks
what behaviors $\B_{\rm controlled}$ can be obtained this way.  For
linear time-invariant systems it is possible to prove that $\B_{\rm
controlled}$ is implementable if and only if
\[ 
\N \subseteq \B_{\rm controlled} \subseteq \P. 
\]
Hence the behaviors that are implementable by means of a controller
are precisely those that are wedged in between the hidden behavior
$\N$ and the realizable plant $\P$.  The problem of control can
therefore be reduced to finding such a behavior.  Of course, the issue
of how to construct $\Sigma_{\rm controller}$ (for example, as a
signal processor from the sensor outputs to the actuator inputs) must
be addressed as well, but this can be done.

We believe that the point of view of control that emerges from this,
as designing a subsystem (with feedback control as a special case)
greatly enhances the scope and applicability of control as a
discipline. In this setting, control comes down to sub-system
design. More details on the behavioral approach to dynamical systems
and control can be found in \cite{paradigms, Book, control}.

\section{Conclusions}

In this chapter, we have attempted to give a broad overview of control
and systems theory.

We started by contrasting open loop control with feedback
control. Whereas there are many important and interesting problems of
open loop control in the form of trajectory optimization, feedback
control is in a sense a more central principle of control
engineering. Feedback control leads straight to the study of
input/output systems.

We subsequently asked whether the input/output framework is the most
logical setting for the study of `open' dynamical systems.  We came to
the conclusion that it is not. Most physical systems simply do not
function as signal processors. We then introduced a more cogent
framework, the behavioral approach. We also discussed that this leads
to a view of control as interconnecting subsystems.






\cchapter{From ancient times to the maximum principle\label{fromto}}

\noindent In this chapter we outline the main steps in the evolution
that led to the discovery of general necessary conditions for an
optimum, focusing on the search for the correct {\bem form} of the
conditions.  We will not worry at this stage about the questions of
rigorous mathematical interpretation and proof techniques.  These are,
of course, important issues, and we shall return to them later.

The evolution can roughly be divided into three periods:

\bi
\item[1.] {\bem The era of special problems solved by ad hoc methods,} 
extending from ancient times until the birth of the calculus of
variations in the 17th century; 
\item[2.] {\bem The era of the calculus of variations,} lasting from
the beginning of the 17th century until the 1950s;
\item[3.] {\bem The era of optimal control,} which began in the 1950s
and continues to the present time.
\ei

\noindent Naturally, the division is only approximate since, for example, 
some calculus of variations problems truly belong to optimal control, as we
shall try  to argue later for the case of the brachistochrone.  

\ssection{The era of special problems solved by ad hoc 
methods}\label{subsec-era-spec} Like all
mathematical theories, the study of general curve minimization problems was
preceded by a long period where particular problems were tackled by special
{\bem ad hoc} methods. In some cases, the effort to solve these problems
led to the invention of new methods of greater 
\mbox{generality}.  \mbox{Eventually},
after many special problems had been solved, and the new 
\mbox{methods} developed
to solve them had in turn been applied to other 
\mbox{problems}, it became clear
that there had to exists a systematic theory based on general principles.

However, in their eager search for new tools that would apply 
to ever larger classes of problems, the creators of the calculus 
of variations were guilty of some sloppiness, in that they 
sometimes forgot to 
acknolewdge their indebtedness to many of the original special problems
without which the field might never have been born, 
by incorporating them into their new theories. 

This is easily seen by looking at a representative list of these special 
\mbox{problems}
and asking how well they fit within the framework of the classical
calculus of variations.  Here we will look at six such problems.

\subsection{Finding the shortest path between two points.}\label{subsecfinding}

This is of course the oldest of all curve minimization problems, 
and the fact that the answer is a straight-line segment has been known 
for at least 3,000 years. Yet, a careful analysis---to be carried out 
in \S \ref{subsec-segment}---will show
that this problem does not quite fit the framework of 
the calculus of variations in a direct and natural way.  One can, of
course,
\bi
\item[a.] prove directly that the solution is a segment, without using the
calculus of variations,
\ei
or 
\bi
\item[b.] use some clever transformation to reduce the problem to one where
the Euler-Lagrange equation is applicable, 
\ei
or
\bi
\item[c.] give a logically questionable argument\Ft{that is, cheat}.
\ei
But, as we shall show in \S \ref{subsec-segment}, it is not possible 
to apply the necessary conditions of Euler-Lagrange directly to this problem
without following one of the three strategies listed above.

\subsection{The classical isoperimetric problem.}

The oldest curve optimization problem after the shortest-path problem
is the {\em isoperimetric problem} (henceforth abbreviated 
as ``IP''), also known as {\em Queen Dido's
problem}, inspired by the \mbox{mythical}
story\Ft{According to Virgil,
``when Dido landed in North Africa, she persuaded the local chief to sell
her as much land as an oxhide could contain. She then cut a hide into very
narrow strips, and joined them to make a continuous thread more than
two-and-a-half miles long. She was then able to enclose between the thread
and the sea the land on which Carthage was built'' (Pars \cite{pars}).}
told by Virgil (70-19 BCE) in the
{\em Aeneid} about the 
\mbox{foundation} of Carthage ({\em ca.} 850 BCE).

\def\hereproof{{\em Proof of the equivalence}: Suppose a closed curve 
$\gamma$ solves Problem I. This means that $\gamma$ has length $L$, 
encloses an area $A$, and every closed curve $\xi$ of length 
$L$ encloses an area $\mathcal{A}(\xi)$ such that
$\mathcal{A}(\xi)\le A$. We claim
that $\gamma$ solves Problem II, that is, 
if $\xi$ is any closed curve that encloses a region with 
area $A$, then the length $\mathcal{L}(\xi)$ of $\xi$ 
satisfies $\mathcal{L}(\xi)\ge L$. Suppose this conclusion was not true.
Let $\xi$ be a closed curve that encloses a region with 
area $A$ and is such that $\mathcal{L}(\xi)<L$. Let 
$\kappa=\frac{L}{\mathcal{L}(\xi)}$. Let $\tau$ be the transformation
$\rr^2\ni(x,y)\mapsto(\kappa x,\kappa y)\in\rr^2$. 
Then $\tau(\xi)$ has length $L$, and encloses an area 
$\kappa^2\mathcal{A}(\xi)=\kappa^2A>A$. This contradicts the fact
that $\gamma$ was a solution to Problem I. So $\gamma$ solves 
Problem II. The converse implication is easily proved
by a similar argument.}



The question is 

\begin{itemize}
\item[(I)] {\large {\em To find the closed plane curve of a given length that
encloses the largest possible area}}.  
\end{itemize}
It is quite easy to see that this is equivalent\Ft{\hereproof} to
the problem
\begin{itemize}
\item[(II)] {\large {\em To determine the shortest of all 
closed plane curves that enclose a given area}}.
\end{itemize}
\label{soliscircle}
It is a fairly obvious guess that the solution must be a circle.
(In modern terms, it is almost evident that the
solution---modulo translations---has to be rotationally symmetric, since
the question, as stated, treats all directions in the plane equally.  This,
however, is not a rigorous argument, since it might conceivably happen that
there is ``symmetry breaking,'' so that the solution modulo translations is
not unique, and each solution individually is not rotationally symmetric,
even though the {\it set} of all solutions is\Ft{This is exactly what 
happens, for example, for
the problem of {\it minimizing} rather than maximizing an enclosed area. In
this case, the set $S$ of solutions is very large, and consists of {\sl
all} loops of length $P$ that enclose a zero area. (For example, $S$
contains all ``degenerate rectangles'' with two opposite sides of length
$P/2$ and the other two of length zero.) None of these solutions is
rotationally symmetric, but of course the family of all the solutions is
symmetric.}.  To transform the rotational symmetry argument into a 
fully rigorous
proof that the solution of the IP is a circle, one would have to establish,
for example, uniqueness of the solution modulo translations.)

The Greeks knew that the solution was  be the circle, and realized
that this needed proof. {\bem Zenodorus}\Ft{For more information on 
Zenodorus's work, cf. Heath \cite{heath}, pp.~382-383.}
\mbox{({\em ca.} 200-140 BCE)} 
showed that
\begin{itemize}
\item[(Z1)] among all
polygons with a given perimeter $P$ and given number $N$ of sides, the
regular polgyon has the largest area, 
\end{itemize}
and
\begin{itemize}
\item[(Z2)] the area enclosed by a circle is larger than that enclosed
any regular polygon of the same \mbox{perimeter}. 
\end{itemize}
\label{zenodorus}

\noindent These observations essentially\Ft{A 20th century mathematician might 
want to add that if
$\xi$ is an arbitrary simple rectifiable closed curve, then $\xi$ can
be approximated by polygons. Supplemented by this observation, the two
results of Zenodorus add up to a complete solution.  Naturally, the
concept of an ``arbitrary rectifiable curve'' only makes sense within
the context of modern analysis.} solve the isoperimetric
problem and \mbox{establish} that the solution is the circle, 
although it took until the 19th century for this
to be proved in a way that meets our contemporary standards of rigor.

The isoperimetric problem will be discussed in
\S\ref{subsec-isoper}. Here, we will just observe that this is one of the
problems for which the classical 
\mbox{calculus} of variations---as
presented, for example, in Euler's 1744 book---works
\mbox{relatively} well,
provided, of course, that one broadens the setting to include
\mbox{problems} with integral constraints. (According to Struik
\cite{struik}, p.~399, it is with
\mbox{Euler's} 1744 \textit{Methodus
\mbox{inveniendi} $\ldots$} that ``the isoperimetric problem enter[ed] into
the calculus.'')

We remark, however, that even Euler, in the title of his 1744 book 
\mbox{(cf. \S \ref{subsec-euler} in} page \pageref{subsec-euler} below), 
felt that he had
to distinguish between {\bem two separate
\mbox{cases}}, \mbox{namely}, the
problems of finding curves ``with a maximum or minimum property'' 
and the ``isoperimetric problems.''  This should be constrasted
with the fact that in optimal control the distinction
between ``standard problems'' and ``problems with integral constraints''
disappears, as was shown in \S \ref{isoper-are-standard}.

\subsection{Light reflection.} 
Hero (or Heron) of Alexandria\Ft{Exact dates unknown. For a long time,
historians were divided, some of them believing that he had lived 
around 150 BCE while others thought he had lived on 250 CE. 
O.~Neugebauer in 1938 identified a description by Hero of an eclipse
as corresponding to an eclipse that took place in 62 CE. Based on this, it
is now believed that Hero was probably born in 10 CE and died 
in 75 CE.   Some historians attribute
his optics work to a ``Hero the Younger,'' who may have lived in the
7th or 8th century CE.}  showed in his {\it Catoptrics} that when a
light ray emitted by an object is reflected by a mirror
it follows a path from the object to the eye which is the shortest of all
possible such paths. Once again, this fact can easily be established by a
direct geometric argument: suppose a path $\pi$ goes from point $C$ on one
side  of the mirror $MM$ to a point $D_*$ on the mirror and then to a point 
$E$ on the same side as $C$. Suppose, in addition, that $\pi$ is the
shortest of all such paths going from $C$ to $E$. Then the parts 
$\pi_-$, $\pi_+$ of $\pi$ that go from $C$ to $D_*$ and from $D_*$ to $E$,
respectively,  must obviously be straight-line segments.  Let $\tilde E$ 
be the point on the other side of $MM$ such that the segment $\sigma$ from 
$E$ to $\tilde E$ is perpendicular to $MM$ and  intersects 
$MM$ at the midpoint of $\sigma$. (That is, $\tilde E$ is the ``mirror
image'' of $E$, or the result of ``reflecting'' $E$ with respect to 
$MM$.) For any choice of the point $D$ on $MM$, let $\pi_D$ be the path
consisting of the segment from $C$ to $D$ 
followed by the segment from $D$ to $E$. Then we can associate 
to $\pi_D$ a path $\tilde\pi_D$ going from $C$ to $\tilde E$, obtained from
$\pi_D$ by substituting the segment $D\tilde E$ for the segment $DE$. 
The new path has exactly the same length as $\pi_D$. Since the shortest
path joing two points is the segment, we see that 
the length of $\pi_D$ is minimized when $D$ is chosen such that 
$\tilde\pi_D$ is a segment. This will happen if and only if 
$$
\theta_i=\theta_r\,,
$$
where $\theta_i$ is the {\em angle of incidence}   
(that is, the angle between the segment $CD_*$ and the line $L$ through
$D_*$ perpendicular to $MM$), and $\theta_r$ is 
the {\em angle of reflection}, (that is, the angle between 
$CD_*$ and $L$).  This is the well know law of reflection. 

\subsection{Light refraction.}  Pierre de Fermat
(1601-1665) formulated the general principle that {\sl light rays
follow the fastest---i.~e., minimum-time---paths}, and showed how this
explained not only Hero's observation about reflection but also
\snell{}' law of refraction.  \snell's law is about the light rays
in plane medium where the speed of light has a constant value 
$c_+$ on one side $S_+$ of a line $L$ and a different constant value $c_-$
on the other side $S_-$. To derive the law one must find
the path joining two points $C_+$, $C_+$ lying on $S_+$ and $S_-$,
respectively, that minimizes travel time. This can easily be done using
calculus, and was done, for example, by Leibniz in his 1684 {\bem Nova
methodus} 
paper (cf. Scholium \ref{schol-snell}, in page \pageref{schol-snell}). 

In modern terms, if we choose Cartesian coordinates $x$, $y$ such that 
$L$ is the $x$ axis and $C_+$ lies on the $y$ axis and above $L$, 
we can let $(0,y_+)$ be the coordinates of $C_+$ and 
$(x_-,y_-)$ be the coordinates of $C_-$.  It is obvious that the minimizing
path will consist of a segment from $C_+$ to a point $\bar C$ on $L$,
followed by the segment from $\bar C$ to $C_-$.  Let $(\bar x,0)$ 
be the coordinates of $C_-$. Then the travel time is
$$
T=\frac{1}{c_+}\sqrt{\bar x^2+y_+^2}+\frac{1}{c_-}
\sqrt{(x_--\bar x)^2+y_-^2}\,.
$$
To minimize $T$ as a function of $\bar x$, we must have 
$$
\frac{\partial T}{\partial\bar x}=0\,,
$$
which yields
$$
\frac{\bar x}{c_+\sqrt{\bar x^2+y_+^2}}+
\frac{\bar x-x_-}{c_-\sqrt{(x_- -\bar x)^2+y_-^2}}=0\,.
$$

%\bigskip
\vbox{
\[
\epsfbox{reflection.eps}
\]
\figurecap{Reflection of light\label{reflection-fig}}
}


Therefore
\begin{equation}\label{snelleq-a}
\frac{\bar x}{c_+\sqrt{\bar x^2+y_+^2}}=\frac{x_--\bar x}
{c_-\sqrt{(x_- -\bar x)^2+y_-^2}}=0\,.
\end{equation}
Now, the quotient $\frac{\bar x}{\sqrt{\bar x^2+y_+^2}}$
is the sine of the ``angle of incidence,'' that is, the angle 
$\theta_i$ between the line $C_+\bar C$ and a line perpendicular to $L$. 
Similarly, 
$\frac{\bar x}{\sqrt{\bar x^2+y_+^2}}=\sin{\theta_r}$, where
$\theta_r$ is the ``angle of refraction,'' that is, the angle 
between the line $\bar CC_-$ and a line perpendicular to $L$. 
Hence (\ref{snelleq-a}) says that 
\begin{equation}\label{snelleq-b}
\frac{\sin{\theta_i}}{c_+}=\frac{\sin{\theta_i}}{c_-}\,.
\end{equation}

\subsection{The catenary and the brachistochrone.}  
These problems were solved by Johann Bernoulli in 1691 and
1696, respectively, and will be discussed in Example III of 
\S \ref{subsec-catenary} and in Chapter \ref{controbrach}.

\def\oneuler{Paul Euler, Leonhard's father, had studied theology at the
University of Basel and also attended Jakob Bernoulli's lectures there. 
Paul and Johann Bernoulli lived in Jakob's house while they were students
in Basel. Paul then became a Protestant minister, but he was also
interested in mathematics and taught some to his son. Paul wanted Leonhard
to become a minister, and sent him to the University of Basel for that
reason. At the University, Leonhard was introduced to Johan Bernoulli, and
asked him to give him private lessons. Johann refused, but recommended some
readings, and allowed Leonhard to visit him on Sundays to ask questions.
Thus Johann soon discovered that this young student was a mathematical
prodigy. Leonhard completed his Master's degree in philosophy in 1723,
and began the study of theology, but he really wanted to pursue
mathematics, and Johann was able to persuade his friend Paul to give his
consent for this change. Euler completed his studies in Basel in 1726, 
and in November of that year he accepted a position in St. Petersburg 
that had become vacant in July, when Nicolaus(II)  Bernoulli died.
Euler arrived in St. Petersburg in May 1727.}


\ssection{Euler}\label{subsec-euler}  {\bem  Leonhard Euler} (1707-1793) 
entered the University of Basel in 1720 at the age of
13\Ft{\label{euler-foot}\oneuler}. There he studied the methods 
developed by the Bernoullis 
for the study of isoperimetric problems, but soon became convinced that
there was a need for a general theory, that would go beyond the solutions
of specific problems. He began to develop his approach in 1731, 
and by 1740 he had achieved his goal and come up with a
general theory. He published his results in 1744, in a book
(Euler \cite{euler}) entitled \label{pageinveniendi}

\bqrm \textit{Methodus inveniendi maximi minime propriate gaudentes sive
\mbox{solutio} problematis isoperimetrici  latissimo sensu accepti}
(A method for discovering curved lines having a maximum or 
\mbox{minimum}
property or the solution of the isoperimetric problem taken in its widest
sense). 
\eqrm

\medbreak\noindent There he gave a
general procedure for writing down what became known as {\it Euler's
equation}.

\gnu{Must discuss precise relation between this and Euler-Lagrange}


The words ``calculus of variations'' do not yet appear in 
Euler's \textit{\mbox{Methodus}
\mbox{inveniendi}.} They were coined by him later,
in 1760, as a name for Lagrange's method that Euler had decided to adopt 
in lieu of his own discretization approach, and appeared in print in 1766. 
\label{needbiolege}

\ssection{Lagrange} In H. Goldstine's words
(\cite{goldstine}, p.~110.):  

\medbreak
\bqrm
On 12 August 1755 one Ludovico de la Grange Tournier,
a 19-year-old from Turin, 
wrote Euler a brief letter to which was attached an appendix containing 
mathematical details of a very beautiful and revolutionary idea. He saw how
to eliminate from Euler's methods the tedium and need for geometrical insight
and to reduce the entire process to a quite analytic machine or apparatus, 
which could turn out the necessary condition of Euler and more,  almost 
automatically.   This basic idea of Lagrange ushered in a new epoch in the 
calculus  of variations. Indeed, after seeing Lagrange's work, Euler dropped 
his own method, espoused that of Lagrange, and  renamed the subject the
{\it calculus of variations}.  

In the summary to his first paper using variations, Euler says
``Even though the author of this [Euler] had meditated a long time
and had revealed to friends his desire yet the glory of first discovery was
reserved to the very penetrating geometer of Turin LA GRANGE, who having used
analysis alone, has clearly attained the same solution which the author had
deduced by geometrical considerations.'' 
\eqrm


\medbreak\noindent
{\bem Joseph-Louis Lagrange} (1736-1813) derived the necessary 
condition for a curve $t\to\xi_*(t)$ to be a solution
of the problem (\ref{standard}),
known today as the ``Euler-Lagrange equation.'' This condition
says\Ft{We use the notations that are common today, not Lagrange's.
The symbol $\partial$ for partial derivative was first used by 
Legendre in 1786.} that the identity
\begin{equation}\label{eulagfirst}
\frac{d\,\,}{dt}\Biggl(\frac{\partial L}{\partial\dot q}\Biggr)=
\frac{\partial L}{\partial q}
\end{equation}
must be satisfied along the curve $\xi_*$.

Equation (\ref{eulagfirst}) makes perfect sense and is a necessary condition
for optimality for a vector-valued variable $q$ as well as for a scalar
one.  It can be written as a system: 
\begin{equation}\label{eulagscalar}
\frac{d\,\,}{dt}\Biggl(\frac{\partial L}{\partial\dot q^i}\Biggr)
= \frac{\partial
L}{\partial q^i}\,,\,\,\,\,\,\,\,i=1,\ldots,n\,.
\end{equation}
Alternatively, we can regard Equation (\ref{eulagfirst}) as a vector
identity, in which 
$$
q=(q^1,\ldots,q^n)
$$ 
denotes an $n$-dimensional vector, and the expressions
$$\frac{\partial L}{\partial q}\,,\qquad\frac{\partial L}{\partial\dot q}
$$ 
stand for the $n$-tuples 
$$
\Big(\frac{\partial L}{\partial q^1},\ldots,
\frac{\partial L}{\partial q^n}\Big)\,,\qquad
\Big(\frac{\partial L}{\partial\dot q^1},\ldots,
\frac{\partial L}{\partial\dot q^n}\Big)\,.
$$
A modern mathematician might be troubled by the use of $\dot q$ both as an
``independent variable'' and as a function of time evaluated along a
trajectory, and might prefer to write (\ref{eulagfirst}) as
\begin{equation}\label{eulagvectoru} 
\fbox{\large{$
\frac{d\,\,}{dt}\!\Biggl[\frac{\partial L}{\partial
u}\Big(\xi(t),\dot\xi(t),t\Big)\Biggr]\!\!= \frac{\partial L}{\partial
q}\Big(\xi(t),\dot\xi(t),t\Big)\,,\,\,i\!=\!1,\ldots,n\,,$}} 
\end{equation} 
where the Lagrangian $L(q,u,t)$ is a function on $\rr^{2n+1}$, i.e.,  a
function of $q\!\!\in\!\!\rr^n$, $u\!\!\in\!\!\rr^n$, $t\!\!\in\!\!\rr$.
This makes it clear that to compute 
the left-hand side of (\ref{eulagfirst}) one first evaluates
$\frac{\partial L}{\partial\dot q}$ ``treating $\dot q$ as an independent
variable,'' then plugs in $\xi(t)$ and $\dot\xi(t)$ for $q$, $\dot q$, and
finally differentiates with respect to $t$.
\label{needbiolegendre}

\ssection{Legendre}  The Euler-Lagrange system 
(\ref{eulagfirst})---or (\ref{eulagvectoru})---was derived by looking at
the ``first variation'' $\delta I$ of the cost functional $I$, and requiring 
that $\delta I=0$, as we shall see in Chapter \ref{frompri}.
So in fact (\ref{eulagfirst}) is a 
necessary condition for the {\em stationarity} of $I$. 

The next natural step was to look at the second variation of $I$,
and this was done in 1786 by {\bem Adrien-Marie 
Legendre} (1752-1833), who found an 
additional necessary  condition  for a minimum.
His condition, derived by him for the scalar case, is
\begin{equation}\label{legendre}
\fbox{\large{$
\frac{\partial^2 L}{\partial\dot q^2\,}\Big(\xi_*(t),\dot\xi_*(t),t\Big)\ge
0\,\,\,\,($}\small{${\rm i.e.,}$}\large{$\,\,\,
\frac{\partial^2 L}{\partial u^2}\Big(\xi(t),\dot\xi(t),t\Big)\ge 0)\,.$}}
\end{equation}
With an appropriate reinterpretation, Legendre's condition
(\ref{legendre}) is also necessary 
in the vector case: all we have to do is read 
(\ref{legendre}) as asserting that the Hessian matrix 
$\Bigl\{\frac{\partial^2 L\,\,\,\,}{\partial u^i\partial u^j}
\Big(\xi_*(t),\dot\xi_*(t),t\Big)\Bigr\}_{1\le i,j\le n}$ 
has to be nonnegative definite. 
\label{needbio}


\ssection{Hamilton misses an opportunity} At this point, we have
reached the first and most critical fork in the road, involving the
work of {\bem William Rowan Hamilton} (1805-1865), published in 1834 and
1835. In a sense, the issue at stake will seem rather trivial, just a
matter of rewriting the Euler-Lagrange system in a different
formalism.  However, sometimes formalisms can make a tremendous
difference.  To understand what happened and what could have happened
but did not, let us see if we can make sense of the two necessary
conditions for a minimum that have been presented so far, namely, the
Euler-Lagrange equation (\ref{eulagfirst}) and the Legendre condition
(\ref{legendre}), and look for a single simple property that would
imply both conditions.

The Legendre condition is clearly the second-order necessary condition
for a minimum of a function, namely, $L(\xi_*(t),u,t)$ regarded as a
function of $u$.  But (\ref{eulagfirst}) does not look at all like the
first-order condition for a minimum of that same function.  It is
natural to ask whether there might be a way to relate the two
conditions. Is it possible that both could be expressed as necessary
conditions for a minimum of one and the same function?  The answer, as
we now show, is ``yes,'' and understanding how this function can be
defined leads straight to optimal control theory, the maximum
principle, and far-reaching generalizations of the classical theory.


First, let us look at an alternative way of writing (\ref{eulagfirst}).
Define the ``momentum'' $\pi_{\xi,L}$ corresponding to $L$
along a curve $\xi$ by letting\Ft{In modern terminology,
$\pi_{\xi,L}$ is a {\em field of covectors} along $\xi$.}
\begin{equation}\label{defofp}
\pi_{\xi,L}(t)\equaldef\frac{\partial L}{\partial u}
\Big(\xi(t),\dot\xi(t),t\Big)\,.
\end{equation}

Define a function $H(q,u,p,t)$ of the three sets of variables $q,u,p$,
and of $t\in\rr$, by letting\Ft{Geometrically,
$H$ is defined on the fiber product of the tangent and cotangent
bundles of the configuration space $Q$.}

\begin{equation}\label{defofh}
\fbox{\large{$H(q,u,p,t)= \langle p,u\rangle-L(q,u,t)\,.$}}
\end{equation}
Now suppose that a curve $t\mapsto\xi_*(t)$ is a solution of 
(\ref{eulagfirst}). 

It is then clear that
$$
\frac{\partial H}{\partial p}=u\,.
$$
Therefore, along our curve $\xi_*$, the equality 
\begin{equation}\label{hamileqa}
\frac{d\xi_*}{dt}(t)=\frac{\partial H}{\partial p}
\Big(\xi_*(t),\dot\xi_*(t),\pi(t),t\Big)
\end{equation}
holds, if we let $\pi(t)=\pi_{\xi_*,L}(t)$.  Also,
$$
\frac{\partial H}{\partial q}=-\frac{\partial L}{\partial q}\,.
$$
It follows that (\ref{eulagvectoru}) says that
\begin{equation}\label{hamileqb}
\frac{dp}{dt}(t)=-\frac{\partial H}{\partial q}\Big(\xi_*(t),
\dot\xi_*(t),\pi(t),t\Big)\,. 
\end{equation}
Finally,
$$
\frac{\partial H}{\partial u}=p-\frac{\partial L}{\partial u}\,,
$$
so (\ref{defofp}) says:
\begin{equation}\label{hamileqc}
\frac{\partial H}{\partial u}\Big(\xi_*(t),\dot\xi_*(t),\pi(t),t\Big)=0\,.
\end{equation}
The system of equations (\ref{hamileqa}), (\ref{hamileqb}), 
(\ref{hamileqc}), usually written more concisely as
\begin{equation}\label{hamileqsys}
\fbox{\Large{$
\frac{dq}{dt}=\frac{\partial H}{\partial p}\,,
\,\,\,\,\,\,\,\,\,\frac{dp}{dt}=-\frac{\partial H}{\partial q}\,,
\,\,\,\,\,\,\,\,\,\frac{\partial H}{\partial u}=0\,,$}}
\end{equation}
is {\bem exactly equivalent} to (\ref{eulagfirst}), provided that $H$ is
defined as in (\ref{defofh}).    

\def\classical{(\ref{hamileqsysham})-(\ref{hamileqsyshamnewhham})-(\ref{pimpl})}

\medbreak\noindent We will call the function $H$ the ``control
Hamiltonian,'' and refer to 
(\ref{hamileqsys}) as the {\bem control Hamiltonian form} of the
Euler-Lagrange equations.  In our view, Formula (\ref{defofh}) is the
definition that Hamilton should have given for the Hamiltonian, and
Equations (\ref{hamileqsys}) are {\em ``Hamilton's equations as he should
have written them.''}

What Hamilton actually wrote was (in our notation, not his)
\begin{equation}\label{hamileqsysham}
\frac{dq}{dt}=\frac{\partial\hham}{\partial p}\,,
\,\,\,\,\frac{dp}{dt}=-\frac{\partial\hham}{\partial q}\,,
\end{equation}
where $\hham(q,p,t)$ is a function of $p,q$ and $t$ alone, defined
by the formula 
\begin{equation}\label{hamileqsyshamnewhham}
\hham(q,p,t)=\langle p,\dot q\rangle-L(q,\dot q,t)\,,
\end{equation}
which resembles (\ref{defofh}) but is not at all the same. The 
difference is that in Hamilton's definition $\dot q$ is supposed to be
treated {\sl not} as an independent variable, but as a function 
of  $q,p,t$,   defined implicitly by the equation
\begin{equation}\label{pimpl}
p=\frac{\partial L}{\partial\dot q}(q,\dot q,t)\,.
\end{equation}
It is easy to see that

\medbreak
\begin{center}
\fbox{\fbox{\begin{minipage}{10.5cm}
{\bf If the map $(q,\dot q,t)\mapsto{}(q,p,t)$ defined
by (\ref{pimpl}) can be inverted,} {\em i.e., if we can ``solve (\ref{pimpl})
for $\dot q$ as a function of $q,p,t$,''  then (\ref{hamileqsysham})
is equivalent to (\ref{hamileqsys}).}   
\end{minipage}}}\end{center}

\medbreak\noindent Indeed, it is clear that
$$
\hham(q,p,t)=H(q,u(q,p,t),t)\,,
$$
where $u=u(q,p,t)$ satisfies  $p=\frac{\partial L}{\partial u}(q,u,t)$. So
\begin{equation}
\frac{\partial\hham}{\partial q}=\frac{\partial H}{\partial q}
+\frac{\partial H}{\partial u}\cdot \frac{\partial u}{\partial q}\,.
\end{equation}
Since 
$$
\frac{\partial H}{\partial u}(q,u,t)=0\quad{\rm if}\quad u=u(q,p,t)\,,
$$
we see that 
$$
\frac{\partial\hham}{\partial q}=\frac{\partial H}{\partial q}
$$
along solutions of (\ref{hamileqsys}), and then 
the first equation of (\ref{hamileqsysham}) holds as well.  Similarly, 
the second equation of (\ref{hamileqsysham}) also holds.  The converse
is also easily proved.

\ssection{The ``control Hamiltonian'' versus the classical Hamiltonian}
It should be clear from the above discussion that the Hamiltonian
reformulation of the Euler-Lagrange equations in terms of the ``control
Hamiltonian'' is at least as natural as the classical version
\classical, and perhaps even
simpler.  Moreover, the control formulation has at least one obvious
advantage, namely,

\medbreak 

\begin{center}
\fbox{\fbox{\begin{minipage}{10.5cm}
{\bf (A1)} {\em the control version of the Hamilton equations is
{\bem exactly equivalent} to the Euler-Lagrange system under
completely general conditions, whereas the classical version
\classical{} only makes sense when the transformation (\ref{pimpl})
can be inverted, at least locally, to solve for $\dot q$ as a function
of $q,p,t$.}
\end{minipage}}}\end{center}

\medbreak\noindent 
We now show that (A1) is not the only advantage of the control view 
over the classical one. To see this, we must take another look at Legendre's
condition (\ref{legendre}).   Since $H(q,u,p,t)$ is equal to 
$-L(q,u,t)$ plus a linear function of $u$, (\ref{legendre}) is completely
equivalent to 
\begin{equation}\label{legendreham}
\frac{\partial^2 H}{\partial u^2}\Big(\xi_*(t),\dot\xi_*(t),
\pi(t),t\Big)\le 0\,,
\,\,\,\,\,\,\,\,{\rm i.e.,\,\,\,\,\,\,\,\,}\frac{\partial^2 H}{\partial\dot q^2\,}\Big(\xi_*(t),\dot\xi_*(t),\pi(t),t\Big)\le 0\,.
\end{equation}
Now let us write (\ref{legendreham}) side by side with the third
equation of  
(\ref{hamileqsys}):
\begin{equation}\label{legendrehamtog}
\fbox{\fbox{\Large {$
\frac{\partial H}{\partial u}=0\,\,\,\,\,\,\,\,$}\small{${\rm and}
$}\Large{$\,\,\,\,\,\,\,\,\frac{\partial^2 H}{\partial u^2}\le 0$}}}\,\,\,,
\end{equation}
and examine the result for a few seconds.

These equations unmistakably suggest something!   They show that 

\medbreak
\begin{center}
\fbox{\begin{minipage}{10.5cm}
\noindent{\bem the Euler-Lagrange and Legendre conditions have to do 
with critical points---with  respect to the velocity variable 
$u$---of a function $H$.  Precisely, 
Euler-Lagrange says that the gradient  $\nabla_uH$ of 
this function has to vanish, and then Legendre adds to this the extra 
requirement that the second 
$u$-derivative of $H$---i.e., the Hessian matrix 
$\{\frac{\partial^2 H}{\partial u_i\partial u_j}\}_{1\le i,j\le n}$---must 
be nonpositive definite.}
\end{minipage}}\end{center}

\medbreak\noindent The message should be clear:  {\bem what must
be happening here is that $H$ attains a \mbox{maximum} as a function of $u$.}

We state this formally as a  conjecture.

\medbreak
\begin{center}
\fbox{\fbox{\begin{minipage}{10.5cm}
\noindent{\bem CONJECTURE M1:}\ {\em Besides 
(\ref{hamileqsys}) (or the equivalent form (\ref{eulagfirst})), 
an additional necessary condition for optimality 
of an arc $\xi_*$ is that $H(\xi_*(t),u,\pi(t),t)$, as a function of $u$, 
have a maximum at $u=\dot\xi_*(t)$ for each t.} 
\end{minipage}}}\end{center} 

\medbreak \noindent We contend  that

\bi
\item[1.] Conjecture M1 is an extremely natural consequence of 
rewriting Hamilton's equations ``as they should originally have 
been written,'' in terms of the ``control Hamiltonian.''

\medbreak
\item[2.] It is reasonable to guess that, if 
``Hamilton's equations'' had been written from the veru beginning
in this alternative way, then Hamilton
himself, or some other nineteenth century mathematician, would have written
(\ref{legendrehamtog}) and been led by it to Conjecture M1. 

\medbreak
\item[3.] On the other hand, it is only by using the Hamiltonian of 
(\ref{defofh}), as opposed to Hamilton's own form of the Hamiltonian, 
that one
can see that  the Euler-Lagrange and Legendre condition really are 
first- and second-order conditions about critical points 
{\bf of the same function.}   This function
cannot be $L$ itself, because the first order conditions do {\it not} say
that $\frac{\partial L}{\partial u}=0$. Nor can it be the classical
Hamiltonian $\hham$, which isn't even a function of $u$. {\em Only the use
of the ``control'' Hamiltonian leads naturally to Conjecture M1.}
\ei

\begin{remark} {\rm 
Assuming the validity of the invertibility condition 
needed to define the velocity as a function of position, momentum
and time, it is possible to rewrite the Legendre condition
in terms of the classical Hamiltonian $\hham$. Indeed, the Legendre
condition is equivalent, in to his case, to the statement
that 
\begin{equation}\label{legen-in-hamil}
{\rm the\quad matrix\quad}\Big\{\frac{\partial^2\hham}{\partial p_i
\partial p_j}(\xi_*(t),\pi_{\xi_*,L}(t),t)\Big\}_{1\le i,j\le n}
\quad{\rm is\quad positive\quad definite.}
\end{equation}
(The proof of (\ref{legen-in-hamil}) is essentially contained in our
discussion of Carath\'eodory's Hamiltonian formulation of Weierstrass'
condition, cf. \S \ref{subsec-carat}. Indeed, we show there that
\begin{equation}\label{elsp1}
\frac{\partial\hham}{\partial p_i}(q,p,t)=\varphi_i(q,p,t)\,.
\end{equation}
Moreover, 
\begin{equation}\label{elsp2}
\sum_{\ell=1}^n
\frac{\partial\chi_j}{\partial \dot q_\ell}(q,\varphi(q,p,t),t)
\frac{\partial\varphi_\ell}{\partial p_i}(q,p,t)=\delta_{j,i}\,,
\end{equation}
since $(q,\dot q,t)\mapsto(q,\chi(q,\dot q,t),t)$ and 
$(q,p,t)\mapsto(q,\varphi(q,p,t),t)$ are inverse maps.  But
\begin{equation}\label{elsp3}
\chi_j(q,\dot q,t)=\frac{\partial L}{\partial \dot q_j}(q,\dot q,t)\,.
\end{equation}
It follows from (\ref{elsp1}), (\ref{elsp2}) and (\ref{elsp3}) that
\begin{equation}\label{elsp4}
\sum_{\ell=1}^n
\frac{\partial^2 L}{\partial\dot q_\ell\partial\dot q_j}
(q,\varphi(q,p,t),t)
\frac{\partial^2\hham}{\partial p_i\partial p_\ell}(q,p,t)=\delta_{j,i}\,.
\end{equation}
So the matrix 
$$
M=\Big\{\frac{\partial^2\hham}{\partial p_i\partial p_\ell}(\xi_*(t),
\pi_{\xi_*,L}(t),t)\Big\}_{1\le i,j\le n}
$$
is the inverse of the Hessian matrix $N$ of $L$ with respect to 
the velocities. The Legendre condition says that 
$N$ is nonnegative definite, and the invertibility hypothesis 
implies that $N$ is nonsingular. Hence $N$ is strictly 
positive definite, and $M$ is positive definite as well.)

Notice, however, that this does not solve the problem of expressing
both the first-order condition and the Legendre condition
as conditions on a critical point of the same function.
Indeed, (\ref{elsp1}) implies that
$$
\nabla_p\hham(\xi_*(t),\pi_{\xi_*,L}(t),t)=\dot\xi_*(t)\,,
$$
so $\pi_{\xi_*,L}(t)$ is {\em not} in general a critical point of the function
$p\mapsto\hham(\xi_*(t),p,t)$.}\proofend
\end{remark}



\medbreak\noindent 



It will turn out that 

\medbreak
\begin{center}
\begin{minipage}{10.5cm}
\noindent{\bf Conjecture M1 is true. Moreover, once its truth is known
then vast generalizations are possible.}
\end{minipage}\end{center}

\medbreak\noindent But before we get there, we must
move to the next three chapters in our tale, and discuss

\bi
\item[(i)] Hamilton's idea of the ``principal function,'' 

\medbreak

\item[(ii)] the work of Weierstrass, who essentially discovered and
proved Conjecture M1, but did it using a language that obscured the
simplicity of his own result,and for that reason missed some of its
most profound implications,
\ei

\medbreak\noindent and

\medbreak

\bi
\item[(iii)] Carath\'eodory's attempt to make sense
of the Weiesrtrass discovery in Hamiltonian terms, 
and how this effort led nowhere because Carath\'eodory
used the wrong Hamiltonian.
\ei




\ssection{J.~L.~Synge on Hamilton's principal function and the
100-year eclipse of a great idea}
At this point, we digress to discuss
another of Hamilton's ideas, namely, the ``principal'' or
``characteristic'' function.  This idea was not truly appreciated for
more than century, until optimal control theory emerged to provide the
right setting that would do it justice.  So we will only describe it
here in broad outline, following the 1944 article \cite{synge} by
J.~L.~Synge.  For a discussion of what this idea becomes in the
optimal control setting, we will have to wait until Chapter \ref{valuefun}.
For the specific case of the brachistochrone, we will show in
\ref{brachprinc} that {\em the search for Hamilton's principal
function leads directly to the discovery of the differential-geometric 
structure underlying the problem}. 

Synge wrote that \label{synge-quot}

\bqrm 
To Hamilton, optics and dynamics were merely two aspects of the calculus of
variations. He was not interested in experiments. To him, optics was the
investigation of the mathematical properties of of curves giving stationary
values to an integral of the type 
\begin{equation}\label{syngeint}
V=\int v\Big(x,y,z,\frac{dx}{du},\frac{dy}{du},\frac{dz}{du}\Big)du\,.
\end{equation}
Hamilton was, in fact, a great contributor---probably the gratest single
contributor of all time---to the calculus of variations. 

Consider two points, $A'$ with coordinates $x',y',z'$ and $A$ with
coordinates $x,y,z$. Consider all possible curves connecting 
$A'$ and $A$.  To each curve corresponds a value of the above 
integral. [{\em Note by HJS}$\,$\&$\,${\em JCW: Hamilton 
was working with Lagrangians
that are homogeneous of degree one with respect to the velocities.
In that case, the choice of the interval of definition of the curves
is immaterial, since every curve can be reparametrized to be defined
on [0,1] without changing the value of the cost integral.}] 
Now compare these
values. Is there one curve that gives a smaller value to the integral than
all others? It would seem that there must be, but it would be a rash
conclusion. Suffice it to say that ``in general'' there is a curve giving a
smallest value to the integral.  Then such a curve is a {\sl ray} in 
optics. In the language of the calculus of variations, it is an extremal. 

Now here is Hamilton's great central idea. {\sl Regard the minimum (or
stationary) value of the integral as a function of the six coordinates
$x',y',z',x,y,z$ of $A'$ and $A$.}  This function is Hamilton's
characteristic or principal function.  There is nothing hard to understand
about that. What is hard to follow is Hamilton's plan to develop all the
properties of the extremals from the characteristic function. In fact, is
is so hard that few mathematicians in the ensuing century have given
serious consideration to this plan. 
\eqrm

\noindent Synge goes on to say that 

\bqrm
Hamilton belonged to the heroic age of mathematics,
\eqrm

\noindent by which he means, among other things, that while 

\bqrm
the modern mathematician weaves an intricate pattern of microscopic precision
\eqrm

\noindent and 

\bqrm
to him, a false statement $\ldots$ is an unforgivable sin,
\eqrm

\noindent  the attitude of ``heroic mathematicians'' was quite different: 

\bqrm
The heroic
mathematician painted with broad splashes of color, with a grand contempt
for singular cases until they could no longer be avoided.
\eqrm

\noindent  Synge denies
that ``Hamilton lived in the clouds and spurned details,'' but goes on to
add that 

\bqrm
the details with which he busied himself were not the details
that a modern mathematician considers important. Few modern mathematicians
like to work with a function without a careful statement regarding its
general properties (continuity and existence of derivatives), or without
assurance of existence under stated conditions.  Hamilton, in the spirit of
the heroic age, goes straight ahead. That is the feature which doubtless
repels modern mathematicians from a more careful study of his work.  It
explains in large measure the eclipse of Hamilton.
\eqrm

\medbreak As will be shown in Chapter \ref{valuefun} below, Synge's 
reference to
Hamilton's ``eclipse'' during ``the ensuing century'' was made at a time
when a new theory was about to be born, in which Hamilton's profound concept
of the principal function---now renamed the ``value function,'' or
``Bellman function''---occupies precisely the central role that Hamilton
and Synge thought it deserved.  This theory is none other than modern
optimal control, and its rise has given a new impetus to the study of
Hamilton's principal function, confirming Synge's assessment of the
significance of Hamilton's insight. Under
the name of ``dynamic programming,'' the method of following
``Hamilton's plan to develop all the properties of the extremals from
the characteristic function'' has become one of the best and most
widely used tools in the modern analysis of optimal control problems.


At the same time, the rise of optimal
control has shown that Synge's complaint against modern mathematicians for
their unwillingness to ignore certain types of ``details'' was not entirely
justified. Indeed, as will be argued in Chapter \ref{valuefun},
modern optimal
control theory shows that a rich and exciting theory of the value function
exists, in which the question of the appropriate regularity
properties of the value function plays a crucial role. 
In many cases, this theory is the best if not the only tool for
the analysis of an optimal control problem.  Moreover, the new theory 
is completely rigorous, and the effort to make it rigorous  has turned out
to be worthwhile, in the sense that the existence and continuity 
issues mentioned by Synge are quite interesting in their own right, as 
we will show for the case of the brachistochrone problem. 

So the issues that Hamilton failed to ``busy himself'' with, ``in the spirit 
of the heroic age,'' have in fact turned out to be of the utmost
importance. It is now clear that, in a sense, the instincts of those
mathematicians who were ``repelled'' by Hamilton's inattention to
``details'' such as ``a careful statement regarding [the] general
properties [of the value function] (continuity and existence of
derivatives)'' were not completely wrong, since the value function
often lacks such properties, and this fact is significant and cannot
be ignored. Therefore both Synge and the people whom he criticized
have been fully vindicated: Synge, because his assessment of the
importance of Hamilton's ``central idea'' was right on target, and the
mathematicians who asked for more rigor and precision, because this
rigor and precision have turned out to be necessary.  With hindsight,
it can now be seen that the ``eclipse,'' however regrettable it may
have appeared to Synge, was perhaps unavoidable, because only with the
tools of late-20th-century mathematics has it become possible to do
full justice to Hamilton's idea.
\label{needbioweie}


\ssection{Weierstrass comes close, but fails to
discover Hamiltonian maximization} The contributions
of Karl Weierstrass (1815-1897) to the calculus of variations
are contained in the notes of his three
series of lectures on the topic, given in 1875, 1879 and
1882, and not formally published as books or journal articles. 

Weierstrass considered the problem of minimizing an integral $I$ of
the form $I=\int_a^bL(q(s),\dot q(s))\,ds$ for Lagrangians $L$ such
that

\medbreak
\begin{center}
\begin{minipage}{12cm}
\bi
\item[{\bem (W)}] {\em $L(q,\dot q)$ is positively homogeneous
with respect to the velocity $\dot q$ (that is, $L(q,\alpha\,\dot q)=
\alpha L(q,\dot q)$ for all $q,\dot q$ and all $\alpha\ge 0$)
and does not depend on time.}\proofend
\ei
\end{minipage}\end{center}

\medbreak\noindent As will become clear from Remark \ref{snott}, we
have a good reason for  
using  $s$ rather than $t$ as the ``time'' variable in the
expression for $I$.


\begin{remark}{\rm \label{snott} In a sense, one can always assume (W) ``without loss of
generality.''  Indeed, suppose we are interested in a completely arbitrary
problem (\ref{standard}).  We can always define a new function
$\Lambda(q,t,u,\tau)=\tau L(q,u/\tau,t)$, and think of $t$ as a new $q$
variable, say $q^0$, and of $\tau$ as $\frac{dq^0}{ds}$, were $s$ is a new
time variable, or ``pseudotime,'' not to be confused with the true time
variable $t$.  Then, using $q_{new}$ for $(q,q^0)$ (so that $q_{new}=(q,t)$),
we have
$$
\int L(q,\dot q,t)dt=\int
L\Biggl(q,\Big(\frac{dt}{ds}\Big)^{-1}\frac{dq}{ds},t\Biggr)\frac{dt}{ds}ds
=\int\Lambda\Biggl(q_{new},\frac{dq_{new}}{ds}\Biggr)ds\,,
$$
so the minimization problem for $\Lambda$ is equivalent to
(\ref{standard}). Moreover, if $\alpha\!\!>\!\!0$, then
$\Lambda(q,t,\alpha\,u,\alpha\,\tau)\!=\!\alpha\,\Lambda(q,t,u,\tau)$, i.e.,
$\Lambda(q_{new},\alpha\,\dot{q}_{new})\!=\!\alpha\,\Lambda(q_{new},
\dot{q}_{new})$, where we are now using the dot to denote differentiation
with respect to $s$ rather than $t$.  So the new Lagrangian $\Lambda$ is
positively homogeneous with respect to the new velocity variable
$\dot{q}_{new}$ and does not depend on the new time $s$.  \proofend }\end{remark}

So when Weierstrass imposed Condition (W) on his Lagrangians, he was doing
so ``without loss of generality.''  However, ``without loss of generality''
is a dangerous phrase, and does not at all entail ``without loss of
insight.''  We shall argue below that this restriction, in conjunction with
the dominant view that Hamilton's equations had to be written in the form
(\ref{hamileqsysham}), may have served to conceal from Weierstrass the true
meaning and the far-reaching implications of the new condition he
discovered.

Weierstrass introduced the ``excess function''
\begin{equation}\label{excess1}
\mathcal{ E}(q,\bar u,u)=L(q,u)-\frac{\partial L}{\partial u}(q,\bar u)
\cdot u\,, 
\end{equation}
depending on three sets of independent variables $q$, $\bar u$ and $u$.  He
then proved  his {\em side condition}:

\bigbreak
\begin{center}
\begin{minipage}{11cm}
\bi
\item[{\large{\bem (SC)}}] {\large{\em For a curve $s\mapsto{} q_*(s)$ to
be a solution of the 
minimization problem, the function $\mathcal{ E}$ has to be $\ge 0$ when
evaluated for $q=q_*(s)$, $\bar u=\dot q_*(s)$, and a completely arbitrary 
$u$.}}
\ei 
\end{minipage}\end{center}

\bigbreak\noindent 
Notice that Lagrangians with Property (W) satisfy the identity
\begin{equation}\label{excess-hom}
L(q,u)=\frac{\partial L}{\partial u}(q,u)\cdot u\,.
\end{equation}
Therefore Weierstrass could
equally well have written his excess function as 
\begin{equation}\label{excess2}
\mathcal{E}(q,\bar u,u)=L(q,u)-\frac{\partial L}{\partial u}(q,\bar u)\cdot u
-\Bigl(L(q,\bar u)-\frac{\partial L}{\partial u}(q,\bar u)\cdot\bar u\Bigr)\,,
\end{equation}
or as
\begin{equation}\label{excess2-withtime}
\mathcal{E}(q,\bar u,u,t)=L(q,u,t)-\frac{\partial L}{\partial u}
(q,\bar u,t)\cdot u-\Bigl(L(q,\bar u,t)-\frac{\partial L}{\partial u}
(q,\bar u,t)\cdot\bar u\Bigr)\,,
\end{equation}
if he had allowed his Lagrangians to depend on time.

\bigskip
\vbox{
\[
\epsfbox{excessnew.eps}
\]
\figurecap{The Weierstrass side condition\label{excess-fig}}
}

\begin{remark}\label{interp-excess}
{\rm Formula (\ref{excess2}) has a simple geometric interpetation. 
For a given point $q$, let $G_q$ be the  graph of 
$$
y=L(q,u)\equaldef L_q(u)
$$ 
as a function of $u$. Fix a value $\bar u$ of $u$, and let 
$$
\bar y=L(q,\bar u)\,,
$$
so the point 
$$
\bar P=(\bar u,\bar y)
$$
lies on $G_q$. Let $\Gamma_{q,\bar u}$ be the tangent hyperplane to $G_q$
at $\bar P$.  Then the equation of $\Gamma_{q,\bar u}$ is 
$$
y=\bar y+\frac{\partial L}{\partial u}(q,\bar u)\cdot(u-\bar u)\,,
$$
that is 
$$
y=L(q,\bar u)+\frac{\partial L}{\partial u}(q,\bar u)\cdot(u-\bar u)\,.
$$
Given any $u$, the ordinates $y_{graph}(u)$, $y_{plane}(u)$ 
of the points of $G_q$, $\Gamma_{q,\bar u}$,
that lie on the line $\{u\}\times\rr$ are given by 
$$
y_{graph}(u)=L(q,u)\,,\quad y_{plane}(u)=
L(q,\bar u)+\frac{\partial L}{\partial u}(q,\bar u)\cdot(u-\bar u)\,.
$$
Therefore
$$
\mathcal{E}(q,\bar u,u)=y_{graph}(u)-y_{plane}(u)\,.
$$
In other words (cf. Figure \ref{excess-fig}): 

\medbreak
\begin{center}
\begin{minipage}{11cm}
{\large{\bem the number $\mathcal{E}(q,\bar u,u)$ tells 
us by how much the actual value of the function $y=L_q(u)$ 
at a particular $u$ is larger than the value at $u$  of the linearization
(i.e., the tangent approximation) of this function at the point
$(\bar u,L(q,\bar u))$.}}
\end{minipage}
\end{center}

\bigbreak\noindent Therefore the Weierstrass side condition says (reverting
now to the more general time-varying fomulation) that 

\medbreak
\begin{center}
\begin{minipage}{11cm}
\bi
\item[{\large{\bem (SC)}}] {\large{\em 
If a curve $s\mapsto\xi_*(s)$ is a solution of the 
minimization problem, then for every $t$, if we draw the tangent 
hyperplane to the graph of the function $u\mapsto L(\xi_*(t),u,t)$ 
at the point of this graph corresponding to $u=\dot\xi_*(t)$, then the graph
must lie entirely above the hyperplane.}}\proofend  
\ei 
\end{minipage}\end{center}}
\end{remark}

Using
$$
\pi(t)=\frac{\partial L}{\partial u}(\xi_*(t),\dot\xi_*(t),t)\,,
$$ as in (\ref{defofp}), we see that 
\begin{equation}
\mathcal{E}(\xi_*(t),\dot\xi_*(t),u,t)=
\Big(L(\xi_*(t),u,t)-\langle \pi(t),u\rangle\Bigr)
-\Bigl(L(\xi_*(t),\dot\xi_*(t))-\langle\pi(t),\dot\xi_*(t)\rangle\Bigr)\,,
\end{equation}
which the reader will immediately recognize as saying that
\begin{equation}\label{indep}
\mathcal{E}(\xi_*(t),\dot\xi_*(t),u,t)=
H(\xi_*(t),\dot\xi_*(t),\pi(t),t)-H(\xi_*(t),u,\pi(t),t)\,,
\end{equation}
where $H$ is our ``control  Hamiltonian.''    So Weierstrass' condition,
expressed in terms of the control Hamiltonian, simply says that

\medbreak
\begin{center}
\begin{minipage}{11.4cm}
\bi
\item[{\large{\bem (MAX)}}] {\large{\em  Along an optimal curve $t\mapsto{}
\xi_*(t)$, if we let $\pi(t)=\pi_{\xi_*,L}(t)$, then for every $t$ the 
value $u=\dot\xi_*(t)$ must maximize the (control) Hamiltonian 
$H(\xi_*(t),u,\pi(t),t)$ as a function of $u$.}}
\ei
\end{minipage}\end{center}

\gnu{JAN: In the earlier versions I talked about 
configurations (or states) $q$ and curves $t\mapsto q(t)$ 
in $q$-space. Then the momentum was $p(t)$, and so on. 
I then decided that it would be better to be more systematic
and use Latin letters for points and Greek lettters for 
functions of time, so the curves are $\xi(t)$, the momentum is
$\pi(t)$, the open-loop controls are $\eta(t)$, etc.
I have done the conversion UP TO THIS POINT, but from here on it's not done.
I don'y have time to do it now. I'll do it when I get back from 
Europe in August.}



\medbreak\noindent In Weierstrass' formulation, the condition was stated in
terms of the 
excess function, for the special Lagrangians that satisfy (W).  (In that
case the resulting $H$ is independent of time, as in our equation
(\ref{indep})).  But, if one rewrites Weierstrass's condition as we have
done, in terms of $H$, then one can take a general Lagrangian, not
satisfying (W), transform the minimization problem into one in
Weierstrass's form, as explained in Remark~\ref{snott}, write the
Weierstrass condition in the form (MAX) (so in particular $H$ is
independent of time) and then undo the transformation and go back to the
original problem.  The result is (MAX), as written, with the control
Hamiltonian of the original problem.  So the Weierstrass condition, if
reformulated as in (MAX), is valid for all problems, with exactly the same
statement.  For comparison, notice that if one applies the same procedure
to the formulation in terms of $\mathcal{ E}$, then one also ends up with a
statement valid for all problems, but the expression for $\mathcal{ E}$ is now
much more complicated, since one has to use (\ref{excess2}) and cannot
substitute  for it the simpler expression (\ref{excess1}).

Moreover, (MAX) can be considerably simplified.  Indeed, 

\begin{center}
\fbox{\fbox{\begin{minipage}{11.3cm}
{\bf\em the requirement that  $p(t)$ be defined via (\ref{defofp}) is now 
redundant}
\end{minipage}}}
\end{center}

\medbreak\noindent because, if $H(q_*(t),u,p(t),t)$, regarded as a 
function of $u$,  has a maximum at $u=\dot q_*(t)$, then
$\frac{\partial H}{\partial u}(q_*(t),\dot q_*(t),p(t),t)$ has to vanish,
so  $p(t)$ has to be given by (\ref{defofp}). Moreover, the vanishing of   
$\frac{\partial H}{\partial u}(q_*(t),\dot q_*(t),p(t),t)$ is also one of 
the conditions  of  (\ref{hamileqsys}). So we can just drop
(\ref{defofp}) altogether, and state (\ref{hamileqsys}) and (MAX) together:

\medbreak
\begin{center}
\fbox{\fbox{\begin{minipage}{12cm}
\begin{minipage}{1.6cm}
\noindent{\large{\bem (NCO)}}\\
\vspace{4cm}
\end{minipage}
\begin{minipage}{10.35cm}
{\large{\em  If a curve $t\mapsto{} q_*(t)$ 
is a solution of the  
minimization problem (\ref{standard}), then there has to exist a function
$t\mapsto{} p(t)$ 
such that ($p$ is absolutely continuous and)
the following three conditions hold for all $t$:}
\begin{eqnarray}
\quad\quad\dot q_*(t)&=&\phantom{-}\frac{\partial H}{\partial p}(q_*(t),
\dot{q}_*(t),p(t),t)\,,\label{nco1}\\
\quad\quad\dot p(t)&=&-\frac{\partial H}{\partial q}(q_*(t),\dot q_*(t),p(t),
t)\,,\label{nco2}\\
\quad\quad
H(q_*(t),\dot{q}_*(t),p(t),t)&=&\max_{u}H(q_*(t),u,p(t),t)\,.\label{nco3} 
\end{eqnarray}}
\end{minipage}\end{minipage}}}\end{center}

\begin{remark}{\rm 
The parentheses for the absolute continuity condition refer to the fact 
that
\bi
\item[(i)] this is a natural technical condition to impose on
$p$ from the 20th century point of view, since (\ref{nco2}) is a
differential equation for $p(\cdot)$,
\ei
while, on the other hand, 
\bi
\item[(ii)] the concept of an absolutely continuous function obviously 
belongs to a circle  of ideas about functions of a real variable that were
not yet available to Weierstrass.  \proofend
\ei
}\end{remark} 

\def\quocar{We follow \cite{carath}, pages 210-212.}

\gnu{JAN: The ``Weierstrass comes close'' section was written before
I looked at Carath\'eodory. When I read Carath\'eodory I was
fascinated by the extra term in Carath\'eodory's formula 
for the excess function, because this term shows that Carath\'eodory 
really didn't get it. So I put in some stuff about Carath\'edory.
But now I should look at the ``Weierstrass comes close'' section 
again, because some things that are there should be moved 
and put AFTER the Carath\'eodory section.}
\label{needbicar}

\medskip
\ssection{Carath\'eodory gets even closer, but misses too}
\label{subsec-carat} 
C. Carath\'eodory (1873-1950) published his work on the
calculus of variations in a 1935 book, cf. \cite{carath}. His discussion of
the excess function, and of how Weierstrass' condition  can be expressed 
in Hamiltonian form, is worth quoting in detail\Ft{\quocar}, because it shows 
that the author understood the importance
of expressing the Weierstrass condition in Hamiltonian form,
but was unable to find its simple Hamiltonian meaning
because he was using an inappropriate Hamiltonian formalism.

Carath\'eodory uses $x_i$ for the position variables (that is, our $q^i$) 
and $y_i$ for the momenta (that is, our $p_i$). He works in the neighborhood
of a ``regular line \mbox{element}'' $e=(t,x_i,\dot x_i)$,   where the
``regularity'' assumption is the requirement that the Hessian matrix 
$\{L_{\dot x_i,\dot x_j}\}_{1\le i,j\le n}$ be positive definite
at $e$.   This condition guarantees that, near $e$, one can express
everything in ``canonical coordinates'' $(t,x_i,y_i)$, i.e., that
the map $(t,x,\dot x)\to (t,x,y)$ is invertible, if  we let
$y_i=\chi_i(t,x,\dot x)$, where 
$\chi_i(t,x,\dot x)=\frac{\partial L}{\partial \dot x_i}(t,x,\dot x)$.

He then expresses the Weierstrass condition, which involves 
the ``line element $(t,x_i,\dot x_i)$'' together with 
a nearby ``line element'' $(t,x_i,x'_i)$,  in terms of the corresponding 
canonical coordinates $(t,x_i,y_i)$ and 
$(t,x_i,y'_i)$.  He uses $H$ to represent what he calls
``the Hamiltonian'' (that is, our classical Hamiltonian), given by 
the formula $H=\sum_{i=1}^ny_i\dot x_i-L(t,x,\dot x)$, where 
$\dot x_i=\varphi_i(t,x,y)$, and $\varphi$ is the inverse 
function of $\chi$ (that is, $\varphi(t,x,\chi(t,x,u))=u$ 
and $\chi(t,x,\varphi(t,x,y))=y$). In addition, he uses 
$H'$ to represent the Hamiltonian evaluated at the other 
point $(t,x,y')$  in phase space, that is,
$H'=\sum_{i=1}^ny'_ix'_i-L(t,x,x')$, where 
$x'_i=\varphi_i(t,x,y')$, $y'_i=\chi_i(t,x,x')$.

With these notations, the Weierstrass excess function becomes 
\begin{eqnarray*}
\mathcal{E}(t,x,\dot x,x')&=&
L(t,x,x')-L(t,x,\dot x)-\frac{\partial L}{\partial\dot x}
(t,x,\dot x)\cdot(x'-\dot x)\\
&=&L(t,x,x')-L(t,x,\dot x)-y\cdot(x'-\dot x)\\
&=&y\cdot\dot x-L(t,x,\dot x)-\Big(y\cdot x'-L(t,x,x')\Big)\\
&=&y\cdot\dot x-L(t,x,\dot x)-\Big(y'\cdot x'-L(t,x,x')\Big)-(y-y')\cdot x'\\
&=&H-H'-(y-y')\cdot x'\,.
\end{eqnarray*} 
Since
\begin{eqnarray*}
\frac{\partial H'}{\partial y'_j}&=&
\frac{\partial\quad}{\partial y'_j}\Big(\sum_{i=1}^ny'_ix'_i-L(t,x,x')\Big)\\
&=&x'_j+\sum_{i=1}^ny'_i\frac{\partial\varphi_i}{\partial y'_j}(t,x,y')
-\frac{\partial\quad}{\partial y'_j}\Big(L(t,x,\varphi(t,x,y'))\Big)\\
&=&x'_j+\sum_{i=1}^ny'_i\frac{\partial\varphi_i}{\partial y'_j}(t,x,y')
-\sum_{i=1}^n\frac{\partial L}{\partial x'_i}(t,x,x')
\frac{\partial\varphi_i}{\partial y'_j}(t,x,x')\\
&=&x'_j+\sum_{i=1}^ny'_i\frac{\partial\varphi_i}{\partial y'_j}(t,x,y')
-\sum_{i=1}^ny'_i\frac{\varphi_i}{\partial y'_j}(t,x,x')\\
&=&x'_j\,,
\end{eqnarray*} 
Carath\'eodory concluded that 
``the equation
\begin{eqnarray*}
\mathcal{E}(t,x,\dot x,x')&=&L'-L-L_{\dot x_j}(x'_j-\dot x_j)\\
&=&H-H'-H'_{y'_j}(y_j-y'_j)
\end{eqnarray*} 
therefore holds, by which the $\mathcal{E}$ function is represented
in the desired form.''


 
\medskip
\medskip
\ssection{A PUZZLE: What has disappeared?} {\em (NCO) display a
{\bf \em remarkable new fact} that makes it fundamentally different
from all our previous necessary conditions.  We urge the reader to
stare at (NCO) and try to figure out what this fact may be before
reading on.}\Ft{\label{cryptic}{\mit A cryptic hint for playful
quiz-loving folks.}  In many situations, you can gain a lot by
focusing on omissions, voids, and vanishing acts, by thinking about
what is missing and ignoring what is visibly in front of you.  This
obvious point might possibly call to mind a story, probably familiar
to you, about a dog that did not bark (cf. \cite{conan}, pp.~347 and
349).  Such a curious fact had no significant implications for
Dr. Watson's simplistic mind, but his famous companion soon saw how
crucial it was.  Similarly, it is as plain as daylight that in your
analysis of Condition (NCO) {\myninebf you should look for what you
cannot find in its formulas} and should stop worrying about things
within your sight.  If you think that all this sounds too cryptic and
hard to grasp, you may go back---without loss---to our discussion of
how an amazingly circuitous path, in many ways astonishingly similar
to that of A. J. Tasman's circumnavigation of Australia, took us from
our old calculus of variations conditions of a distant past to today's
work on optimal control and dynamic programming.  Or, if this hint is
too baffling for you, thanks to your inability to find what is
obviously missing in it, you may look at Scholium \ref{schol-adair}
for a good indication of what this is all about and why our words may
sound so odd to you. You will find in that scholium two quotations
from a translation, by G. Adair, of a truly unusual and amazing 
work of fiction by a famous Gallic author (\cite{perecad}), and as
soon as you grasp what that book is about all your
doubts will finally go away. But you must insist anyhow on doing your
own thinking, and in no way allow that odd book to put wrong thoughts
into your mind.  In particular, you should avoid following that book's
unjustifiably critical opinion about Pontryagin.  (In p.~47 of Adair's
translation, \cite{perecad} wrongly says that ``Though it took him
from 1935 to 1955, working at it virtually nonstop, Pontryagin finally
had to admit to accomplishing nothing.'' What a ridiculous thing to
say! It's totally absurd! Naturally, you could think, if you wish,
that Pontryagin was an awful man, but it is obvious that his work was
important and rich in major innovations.)}  {\bem The answer will be
given below.}  \proofend

\medskip
As a version of the necessary conditions for optimality, (NCO) encapsulates
in one single statement the combined power of the Euler-Lagrange necessary
conditions and the Weierstrass side condition as well, of course, as the
Legendre condition, which obviously follows from (MAX).  Notice the
elegance and economy of language achieved by this unified statement: there
is no need to bring in an extra entity called the ``excess function.'' Nor
does one need to include a formula specifying how $p(t)$ is defined, since
(\ref{nco3}) does this
automatically.  So the addition of the new Weierstrass condition to the
three equations of (\ref{hamileqsys}) results in a new set of
three, rather than four, conditions, a set much ``simpler than the sum of
its parts.''  Notice moreover that (MAX)---or, more precisely, the
Weierstrass side condition part of (MAX)---is exactly Conjecture M.  So we
can surmise at this point that (MAX), as stated, could probably
have been discovered soon after the work of Hamilton, since it is 
strongly suggested by (\ref{legendrehamtog}), and almost certainly
by Weierstrass, if only  Hamilton's
equations had been written in the form (\ref{defofh}),
(\ref{hamileqsys}). 

So we can now add two new items to our list of advantages of the ``control
formulation'' of Hamilton's equations over the classical one:

\medbreak\noindent{\bf (A2)}\ {\em Using the control Hamiltonian, it would
have been an obvious 
next step to write Legendre's condition in ``Hamiltonian form,''
as in (\ref{legendrehamtog}), and this would have led immediately 
to the formulation of Conjecture M, a proof of which would then have 
been found soon after.}

\medbreak\noindent{\bf (A3)}\ {\em In terms of the control Hamiltonian, 
the Weierstrass side condition
has a much simpler statement, not requiring the introduction of an 
``excess function,'' and can be combined with the Hamilton equations
into an elegant unified formulation (NCO) of the necessary conditions for 
optimality.}

\medbreak\noindent But this is by no means the end of our story.  There is
much more 
to the new formulation (NCO) than just elegance and simplicity.  If you
compare (NCO) with all the other necessary conditions that we had written
earlier, a remarkable new fact becomes apparent.  
Quite amazingly, {\bem the derivatives with respect to the $u$ variable are
completely gone.}  ({\em This is the answer to our puzzle.})
All the earlier equations involved $u$-derivatives of
$L$ or of $H$, and even if we use the classical version
(\ref{hamileqsysham}) of Hamilton's equations---which involves no functions
of $u$ and therefore no $u$-derivatives---the fact remains that in order to
get to (\ref{hamileqsysham}) we first have to solve (\ref{pimpl}), which
does involve a $u$-derivative.

Now, if our necessary conditions for optimality can be stated without any
references to $u$-derivatives, it is reasonable to guess that the
$u$-derivatives of $L$ are not needed.  Also, the minimization that occurs
in (\ref{nco3}) makes sense over any subset $U$ of $\rr^n$, so there is
no longer any reason to insist that the range of values of $u$ be the whole
space.   This leads us to 

\medbreak
\begin{center}
\begin{minipage}{12cm}
\noindent{\bem CONJECTURE  M2:}\ {\em  (NCO)  should still be a 
necessary condition for
optimality even for problems where the derivative $\dot q$ is restricted 
to belong to some subset $U$ of $\rr^n$, and $L(q,u,t)$ is not required 
to be differentiable or even continuous with respect to $u$.} \proofend
\end{minipage}\end{center}

\bigskip\noindent
Naturally, once $u$ is restricted to a subset $U$ of $\rr^n$, and
$L(q,u,t)$ is not required to be differentiable with respect to $u$,
the maximization condition (\ref{nco3}) no longer implies
(\ref{pimpl}).  So we see that {\bf\em (\ref{nco3}) is more
fundamental than (\ref{pimpl})}. 

Notice also, that there has been a subtle change in the formulation of
the necessary conditions for optimality.  In the classical conditions
of the calculus of variations (including Weierstrass' own version of
the side condition), the vector $p$---the ``momentum''---is defined
by (\ref{pimpl}), and the necessary conditions are equations happening
along the trajectory $q_*$. These equations can be formulated somewhat
more simply using $p$, but $p$ can also be completely eliminated,
since one can always substitute for it its value given by
(\ref{pimpl}).  The new version of Conjecture M2 is of a different
form: rather than stating that certain equations hold along $q_*$, the
conditions are now an {\em existential statement}, asserting the
existence of a vector-valued function $t\mapsto p(t)$ with certain
properties.  The elimination of $p$, by expressing $p(t)$ in terms of 
$q_*(t)$, $\dot q_*(t)$ and $t$, is only possible when $U=\rr^n$
and $L$ is differentiable with respect to $u$, in which case
(\ref{pimpl}) follows from (\ref{nco3}).  Otherwise, $p$ is a ``new,''
``independent'' object.        

Since Weierstrass was working under assumptions that imply 
(\ref{pimpl}), he missed the chance to see that his condition
was really about $p$, not about $\frac{\partial L}{\partial u}$
evaluated along $q_*$, and that, stated this way, his condition
was valid for much more general problems, for which $p(t)$ 
no longer equals $\frac{\partial L}{\partial u}(q_*(t),\dot q_*(t),t)$. 

\ssection{Three toy problems} To test Conjecture M2, let us
look at three toy problems where the answer is  
reasonably easy to guess directly, and see whether using 
(NCO) gives the same result. 

\begin{example}{\rm \label{toy1}
To test the differentiability part of Conjecture M2, 
let us  suppose  that we are given $L>0$, and 
we want to find a real-valued (Lipschitz continuous) function 
$t\mapsto{}x(t)$ on the interval $[0,L]$ that satisfies 
\begin{equation}\label{thecoend}
x(0)=1\,,\,\,\,\,{\rm and\,\,\,\,\,\,\,\,}x(L)=1\,,
\end{equation}
and minimizes the integral  $\int_0^L|\dot x(t)|\,dt$ among all the
functions  on $[0,L]$ 
satisfying the constraints (\ref{thecoend}).  This looks exactly like a
calculus 
of variations problem of the classical sort, with a Lagrangian 
$L(x,\dot x,t)=|\dot x|$ (i.e., $L(x,u,t)=|u|$), except that 
$L$ is no longer everywhere differentiable with respect to $u$.
Moreover, it is quite obvious that the solution of this minimization
problem is given by $u(t)\equiv 0$, i~.e., by $x(t)\equiv 1$.
Therefore the nondifferentiability 
issue truly matters for the applicability of the necessary conditions,
since $\{(x,u,t):u=0\}$ is precisely where $\frac{\partial L}{\partial u}$
fails to exist.  In particular, the classical Euler-Lagrange equation
cannot even be written, let alone solved. 

On the other hand, if we apply (NCO) as stated, there is no problem
whatsoever.  We have $H=p.u-|u|$ (so $H$ in this case does not depend on
$x$ and $t$), and then (\ref{nco1}) and (\ref{nco2}) become
$\dot{x}=u$ and $\dot p=0$.  Therefore the function $t\mapsto{} p(t)$ is a
constant $\bar{p}$.  Now, given any value of $\bar p$, one can easily check
that the set of values of $u$ where the function $u\mapsto{} \bar pu-|u|$
has a maximum is always a subset of $[0,\infty[$ or of $]-\infty,0]$.  (If
$|\bar p|<1$, then the maximum is attained at $u=0$ only.  If $|\bar p|>1$,
then there is no maximum. If $\bar p=1$ then the maximum is attained at all
values of $u$ such that $u\ge 0$.  If $\bar p=-1$ then it is attained at
all $u$'s such that $u\le 0$.) Since $u(t)$ has to be chosen, for each $t$,
so as to maximize $\bar pu-|u|$, we see that either $u(t)\ge 0$ for all
$t$, or $u(t)\le 0$ for all $t$. In either case, the only way that $x(0)$
and $x(L)$ can be equal is if $u(t)\equiv 0$.  (Of course, what we should
really say is ``$u(t)=0$ a.~e.''.) So (NCO) works perfectly, and singles
out the solution we had already found. \proofend 
}\end{example}

                
\begin{example}{\rm \label{toy1-2d}
We now look at a two-dimensional analogue of the previous example. 
We try  to find a Lipschitz continuous curve
$t\mapsto{}\xi(t)=(\xi_1(t),\xi_2(t))\in\rr^2$, defined
on the interval $[0,1]$, that satisfies 
\begin{equation}\label{thecoend-2d}
\xi(0)=(0,0)\,,\,\,\,\,{\rm and\,\,\,\,\,\,\,\,}\xi(1)=(1,2)\,,
\end{equation}
and minimizes the integral  
\begin{equation}\label{thecoend-2d-int}
J(\xi)=\int_0^1\Big(|\dot\xi_1(t)|+|\dot\xi_2(t)|\Big)\,dt
\end{equation}
in the set $\Xi$ of all the Lipschitz functions 
$[0,1]\ni t\mapsto\xi(t)\in\rr^2$
satisfying the constraints (\ref{thecoend-2d}).  Once again, 
this looks like a calculus of variations problem of the classical sort, 
with a Lagrangian 
$$
\rr^2\times\rr^2\times[0,1]\ni(x_1,x_2,u_1,u_2,t)
\mapsto L(x_1,x_2,u_1,u_2,t)\equaldef|u_1|+|u_2|\,.
$$
which is not everywhere differentiable with respect to $u$.
It is easy to see that
$$
J(\xi)\ge 3\quad{\rm for\,\,\,every}\quad\xi\in\Xi\,.
$$
Indeed, if $\xi\in\Xi$ then 
\begin{equation}\label{conve-lip1-aa}
1=\xi_1(1)-\xi_1(0)=\int_0^1\dot\xi_1(t)\,dt\le
\int_0^1|\dot\xi_1(t)|\,dt\,,
\end{equation}
and a similar argument shows that 
\begin{equation}\label{conve-lip2-aa}
2\le\int_0^1|\dot\xi_2(t)|\,dt\,.
\end{equation}
Therefore $J(\xi)=\int_0^1\Big(|\dot\xi_1(t)|+|\dot\xi_2(t)|\Big)\,dt\ge 3$, 
as stated.

So any arc $\xi\in\Xi$ for which $J(\xi)=3$ is a solution of our minimization
problem. One example is the arc $\xi_*:[0,1]\mapsto\rr^2$ given
by
$$
\xi_*(t)=(t,2t)\quad{\rm for}\quad 0\le t\le 1\,.
$$
Another example is the arc $\xi_{\#}:[0,1]\mapsto\rr^2$ given by
$$
\xi_{\#}(t)=\left\{\begin{array}{lcl}
(2t,0)&\,\,{\rm if}\,\,&0\le t\le\frac{1}{2}\\
(1,4t-2)&\,\,{\rm if}\,\,&\frac{1}{2}\le t\le 1\,.\end{array}\right.
$$
Notice that {\bem the arc $\xi_{\#}$ is such that
for every $t$ the vector $\dot\xi_{\#}(t)$ belongs to the set 
of $(u_1,u_2)$ where $L$ is not differentiable}.
Therefore, exactly as in the previous example, 
the classical Euler-Lagrange equation
cannot even be written, let alone solved. 

Let us now apply (NCO) and see what we get.  The Hamiltonian is now given
by
$$
H(x_1,x_2,u_1,u_2,p_1,p_2,t)=p_1u_1+p_2u_2-|u_1|-|u_2|\,,
$$ 
(so in particular $H$ does not depend on
$x$ and $t$). Then (\ref{nco1}) and (\ref{nco2}) tell us that
$$
\dot\xi=u\quad{\rm and}\quad\dot\pi=0\,.
$$
Therefore the function $t\mapsto{}\pi(t)$ is a constant 
$\bar\pi=(\bar\pi_1,\bar\pi_2)\in\rr^2$.  

As in the previous example, the set of values of $u_1$ where the 
function $u_1\mapsto\bar\pi_1u_1-|u_1|$
has a maximum is always a subset of $[0,\infty[$ or of $]-\infty,0]$.  
Since the function $t\mapsto\dot\xi_1(t)$ has to be chosen so that
for each $t$ the value $u_1=\dot\xi_1(t)$ maximizes $\bar\pi_1u_1-|u_1|$, 
we see that either $\dot\xi_1(t)\ge 0$ for all
$t$, or $\xi_1(t)\le 0$ for all $t$. Since 
$\int_0^1\dot\xi_1(t)\,dt=1$, the possibility that 
$\xi_1(t)\le 0$ for all $t$ is excluded. So $\xi_1$ is an arbitrary
Lipschitz function such that 
\begin{equation}\label{conve-lip1} 
\xi_1(0)=0\,,\quad\xi_1(1)=1\,,\quad{\rm and}\quad
\xi_1\,\,\,{\rm is\,\,\,monotonically\,\,\,nondecreasing}\,.
\end{equation}
Similarly, $\xi_2$ is an arbitrary Lipschitz function such that 
\begin{equation}\label{conve-lip2}
\xi_2(0)=0\,,\quad\xi_2(1)=2\,,\quad{\rm and}\quad
\xi_2\,\,\,{\rm is\,\,\,monotonically\,\,\,nondecreasing}\,.
\end{equation}
Conversely, {\em every Lipschitz curve 
$[0,1]\ni t\mapsto\xi(t)=(\xi_1(t),\xi_2(t))\in\rr^2$
such that (\ref{conve-lip1}) and (\ref{conve-lip2}) hold is a minimizer}.
(This is easy to prove. It suffices to observe that under
those conditions the inequalities occurring in
(\ref{conve-lip1-aa}) and (\ref{conve-lip2-aa}) become equalities,
so $J(\xi)=3$.)

So we see once again that {\bem (NCO) works perfectly, and singles
out exactly all the solutions of our minimization problem}. \proofend 
}\end{example}

                
\begin{example}{\rm \label{toy2}
We now test the other part of Conjecture M2, by looking at
a problem where the range of $u$ is no longer the whole space.

Suppose that we want to find a real-valued (Lipschitz) function
$t\mapsto{}x(t)$ on the interval $[0,L]$ that satisfies
\begin{equation}\label{theco}
x(0)=1\,,\,\,\,\,x(L)=1\,,\,\,{\rm and}\,\,\,|\dot x(t)|\le 1
\,\,\,{\rm for\,\,\, all\,\,\,}t\in[0,L]\,,
\end{equation}
and minimizes the integral $\int_0^Lx(t)^2dt$ among all the functions
satisfying the constraints (\ref{theco}).  Once again, this looks
exactly like a calculus of variations problem of the classical type---with
a Lagrangian $L(x,\dot x,t)=x^2$---except that the derivative $\dot x$ is
required to satisfy an ``inequality constraint'' $|\dot x|\le 1$.  If we
apply the classical Euler-Lagrange equations formally, we get
$\frac{\partial L}{\partial x}=0$ (since $L$ does not depend on $\dot x$,
so $\frac{\partial L}{\partial\dot x}=0$).  So $x(t)\equiv 0$,
contradicting the endpoint constraints.  So there is no way at all to
satisfy Euler-Lagrange together with the boundary conditions.


On the other hand, we can apply (NCO) formally, making the sensible guess
that in this case the maximization with respect to $u$ should be made over
the set $U=[-1,1]$ of permissible values of $u$. Then (\ref{nco1}) and
(\ref{nco2}) become the equations $\dot x=u$, $\dot p=2x$. This
suggests, among other things, that $p$ as a function of $t$ should be of
class $C^1$, since $\dot p$ has to be continuous. In addition, (\ref{nco3})
says that $u=1$ when $p>0$, and $u=-1$ when $p<0$.  (When $p=0$ all values
of $u\in U$ are solutions.)  So when $p>0$ we have $\ddot p=2$, and when
$p<0$ we have $\ddot p=-2$.  This shows that $p(\cdot)$ is a convex
function of $t$ when $p(t)>0$ and a concave function when $p(t)<0$.  Now, a
continuous function on $[0,L]$ which is convex wherever it is $>0$ and
concave wherever it is $<0$ can only be of one of the 13 types $P$, $N$,
$Z$, $PZ$, $PN$, $ZP$, $ZN$, $NP$, $NZ$, $PZP$, $PZN$, $NZP$ or $NZN$,
where ``$Z$'' means ``identically zero,'' ``$P$ means ``$>0$,'' ``$N$''
means ``$<0$,'' and, for example, a symbol such as $PZN$ means ``a positive
piece followed by a zero piece and then by a negative piece.''  (This is
easy to prove. For example, $ZPZ$ is impossible because the $P$ part would
have to vanish at two points and be convex and $>0$ in between, which
clearly cannot happen.  The reader should analyze all possibilities and
verify that only our 13 cases can arise.)  Clearly, $P$ corresponds to
$u=1$, and $N$ to $u=-1$.  As for $Z$, it corresponds to $p(t)\equiv 0$,
which yields $x(t)\equiv 0$ (since $\dot p=2x$) and then $u(t)\equiv 0$
(since $\dot x=u$).  So $Z$ corresponds to $x=u=0$.  (Notice that
(\ref{nco3}) does not directly determine the value of $u$ when $p=0$, since
in this case every value of $u$ satisfies (\ref{nco3}), but by suitably
combining the three conditions of (NCO) we have still been able to find $u$
for $p=0$.)  With this in mind, it is clear that the only way to satisfy
the endpoint conditions $x(0)=1$, $x(L)=1$, is with a $PN$ or an $NP$ or an
$NZP$ curve.  Of these three possibilities, $PN$ is obviously not optimal,
so we are left with $NP$ and $NZP$, and then a direct analysis shows that
the minimum is $NP$ if $L\le 2$ and $NZP$ if $L>2$. This result can of
course be proved directly, and corresponds to the obvious intuition that to
minimize $\int x^2$ one should try to move from the initial value $x=1$ towards
$x=0$ as fast as possible, stay at $x=0$ as long as as possible, and then
move back towards $x=1$, choosing the departure time {}from $x=0$ so as to
arrive to $x=1$ exactly at time $L$.  \proofend 
}\end{example}

Examples \ref{toy1} and \ref{toy2} show that (NCO), which is none other
than the combination of the classical Euler-Lagrange condition and the
Weierstrass side condition, is a surprisingly powerful tool, provided only
that it is properly reinterpreted as we have done.  The formal application
of the new version of (NCO) has been shown to yield a complete solution for
at least some problems that do not fit within the framework of the
classical calculus of variations, either because of nondifferentiability of
$L$ with respect to $u$, or because $\dot q$ is subject to an inequality
constraint.  All that is really needed is to use the Hamiltonian formalism
in its ``control'' version, and to apply the conditions with their full
force, expressed in (NCO). In addition, it helps to be flexible and be
willing to depart from the classical way of ``solving problems
analytically'' since, as our two examples show, one has to be ready to
combine calculus arguments of a classical sort with a little bit of
``piecewise smooth analysis'' and qualitative reasoning.

\ssection{Two more toy problems, and the ``abnormal''
multiplier} So far we have shown that Conjecture
M2 is almost forced on us if one looks at the classical condition from the
right perspective and with the right formalism. But we have not yet said if
it is actually true, nor have we given any indication as to how one might
go about proving it.   

So it may not be a bad idea to look at a few  more toy problems.  We
will consider two of them.
\begin{example}{\rm \label{toy4}
Suppose $L>0$ is given, and we want to {\em maximize}
the integral 
$$
J=\int_0^L\dot\xi(t)^2dt
$$
among all Lipschitz functions
$\xi(\cdot):[0,L]\mapsto\rr$ that satisfy the velocity constraint 
$$
\dot\xi(t)\ge 0\qquad{\rm for\,\,\,almost\,\,\,all}\quad t\,,
$$
as well as the endpoint conditions 
$$\xi(0)=\xi(L)=1\,.$$
In this case the set $U$ of possible velocities and the 
Lagrangian are given by
$$U=[0,+\infty]\,,\,\,L(q,u,t)=-u^2\,.
$$
Therefore the Hamiltonian is 
$$
H(q,u,p,t)=pu+u^2\,.
$$
We could go on and write (\ref{nco1}) and (\ref{nco2}) for our problem, but
there is no need to do so. Just by looking at $H$, we can see that for
every value of $p$ the function $u\mapsto H(q,u,p,t)$ (which is in
fact independent of $q$ and $t$) does not have a maximum $U$.
So no trajectories at all satisfy the conditions of (NCO).
On the other hand, our optimization problem does have a solution, since
$t\mapsto\xi(t)=1$ is the only curve that satisfies our endpoint conditions as
well as the constraint $\dot\xi(t)\ge 0$.  So (NCO) is {\em not} satisfied by
the solution.  \proofend
}\end{example}

\begin{example}{\rm \label{toy4-bis}
We want to minimize the integral 
$$
J=\int_0^1\xi_1(t)\xi_2(t)\,dt
$$
among all Lipschitz functions
$\xi(\cdot):[0,1]\mapsto\rr^2$ that satisfy the velocity constraint 
$$
\dot\xi_1(t)^2+\dot\xi_2(t)^2\le 1\qquad{\rm for\,\,\,almost\,\,\,all}
\quad t\,,
$$
as well as the endpoint conditions 
$$\xi(0)=(0,0)\,,\,\,\,\xi(1)=(1,0)\,.$$
In this case the set $U$ of possible velocities and the 
Lagrangian are given by
$$U=\{(u_1,u_2)\in\rr^2:u_1^2+u_2^2\le 1\}
\,,\,\,L(q_1,q_1,u_1,u_2,t)=q_1q_2\,.
$$
It is clear that there exists only one feasible path $\xi$,
namely, the arc given by
$\xi(t)=(t,0)\,,\quad{\rm for}\quad 0\le t\le 1\,.
$
Let us see whether this arc satisfies (NCO). 

The Hamiltonian is 
$$
H(q_1,q_2,u_1,u_2,p_1,p_2,t)=p_1u_1+p_2u_2-q_1q_2\,.
$$
The momentum $t\mapsto\pi(t)=(\pi_1(t),\pi_2(t))$, if it exists, 
would have to satisfy Hamilton's equations
$$
\dot\pi_1(t)=\xi_2(t)\,,\,\,\,\dot\pi_2(t)=\xi_1(t)\,,
$$
because 
$$
\frac{\partial H}{\partial q_1}=-q_2\quad{\rm and}\quad
\frac{\partial H}{\partial q_2}=-q_1\,.
$$
Since $\xi_1(t)=t$, we find 
\begin{equation}\label{orph1}
\dot\pi_2(t)=\xi_1(t)\,.
\end{equation}
On the other hand, the Hamiltonian maximization condition 
says that for almost all $t$ the function
\begin{equation}\label{orph2}
U\ni(u_1,u_2)\mapsto\pi_1(t)u_1+\pi_2(t)u_2-\xi_1(t)\xi_2(t)
\end{equation}
must be maximized by $u_1=\dot\xi_1(t)=1$, $u_2=\dot\xi_2(t)=0$.
On the other hand, if $\pi(t)\neq 0$, then the unique 
value $u^{max}$ of $u=(u_1,u_2)$ that maximizes (\ref{orph2}) is 
$$
u^{max}=\frac{\pi(t)}{\|\pi(t)\|}\,.
$$
It follows that $\pi(t)\neq 0\Rightarrow\pi_2(t)=0$.
Since $\pi(t)=0\Rightarrow\pi_2(t)=0$ as well, we have shown that
\begin{equation}\label{orph3}
\pi_2(t)\equiv 0\,.
\end{equation}
Clearly, (\ref{orph3}) contradicts (\ref{orph1}). Therefore 
$\pi$ does not exist, and (NCO) is {\em not} satisfied by
the solution.  \proofend
}\end{example}

Once again, as in Example \ref{toy3}, we seem to be in trouble.  But, as
opposed to what happened in Example \ref{toy3}, this time there is no
mistake.  The problem of Example \ref{toy4} is for a fixed time interval,
and we have applied (NCO) correctly.  So we have no choice, and have to
acknowledge that the conjecture that seemed so natural is in fact
{\bem}false.

It turns out, however,
that only a minor modification is needed to make it 
true.  All we have to do is introduce a new $p$-variable $p_0$, called
the ``abnormal multiplier,''\Ft{The need for the 
abnormal multiplier had already been noticed by Bolza in 1913,
cf. \cite{bolza}.}  and write  the Hamiltonian as
\begin{equation}\label{hamilmaxiprin}
H(q,u,p,p_0,t)=\langle p,f(q,u,t)\rangle - p_0 L(q,u,t)\,.
\end{equation}
Everything we had done until now corresponded to taking $p_0=1$.  We now
impose, instead, the weaker requirement that $\dot p_0=0$ (i.e.,  $p_0$ is a
constant) and $p_0\ge 0$.  We then observe that, if we use this new $H$
rather than the old one, then the three conditions of (NCO) are always
satisfied if we make the trivial choice $p(t)\equiv 0$, $p_0=0$. So our new
conditions will give nothing interesting unless we impose a further
{\em nontriviality condition}, stating that this possibility is excluded.  

In our case, it turns out that, if we choose 
$p(t)\equiv -1$ and $p_0=0$, then the adjoint equation 
$\dot p=-\frac{\partial H}{\partial x}$ holds along our candidate trajectory,
and the Hamiltonian $H=pu+p_0u^2$ is maximized, 
on the set $U=[0,+\infty[$, by choosing %]]
$u(t)\equiv 0$.   So our trajectory satisfies the modified 
version of the necessary conditions, with an ``abnormal multiplier''
equal to zero.   

\ssection{A stronger conjecture and a third toy
problem} Conjecture M2 says that, now that we have liberated ourselves 
{}from the constraint that $L$ be differentiable with respect to $u$, it
ought to be possible for $u$---i.e., $\dot q$---to be {\bem anything},
and (NCO) will still work.  Once this is understood, the next natural step
is to allow $\dot q$ to be even ``more arbitrary,'' for example a
general function of some other variable $u$, and of $q$ and $t$.  So
instead of letting $\dot q$ be $u$, we can write $\dot{q}=f(q,u,t)$
for a general function $f(q,u,t)$.  The natural conjecture now is 

\medbreak
\begin{center}
\begin{minipage}{12cm}
\noindent{\bem CONJECTURE  M3:}\ {\em (NCO)  should still be a necessary
condition for 
optimality even for problems where $q$ is restricted 
to satisfy a differential equation $\dot q=f(q,u,t)$, with 
the ``control function'' $t\mapsto u(t)$ taking values 
in some set $U$ and allowed to be a
``completely arbitrary'' $U$-valued function of $t$.} \proofend
\end{minipage}\end{center}


\medbreak\noindent There is a problem, though.  If $u$ is going to take 
values in an arbitrary set $U$, whereas $p$ obviously will still take 
values in $\rr^n$, what should we make of the term $\langle p,u\rangle$ that
occurs in (\ref{defofh})?

There is a fairly natural guess as to what should replace $u$ in the
expression $\langle p,u\rangle$. In the classical context, $u$ was 
$\dot q$, so $\langle p,u\rangle$ was $\langle p,\dot q\rangle$
More precisely, $H(q,u,p,t)$ was the sum of $-L(q,u,t)$ plus 
$p$ times ``$\dot q$ as a function of $u$,'' whatever that means.
In our new setting,  ``$\dot q$ as a function of $u$'' is $f(q,u,t)$,
so one can supplement Conjecture M3 with 

\medbreak
\begin{center}
\begin{minipage}{12cm}
\noindent{\bem SUBCONJECTURE  M3.a:}\ {\em The Hamiltonian $H$ that must be
used to apply (NCO) under the conditions of Conjecture M3 is given by} 
\begin{equation}\label{defofhcontrol}
H(q,u,p,t)=\langle p,f(q,u,t)\rangle-L(q,u,t)\,.
\end{equation}
\end{minipage}\end{center}

\medbreak\noindent As before, we can test this with a simple toy problem 
where the answer can easily be found directly. 
\begin{example}{\rm \label{toy3}
Suppose we have a point in $\rr^2$ with
coordinates $x,y$, moving according to $\dot x=y$, $\dot y=u$, $|u|\le
1$. (Think of $x$ as the position of a point moving in the real line $\rr$.
Then $y$ is the velocity, $u$ is the acceleration, and we are constraining
the motion by imposing a bound on the acceleration.)  Starting at $x=4$,
$y=1$, we want to get to $x=0$, $y=0$ in minimum time.  (That is, we want
to get to the target position $x=0$ and arrive with zero velocity, i.e., we
have a ``soft landing in minimum time'' problem.)
The reader should be able to solve this directly.  (Clearly, since we are
starting off with a velocity vector pointing away from the target, we
should decelerate, i.e., use $u=-1$, at least initially. If we just 
use $u=-1$ all the time we will have $y=1-t$, $x=4+t-\frac{t^2}{2}$,
so we get to $x=0$ at time $t=4$, but then $y=-3$, so we do not soft-land.
So we should use $u=-1$ for a certain time $T$ and then switch to $u=1$,
i.e., slow down, so as to guarantee that $y=0$ when $x=0$.   The time $T$
can be computed exactly, and then one can give a direct proof that the
resulting motion is indeed optimal.)  If we use (NCO) with the 
Hamiltonian defined according to Subconjecture M3.a,  we 
get (since the ``Lagrangian'' is now equal to $1$)
$H=\xi y+\eta u-1$, and then $\dot x=y$, $\dot y=u$, 
$\dot\xi=0$, $\dot\eta=-\xi$, and, finally, $u=1$ if $\eta>0$, $u=-1$ if
$\eta<0$. (Here $(\xi,\eta)$ are the components of the vector $p$. Recall
that in this problem $q$ and $p$ are in $\rr^2$.)
So $\xi(t)$ is a constant, $\eta(t)$ is a linear function of $t$, 
and then our curve must be of one of the types $P$, $N$, $NP$ or 
$PN$, {\bem unless $\eta(t)\equiv 0$}. (Here $P$ means $u\equiv 1$, $N$ means
$u\equiv -1$,  and we are using symbols such as $PN$ to denote
concatenations, as we did in Example \ref{toy2}.)
A $P$ curve can never go from 
$(4,1)$ to $(0,0)$, since along such a curve $y$ will steadily increase.
We have already computed the $N$ curve and shown that it gets us to $x=0$
with $y=-3$, so this possibility is also excluded. 
A $PN$ curve where $u$ switches values at time $t_0$ will get 
us to $(4+t_0+h+t_0h+\frac{t_0^2-h^2}{2},1+t_0-h)$ in time $t_0+h$, and it
is easy to see that this curve does not go through $(0,0)$. 
One can compute an $NP$ curve going from $(4,0)$ explicitly, and this curve 
turns out to be unique. 

Naturally, the curve we have just determined ought to be 
the solution of our minimization problem, and our argument 
seems to have led us to this solution by applying Conjecture M3,
which has thus been verified.

All is not well, however, since {\bem we have not yet disposed of the
possibility that $\eta\equiv{}0$}.  And, unfortunately for us, this could
indeed happen. In fact, if we take $\eta\equiv 0$ and $\xi\equiv 0$, and a
completely arbitrary curve $t\mapsto{} (x(t),y(t))$ satisfying our
constraints, we can easily see that {\em all the conditions of (NCO) hold.}
So, after all our hard work, we end up making the disappointing discovery
that {\bem every curve satisfies the necessary condition of (NCO), so
(NCO) appears to be completely useless as a tool to single out the
solution of our problem.}

At this point it may seem that we ought to abandon all hope and give up on
Conjecture M3.  It turns out, however, that Conjecture M3 is quite sound,
if properly interpreted, and the apparent setback was caused by our
mistaken understanding of its meaning.   In fact, we contend that 
{\em Weierstrass would not have been deterred by this particular
setback}, because Weierstrass knew a trick that takes care of our
problem, a trick that we have already described before, in Remark 
\ref{snott}.  This is how Weierstrass would have reasoned: ``it was {\em our}
mistake that we sought to apply (NCO) to a problem in which the time
interval should be allowed to vary, even though all our necessary
conditions for optimality, including (NCO), were for a {\em fixed} time
interval.  It is of the very essence of a minimum-time problem that the
time interval not be fixed, for a minimum-time problem with a fixed time
interval $[a,b]$ is obviously silly. More precisely, {\em all} arcs are
solutions of such a problem, since they all take the same time, namely,
$b\!-\!a$. So it was to be expected that applying (NCO) would have given us
``all curves'' as the answer, because that is indeed the right answer
for a minimum time problem when the time interval is fixed. 
(NCO) is still fine, and Conjecture M3 has not yet been refuted. It was our
fault that we asked the wrong question.  To apply the fixed-time-interval 
condition (NC0) to a variable-interval problem such as ours, 
we first have to transform our
problem to one with a fixed time interval!$\,\,\,.\,.\,.\,\,\,\,$Actually,
we know very well how to do that, because this is exactly what we were
doing in Remark~\ref{snott} when we transformed a general problem to one
with a Lagrangian satisfying (W).  The thing to do is to introduce a
`pseudotime' parameter $s$, not to be thought of as true physical time,
just making sure that $dt/ds>0$.  So if we write $dt/ds=v$, we have a new
system of equations $dx/ds=vy$, $dy/ds=vu$, where $v>0$ and $|u|\le 1$.
With such a reparametrization, we can always work on a fixed pseudotime
interval, for example $[0,1]$. Now the functional to be minimized is the
integral $\int_0^1v(s)ds$, so our new Lagrangian is $v$.''

Let us pursue this suggestion of our imaginary Weierstrass.  Our equations
are $\dot x=vy$, $\dot y=vu$, with two ``controls'' $u$, $v$, restricted to
$|u|\le 1$, $v>0$.  (The dot now stands for differentiation with respect to
$s$.)  Now the ``control Hamiltonian'' is $H=\xi vy+\eta vu-v$, and (NCO)
gives $\dot x=vy$, $\dot y=vu$, $\dot\xi=0$, $\dot\eta=-\xi v$. The tricky
part now is to maximize $H$.  Since $H=hv$, where $h=\xi y+\eta u-1$, we
see that $h$ must vanish, for if $h\neq 0$ then $H$, as a function of $v$,
would not have a maximum on the set $\{v:0<v<\infty\}$. So $h\equiv 0$.
Now, given $s$, we still have to make sure that $u(s)$ maximizes $H$ as a
function of $u$, with $v=v(s)$. Since $v(s)>0$, maximizing $H$ amounts to
maximizing $h$. Now $h$ is exactly the Hamiltonian of our first, failed
attempt to solve the problem.  So as before we get that our curve must be
$P$, $N$, $NP$ or $PN$.  The argument changes slightly: $\xi$ is still a
constant, but $\dot\eta=-\xi v$ no longer implies that $\eta$ is linear.
However, $\dot\eta=-\xi v$ does imply that $\eta$ is strictly 
increasing---if $\xi<0$---or strictly decreasing---if $\xi>0$---since
$v>0$. Then 
our curve has to be $P$, $N$, $NP$ or $PN$.  In the degenerate case when
$\xi=0$ we get $\eta=c={\rm constant}$, which leads to a $P$ or $N$ curve
as before, unless $c=0$.  {\bf\em But now the possibility that $\xi=\eta=0$
is automatically excluded}, because $\xi=\eta=0$ would imply $h=-1$, which
is out of the question since $h\equiv 0$.  So we get exactly the same
conclusion as in our failed attempt, except that now the ``degenerate
case'' is excluded.  So Conjecture M3 seems to work after all. \proofend
}\end{example}

Those readers who are familiar with optimal control theory will of course 
have recognized Conjecture M3 as being essentially\Ft{\label{essen} We say 
``essentially''  for a good reason. We will get to it soon.}
the same thing as the celebrated ``Pontryagin maximum principle.''

And we hope to have convinced {\em all} readers, even those who are not
control theorists, that (NCO) is a very natural condition.  It should be
clear from our discussion that (NCO) could have been guessed almost
immediately from ``Hamilton's equations as Hamilton should have written
them,'' together with the Legendre condition, and would have been an almost
obvious conjecture to make once the Weierstrass side condition is known,
{\em provided that one uses the ``correct'' Hamiltonian formalism, as in
(\ref{defofh}) and (\ref{hamileqsys})}.

\medskip
\begin{remark}{\rm \label{notwithham} Notice that what we did in 
Example \ref{toy3}---i.e., transforming a problem into one with a fixed
time interval and a Lagrangian that satisfies Condition (W), and then using
(NCO)---{\em could 
not possibly have been done with Hamilton's Hamiltonian}.  Indeed, the
condition needed for the classical Hamilton equations to be possible is the
solvability of (\ref{pimpl}) for $\dot q$ as a function of $q$, $p$, and
$t$.  This condition {\em never} holds when (W) holds because, if $L$ is
positively homogeneous of degree 1 with respect to $\dot q$, then
$\frac{\partial L}{\partial\dot q}$ is positively homogeneous of degree 0,
i.e., constant along rays in $\dot q$ space. So the map $\dot q\mapsto{}
\frac{\partial L}{\partial\dot q}(q,\dot q,t)$ is not one-to-one locally,
in the neighborhood of any point, for any $q,t$.  Therefore Weierstrass'
use of Lagrangians with property (W), while involving ``no loss of
generality,'' had the effect of putting him in a situation where the use of
a Hamiltonian formalism, as was known at the time, was {\em a priori}
excluded.  The ``control'' Hamiltonian formalism, on the other hand, would
have worked perfectly. \proofend 
}\end{remark} 

\scholium{\snell, Descartes, Fermat, Leibniz, Huygens on 
refraction}\label{schol-snell}

\def\snellfoot{The Dutch mathematician 
{\bem Willebrord van Roijen Snell} (1580-1626), 
studied law at the University of Leiden, the Netherlands,
where his father, Rudolph Snell (1546-1613), was professor of mathematics.
Willebrord's  true
interests were mathematics and astronomy, and he 
became a professor of mathematics at the University
of Leiden in 1613, succeeding his father. He devised techniques for 
for measuring the Earth, and in particular the method of triangulation.
He developed new approaches for calculating approximate values
of $\pi$. Using his method with polygons of $96$ sides
he got a correct value of $\pi$ to $7$ decimal places. 
He studied navigation, and described the curve on the sphere 
---called the {\bem loxodrome}---that intersects the meridians at equal
angles.}

The famous ``\snell{} law'' of refraction was discovered
by \snell\Ft{\snellfoot}{} in 1621, but he did not publish it at the time.
The law only become and only in 1703 did it become known
when Huygens published \snell's result in Dioptrica. \snell{} also discovered
the sine law.

Descartes published the law of refraction in 1637 in his {\bem Dioptrique},
together with his own explanation.  Fermat was strongly opposed to this law
at first, claiming that Descartes' own deduction was incorrect, and indeed
Descartes' argument was wrong, because his theory required that light
travel {\it faster} in a denser medium.

Subsequently, Fermat accepted the law, of which he gave a derivation based
on the minimum-time principle: {\bem in a medium with variable speed of light,
the light rays are paths that minimize travel time from the intial point to
the terminal point}. Fermat's principle yielded the correct law of
refraction only if light was assumed to travel more slowly in a denser
medium.  (Later, in 1690, C.~Huygens 
(1629-1695) provided a justification of Fermat's
principle by deriving it from wave optics.)

Descartes was very angry at Fermat's criticism, and strongly attacked
Fermat's methods of maxima, minima and tangents. After Roverbal, Etienne
Pascal\Ft{The father of Blaise Pascal} and Desargues intervened in the
dispute, Descartes eventually admitted that Fermat was right, and wrote

\bqrm
 $\ldots$ ... seeing the last method that you use for finding tangents to
curved lines, I can reply to it in no other way than to say that it is 
very good and that, if you had explained it in this manner at the outset, I
would have not contradicted it at all.
\eqrm

\noindent Nevertheless, Descartes went on attacking Fermat and trying to
damage his reputation, for example by writing a letter to Mersenne
saying that Fermat's determination of the tangent to a cycloid 
was incorrect, and that Fermat was not a good mathematician,
even though Fermat's work was correct and Descartes had said
so in a letter to Fermat.

A readable summary of the early history of the refraction problem 
(not mentioning the contributions of \snell{} and Descartes) is given
by none other than Johannm Bernoulli in his 1967 brachytochrone 
paper \cite{bernoulli-a}. We quote from Struik \cite{struik}, p.~393':

\bqrm
Fermat, in a letter to De la Chambre, has shown that a light ray passing
from a thin to a more dense medium is bent toward the perpendicular in such
a way that, under the supposition that the ray moves continuously 
from the light to the illuminated point, it follows the path that requires
the shortest time. With the aid of these principles he showed that 
the sine of the angle of incidence and the sine of the angle of refraction
are in inverse proprtion to the densities of the media, hence directly
as the velocities with which the light penetrates those media. 
Later Leibniz $\ldots$ and soon afterward the famous Huygens in his
{\bem Treatise on light}, p~40, have demonstrated this more comprehensively
and, by most valid arguments, have established the physical, or better the
metaphysical, principle which Fermat seems to have abandoned at the
insistence of Clerselier, remaining satisfied with his geometric proof and
giving up his rights all too lightly. 
\eqrm

The work of Leibniz alluded to by Johann is his his 1683 {\bem Acta
Eruditorum} paper \cite{leibniz-a}, on a unified foundation for optics,
catoptrics and dioptrics, where the great philosopher announced for the first
time that he had a method for solving problems of maxima and minima, of
such power that, according to Leibniz' own assessment in his later
paper \cite{leibniz-b}, 

\bqrm
Other very learned men have sought in many devious ways what someone versed
in this calculus can accomplish in these lines as by magic. $\ldots$ This 
is the beginning of much more sublime Geometry, pertaining to even the
most difficult and most beautiful problems of applied mathematics,
which without our differential calculus or something similar 
noone could attack with any such ease.  [Quoted from Struik 
\cite{struik}, p.~279.]
\eqrm

Subsequently, in 1684, Leibniz published a much longer paper \cite{leibniz-b},
with the title 

\bqrm
A NEW METHOD FOR MAXIMA AND MINIMA AS WELL A TANGENTS, WHICH IS NEITHER 
IMPEDED BY FRACTIONAL NOR IRRATIONAL QUANTITIES, AND A REMARKABLE TYPE OF
CALCULUS FOR THEM.
\eqrm

\noindent This paper gave a more detailed account of the method, as well 
as many examples. Here we shall focus on one example of particular interest
to us, because it involves a {\bem curve minimization problem.} 

Leibniz proposes to solve the following problem:

\bqrm
Let two points $C$ and $E$ be given and a line $SS$ in the same plane. 
It is required to find a point $F$ on $SS$ such that when $C$ and $E$ are
connected the sum of the rectangle of $CF$ and a given line $h$ and the 
rectangle of $FE$ and a given line $r$ are as small as possible. In other
words, if $SS$ is a line separating two media, and $h$ represents the
density of the medium on the side of $C$ (say water), $r$ that of the
medium on the side of $E$ (say air), then we ask for the point $F$ such
that the path from $C$ to $E$ via $F$ is the shortest 
possible.  [Quoted from Struik \cite{struik}, p.~278.]
\eqrm

\noindent By ``the rectangle of $CF$ and a given line $h$'' Leibniz means 
``the area of a rectangle with base $CF$ and height $h$,'' so the problem 
he is solving is that of making the sum 
$$
T=h|CF|+r|FE|
$$
as small a possible, subject to the condition that $F$ belong to the given
line $SS$. Leibniz then applies his result to dioptrics and 
finds that 
\bqrm
the sines of the angles of incidence and of refraction $\ldots$ are 
in inverse ratio to $r$ and $h$, the densities of the media in which the
incidence and the refarction take place.
\eqrm
Since these densities represent ``the resistance with which the light rays
meet,'' they are inversely proportional to the speed of propagation of
light in the two media, so Leibniz' solution is in fact the familiar
law of \snell{} about refraction.   


\scholium{Two excerpts from G. Adair's English translation of Georges 
Perec's ``La Disparition''}\label{schol-adair}

Georges Perec's remarkable 285-page novel ``La Disparition'' does not
contain a single letter ``e.''  The book has been translated into
Engish by G. Adair, under the title``A Void.''  Here are two excerpts
from the English translation, the second one of which contains the
passage about Pontryagin cited in Footnote \ref{cryptic}.


\begin{Quoterm}

\noindent Call him Ishmail, too. Having had to sustain many, almost inhuman
hardships, this Ishmail drops anchor on an island that is, at first
sight, totally without inhabitants.  Initially, things go badly, with
his almost dying of cold and starvation, burrowing into a sand pit,
shaking, suffocating, coming down with malaria, curling up both day
and night in a tight ball, simply waiting to pass away.

$\quad$ But six days go past, and his unusually robust constitution allows
him finally to sit up and look around him. Our protagonist is now
disturbingly skinny but can still, if with difficulty, climb out of
his pit, a pit that was almost his tomb. So Ishmail starts living
again by first luxuriantly shaking his thirst; now swallows an acorn
but instantly spits it out in disgust; and soon works out which fruits
and mushrooms won't do him any harm.  A particular fruit (akin to an
apricot) brings him out in a rash of itchy, purplish spots, but 
bananas, avocados, nuts and kiwi fruit abound.

$\quad$ At dusk, using a sharp rock, Ishmail cuts a notch on a stick (to tick
off his first night as a castaway); and, with a total of 20 such cuts,
constructs a hut, a sort of impromptu shack, with a door, four walls,
and flooring and roofing built out of mud.  With no matchsticks at
hand, his food is invariably raw. Constant, too, is his panic that an
animal might attack his camp; but, as it turns out (such, anyway, is
his supposition), no lynx or puma, no jaguar, cougar or bison, stalks
his island.  At most, comfortably far off, a solitary orangutan
prowling around but not caring (or daring?) to approach him. 

\medbreak\hfill (\cite{perecad}, page 17)

\end{Quoterm}

\bigbreak

\begin{Quoterm}

\noindent Who first had this particular notion, who brought it to its maturity,
who found a solution to it? Was is Gauss or Galois?  Nobody could
say. Nowadays, though, all of us know. But it's said that, just prior
to dying, at night, at about 4 or 5 p.m., Galois put in writing on his
jotting pad ... a long, continuous chain of factors in his own form of
notation..  As part of his manuscript is missing, nobody knows to this
day what conclusions Galois was hoping to draw from his calculations.

$\quad$ Cantor, Douady and Bourbaki thought, on many and various grounds (from
algorithms to topoi, from M\"obius strips to C-star, from Shih's
K-functor to Thom's \raise3pt\hbox{\fbox{$\,$}$\,$}s, 
and including all sorts of
distributions, convolutions, Schwartz, Koszul, Cartan and Giorgiutti)
of following a hunch that would surmount such an abrupt hiatus. It was,
alas, all in vain. 

$\quad$ Though it took him from 1935 to 1955, working at it virtually nonstop,
Pontryagin finally had to admit to accomplishing nothing.
	
\medbreak\hfill (\cite{perecad}, pages 46-47)

%GNU: check these page numbers


\end{Quoterm}

\cchapter{The maximum principle of optimal control theory\label{themaxprin}}

\noindent With Conjecture M3
adjusted\Ft{It is because Conjecture M3 is false 
unless this adjustment is made that we used the word ``essentially''
in Footnote \ref{essen}.} with the introduction of the new variable 
$p_0$ we have, finally, reached the justly celebrated 
{\bem Maximum Principle}: 

\medbreak


\noindent{\bf (MP)} {\em Consider a standard point-to-point
optimal control problem 
\begin{equation}\label{standardcontrol-repi}
\fbox{\fbox{$
\left.\begin{array}{lcl}
{\rm Given}&\,\,\,&Q,\,U,\,f,\,L,\,a,\,b,\,\bar q,\,\hat q,\,\mathcal{C}\,,\\
{\rm minimize}&\,\,\,&I=\int_a^bL(\xi(t),\eta(t),t)dt\,,\\
{\rm subject \,\,\,to}&\,\,\,\,&\xi(a)=\bar q\,,\,\,\xi(b) =\hat q\,,\\
\,\,\,&\,\,\,\,&(\xi,\eta)\in\mathcal{C}\,,\\
{\rm and}&\,\,\,\,&\dot\xi(t)=f(\xi(t),\eta(t),t)\,\,{\rm for}\,\,\,
a\le t\le b\,,
\end{array}\right.$}}
\end{equation}
where 

\begin{itemize}
\vspace{-2mm}
\item[1.] $Q$ is an open subset of $\rr^n$, 

\vspace{-3mm}
\item[2.] $U$ is a set,

\vspace{-3mm}
\item[3.] $a,b\in\rr$ and $a\le b$,

\vspace{-3mm}
\item[4.] $\bar q\in Q$ and $\hat q\in Q$, 

\vspace{-3mm}
\item[5.] $Q\times U\times[a,b]\ni(q,u,t)\mapsto(f(q,u,t),
L(q,u,t))\in\rr^n\times\rr$ is a map,
\end{itemize}
Then, under suitable technical conditions,  if
a pair $(\xi_*,\eta_*)\in\mathcal{C}$ 
is such that $[a,b]\ni t\mapsto\eta_*(t)\in U$ is a control
and the curve $[a,b]\ni t\mapsto\xi_*(t)\in Q$ is a corresponding solution of 
(\ref{dyna}), a necessary condition for $(\xi_*,\eta_*)$ 
to solve the minimization problem is that
there exist a ``momentum'' function 
$[a,b]\ni t\mapsto\pi(t)\in\rr_n$ and a constant $\pi_0\ge 0$
such that
\bi\item[]\bi
\item[(NT)] $\quad(\pi(t),\pi_0)\neq (0,0)$ for all $t\in[a,b]$;
\item[(HS)] $\quad\dot\xi_*(t)=\frac{\partial H}{\partial
p}(\Xi_*(t))$ and $\quad-\dot\pi(t)=\frac{\partial H}{\partial
q}(\Xi_*(t))$ for $t\in[a,b]$;
\item[(MC)] $\quad H(\Xi_*(t))={\rm max}_{u\in U}H(\xi_*(t),u,\pi(t),\pi_0,t)$
for $t\in[a,b]$, 
\ei\ei 
where we write  $\Xi_*(t)=(\xi_*(t),\eta_*(t),\pi(t),\pi_0,t)$, 
and the Hamiltonian \mbox{function}
$Q\times U\times\rr_n\times\rr\times[a,b]\ni(q,u,p,p_0,t)\mapsto
H(q,u,p,p_0,t)\in\rr$ is given by 
\begin{equation}\label{hamilmaxiprin-gg}
H(q,u,p,p_0,t)=\langle p,f(q,u,t)\rangle - p_0 L(q,u,t)\,.
\end{equation}
}


\bigbreak\noindent The three
statements (NT), (HS) and (MC) are known, respectively,
as the {\bem \mbox{nontriviality} condition}, the {\em Hamiltonian system},
and the {\bem  Hamiltonian 
maximization condition}.  Notice that (HS) is just a restatement of
(\ref{nco1}) and (\ref{nco2}), with the new $H$, and (MC) is a restatement
of (\ref{nco3}).  The second equation of (HS) is called the {\em adjoint
equation}.

We will refer to pairs $(q(\cdot),u(\cdot))$---where $t\mapsto u(t)$ is a
$U$-valued function on $[a,b]$ and $q(\cdot)$ is a solution of
(\ref{dyna})---as {\em trajectory-control pairs}.  (Many authors use
the much less informative phrase ``admissible pairs.'')  A
trajectory-control pair that satisfies the condition of (MP)---i.e., is
such that there exist $p$ and $p_0\ge 0$ for which (NT), (HS) and (MC)
hold---will be called an {\em extremal}.  So the maximum principle says
that a necessary condition for a trajectory-control pair to be a solution
of the minimization problem is that it be an extremal. 

For a given trajectory-control pair
$\gamma_*=(q_*,u_*)$, a vector-valued function $t\mapsto p_*(t)$ for which
Conditions (NT), (HS) hold for some $p_0\ge 0$ will be called an {\em
adjoint vector\Ft{For differential geometers: 
{\mit p} is really a {\bem field of covectors} along 
{\mit q$_*$}.}} for $\gamma_*$. If, in addition, $p_0$ can be chosen so
that (MC) holds as well, then $p_*$ will be called a {\em minimizing
adjoint vector} for $\gamma_*$.  Often, the whole pair $(p_*,p_0)$ is also
referred to as an adjoint vector, or a minimizing adjoint vector. We will
refer to a triple $(q_*,u_*,p_*)$ (or a quadruple $(q_*,u_*,p_*,p_0)$) such
that $(q_*,u_*)$ is a trajectory-control pair and $p_*$ (or $(p_*,p_0)$) is
a minimizing adjoint vector for $(q_*,u_*)$ as an {\em extremal lift} of
$(q_*,u_*)$.  A extremal lift $(q_*,u_*,p_*,p_0)$ of a pair $(q_*,u_*)$ is
{\em normal} if $p_0\neq 0$, and {\em abnormal} if $p_0=0$.  An extremal is
{\em normal} if it has a normal extremal lift, and {\em abnormal} if it has
an abnormal extremal lift.

Some authors use ``extremal'' for what we are calling ``extremal lift''
here.  Naturally, for those authors the categories of normal and abnormal
extremal are mutually exclusive.  This is not so with our terminology: it
can happen that an extremal is both normal and abnormal, since the extremal
lift need not be unique.

When the abnormal multiplier $p_0$ is $\neq\!\!0$, i.e., $>0$, it can always be
set equal to $1$, since multiplication of both $p_*$ and $p_0$ by a
strictly positive constant leaves all the conditions of (MP) unchanged.  As
long as $p_0\!=\!1$, (MP) is exactly the same as our previous Conjecture M3. So
the only possibility that has been added now is the case $p_0=0$. In other
words, what the maximum principle asserts is that a solution of the
minimization problem problem always satisfies either (i)~the necessary
condition given by Conjecture M3 as stated, or (ii)~the necessary condition
given by Conjecture M3 with $L\equiv 0$ and the extra requirement that $p$ be
$\neq 0$.

The reader will notice that in Examples \ref{toy1}, \ref{toy2} and
\ref{toy3} one can always either exclude the possibility that $p_0=0$ or
show that it leads to nothing new. Indeed, in Examples \ref{toy1} and
\ref{toy2} the Hamiltonian for $p_0=0$ is $H=p.u$, so the adjoint equation
is $\dot p=0$.  Hence $p$ is a constant, which cannot equal $0$ by (NT).
So $u(t)\equiv 1$ or $u(t)\equiv-1$, and in either case the endpoint
conditions have to be violated.  In Example \ref{toy3} (properly modified
as suggested by our imaginary Weierstrass), the abnormal Hamiltonian is
$H=\xi vy+\eta vu$, and $H=hv$, with $h=\xi y+\eta u$.  Exactly as in our
analysis of Example \ref{toy3}, we see that $h$ has to vanish, and this
implies that our curve is of one of the forms we have already found, unless
we are in the singular case $\xi\equiv \eta\equiv 0$.  In the discussion
of Example \ref{toy3},
the singular case was excluded because it led to $h=-1$.  Though this no
longer works now, the singular case is still excluded anyhow, because of
(NT).

As for Example \ref{toy4}, it is easy to see that the trajectory $x_*$
given by $x_*(t)=1$---which is the only one that satisfies the boundary
conditions and the constraint on $\dot x$---is an abnormal extremal.
Indeed, the abnormal Hamiltonian is $H=pu$, and then the adjoint equation
is $\dot p=0$.  We can then choose $p_*(t)\equiv -1$, and with this choice
$u\mapsto H(x_*(t),u,p_*(t),0,t)$ is maximized by $u=0$.

Finally, we remark that for classical calculus of variations problems (MP)
yields exactly the same conclusion as (NCO). Indeed, in this case we can
exclude the possibility that $p_0=0$, because the abnormal Hamiltonian is
$\langle p,u\rangle$, which is a linear function of $u$. Such a linear
functional, if its domain is the whole space, can never have a maximum,
unless $p=0$. But $p=0$, together with $p_0=0$, is excluded by (NT).

So (MP), as stated, is a true generalization of the necessary conditions
(NCO), which covers many cases that cannot be handled by means of the
classical calculus of variations, as has been seen in Examples \ref{toy1},
\ref{toy2}, \ref{toy3} and \ref{toy4}.

We conclude by presenting the analogue of (MP) for problems with a variable
time interval:

\medbreak
\begin{center}
\begin{minipage}{12cm}
\noindent{\bem (MP')}\ {\em For a minimization problem of the kind
discussed in 
(MP), but with the time interval $[a,b]$ not fixed in advance, assuming
that $f$ and $L$ do not depend on $t$, the necessary conditions are exactly
the same as those of (MP), plus the extra requirement that
$H(q_*(t),u_*(t),p_*(t),p_0)\equiv 0$.}\proofend
\end{minipage}\end{center}

\medbreak\noindent This is easily derived from (MP), by means of the method
that we used in our analysis on Example \ref{toy3}.  One has to transform
the given problem to a fixed-time problem by means of a time
reparametrization.  What was done in Example \ref{toy3} works in general,
so we will not repeat the argument here.

Statement (MP') applies in particular to {\em minimum-time problems}, i.e.,
problems where $L\equiv 1$.

\ssection{Example I: the straight-line segment}\label{subsec-segment}

\gnu{JAN: This section still has to be written. If you want an idea of
what I want to put in, look at the corresponding section in
my French paper (item no. 58 in the ``papers available on the Web''
link of my Web page), where I dicuss the ``rope-stretching problem''
and related stuff.}

\ssection{Example II: the isoperimetric problem}\label{subsec-isoper}. 

As explained in \S\ref{subsec-era-spec}.2, the question is 

{\large\em
\begin{itemize}
\item[(IP)] Of all plane closed curves of a given 
length $L$, find those that enclose the largest possible area,
\end{itemize}}
or, equivalently, 
{\large\em
\begin{itemize}
\item[(IP')] Of all plane closed  curves 
that enclose a given area $A$, find those of minimum length.
\end{itemize}}

To formulate the problem mathematically, we 
have to make precise sense of the concepts of
``closed curve,'' the ``length''
of a curve, and the area ``enclosed'' by a closed curve.

A {\em closed curve} in $\rr^2$ is a continuous map
$\xi:[a,b]\mapsto\rr^2$, defined on some interval
$[a,b]$, such that $\xi(a)=\xi(b)$. 

Curves that have a well defined length are called ``rectifiable.''
(This concept is defined precisely in \S GNU below.) The ``area enclosed
by a closed curve $\xi$'' is the area of the region 
enclosed by the curve. The concept of ``area'' of a 
``region'' $\Omega$ is relatively unproblematic, at least if $\Omega$
is an open bounded set, since we can define the area 
of $\Omega$ to be its Lebesgue measure, which could also be characterized
in a more elementary way as the supremum of all
the sums $\sum_{j=1}^N{\rm Area}(R_j)$, taken over all 
$N$ and over all finite sequences $R_1,\ldots,R_N$ of pairwise
disjoint rectangles contained in $\Omega$. 

On the other hand, the concept of ``region enclosed
by a closed curve'' turns out to be suprisingly delicate. The curves
we are interested in for the IP are {\em simple} closed curves, that is,
closed curves $\xi$ without double points. (This means that,
if $\xi:[a,b]\mapsto\rr^2$, then $\xi$ is a one-to-one
map on the interval $[a,b\,[\,$; in other words, if $t_1,t_2\in[a,b]$,
$t_1\le t_2$ and $\xi(t_1)=\xi(t_2)$, then either
(1)~$t_1=t_2$ or (2)~$t_1=a$ and $t_2=b$.)
It is intuitively clear that a simple closed curve $\xi$
divides the plane \mbox{into} \mbox{two} regions, the ``inside''
${\rm Ins}(\xi)$ and the ``outside'' ${\rm Out}(\xi)$. 
However, it is not \mbox{completely} trivial how to make 
this seemingly evident statement precise, because 
to do so one has to make sense first of the word ``divides.''
But this is relatively easy: one defines a
{\em connected component} of an open set 
$\Omega$ to be a subset $C$ of $\Omega$ which is, for some $c\in\Omega$,  
the set of all points of $\Omega$ that can be joined to $c$ by some
continuous path in $\Omega$; it is then easy to see 
that two connected components of $\Omega$ are either
disjoint or equal. The statement about ``dividing''
the plane can then be reformulated by saying that

\begin{itemize}\item[]\begin{itemize}
\item[(JCT)]
{\em If $\xi:[a,b]\mapsto\rr^2$ is a simple closed curve,
and $|\xi|$ denotes the set $\{\xi(t):a\le t\le b\}$, then
$\rr^2\backslash|\xi|$ has exactly two connected components.}
\end{itemize}\end{itemize}

\medbreak\noindent Assertion (JCT) is the {\em Jordan curve
theorem.} Although it is intuitively obvious, as we have explained, 
the proof is suprisingly difficult. (We present a complete
self-contained proof in \S GNU below.) 

If we grant the truth of (JCT), then we can define 
${\rm Ins}(\xi)$ and ${\rm Out}(\xi)$ precisely. Indeed, if
$r$ is large enough (such that $\|\xi(t)\|\le r$ for all $t$), then one 
of the two connected components of 
$\rr^2\backslash|\xi|$ must contain the set
$S_r=\{(x,y):x^2+y^2>r^2\}$, since $S_r\cap|\xi|=\emptyset$,
and any two points of $S_r$ can be joined by a continuous path in
$S_r$; the component that contains $S_r$ is obviously 
unbounded; the other component will then not intersect
$S_r$, so it will be bounded; we define 
${\rm Ins}(\xi)$ and ${\rm Out}(\xi)$ to be, respectively,
the bounded and unbounded components 
$\rr^2\backslash|\xi|$.

Now that we know what ``${\rm Ins}(\xi)$'' means, we 
can use use the well-known ``Green's  formula''
\begin{equation}\label{greenformula}
{\rm Area}\Big(\,{\rm Ins}(\xi)\,\Big)=|A(\xi)|\,,
\qquad{\rm where}\qquad
A(\xi)=\frac{1}{2}\int_{\xi}(x\,dy-y\,dx)
\end{equation} 
for the area of ${\rm Ins}(\xi)$.
(Actually, the area of ${\rm Ins}(\xi)$ is $A(\xi)$ 
if the simple closed curve $\xi$ is ``oriented counterclockwise.''
If $\xi$ is ``oriented \mbox{clockwise},'' then 
$A(\xi)$ is negative, and the area of ${\rm Ins}(\xi)$ 
is $-A(\xi)$, i.e., $|A(\xi)|$. The issue of \mbox{orientation} 
will be discussed in detail in \S GNU.)

Naturally, (\ref{greenformula}) requires proof, and
the proof is not trivial\Ft{Actually, even 
assigning a precise meaning to (\ref{greenformula}) 
requires some work.  Indeed, once we have given
the precise definition of ``rectifiable curve,''  
it will turn out that $\xi$ is \mbox{rectifiable}
if and only if it is continuous and 
of bounded variation. Then the right-hand side of the formula
defining $A(\xi)$ makes sense as a Riemann-Stieltjes
integral.}. We will give a rigorous proof in
\S GNU, but here we will just accept the validity 
of (\ref{greenformula}).

The last point that needs to be dealt with is that of writing
the length of $\xi$ as an integral.  It turns out 
that every rectifiable curve $\xi$ admits a reparametrization
of time that transforms it into an absolutely \mbox{continuous}
curve, without changing the length or the area.
This means that we might as well limit ourselves to working
with absolutely continuous curves\Ft{In general,
$W^{k,p}(\Omega,\rr^n)$ denotes the Sobolev space of all
$\rr^n$-valued functions $f$  on $\Omega$ such that
$f$ and all the distributional derivatives of
$f$ of order $\le k$ are in $L^p$. In particular,
$W^{1,1}([a,b],\rr^n)$ consists of the functions
having an integrable distributional derivative,
i.e., of the absolutely continuous functions.
So we are choosing $W^{1,1}([a,b],\rr^n)$ as our 
space of curves.}

Let us use ``AC'' as an abbreviation for ``absolutely continuous.''
If $\xi$ is AC, then the length of $\xi$ is just the integral
$\int_a^b\|\dot\xi(t)\|\,dt$. If $n=2$, and we write
$\xi(t)=(x(t),y(t))$, then the length of $\xi$ is 
equal to \mbox{$\int_a^b\sqrt{\dot x(t)^2+y(t)^2}\,dt$}. 

\medbreak
We are now ready to formulate the isoperimetric problem as follows,

\begin{center}
\fbox{\fbox{\begin{minipage}{11cm}
\null\vspace{1mm}
\begin{center}
{\bf THE ISOPERIMETRIC PROBLEM, VERSION I}
\end{center}
$$
\left.\begin{array}{ll}
{\rm Given}&{\rm a\,\,number}\,\, L\,\,{\rm such\,\,that}\,\,L>0\,,\\
{\rm maximize}&\int_0^1\Big(\,x(t)\dot y(t)-y(t)\dot x(t)\,\Big)\,dt\\
{\rm subject\,\,\,to}&\int_0^1\sqrt{\dot x(t)^2+\dot y(t)^2}\,dt=L\,,\\
\,\,\,&x(1)=x(0)\,,\,\,y(1)=y(0)\,,\\
\,\,\,&{\rm the\,\,\,curve}\,\,(x(\cdot),y(\cdot))\,\,{\rm is\,\,simple\,\,
and\,\,AC}\,.
\end{array}\right.
$$
\end{minipage}}}
\end{center}
(We do not include the requirement that the curve should be ``oriented
\mbox{counterclockwise}'' because for simple curves $\xi$ that are oriented
clockwise the integral of (\ref{greenformula}) is negative,
so these curves are automatically eliminated as possible solutions.)

Alternatively, we can use the equivalence between Problems (IP) and (IP')
above to cast our problem as one of minimizing the perimeter 
given the enclosed area:

\begin{center}
\fbox{\fbox{\begin{minipage}{11cm}
\null\vspace{1mm}
\begin{center}
{\bf THE ISOPERIMETRIC PROBLEM, VERSION II}
\end{center}
$$
\left.\begin{array}{ll}
{\rm Given}&{\rm a\,\,number}\,\, A\,\,{\rm such\,\,that}\,\,A>0\,,\\
{\rm Minimize}&\int_0^1\sqrt{\dot x(t)^2+\dot y(t)^2}\,dt\,,\\
{\rm subject\,\,\,to}&\int_0^1\Big(\,x(t)\dot y(t)-y(t)\dot x(t)
\,\Big)\,dt=A\,,\\
\,\,\,&x(1)=x(0)\,,\,\,y(1)=y(0)\,,\\
\,\,\,&{\rm the\,\,\,curve}\,\,(x(\cdot),y(\cdot))\,\,{\rm is\,\,simple
\,\,and\,\,AC}\,.
\end{array}\right.
$$
\end{minipage}}}
\end{center}

\medbreak\noindent Versions I and II are ``isoperimetric problems''
in the precise sense of \S\ref{sect-in-isop}. As explained in
\S\ref{isoper-are-standard}, such problems can be transformed
into optimal \mbox{control} problems of the standard form,
by adding extra state variables for the 
\mbox{constraints}. If we do that for our two versions, we arrive at

\begin{center}
\fbox{\fbox{\begin{minipage}{11cm}
\null\vspace{1mm}
\begin{center}
{\bf THE ISOPERIMETRIC PROBLEM, VERSION III}
\end{center}
$$
\left.\begin{array}{ll}
{\rm Given}&{\rm a\,\,number}\,\, L\,\,{\rm such\,\,that}\,\,L>0\,,\\
{\rm maximize}&\int_0^1\Big(\,x(t)v(t)-y(t)u(t)\,\Big)\,dt\,,\\
{\rm subject\,\,\,to}&\dot x(t)=u(t)\,,\\
\,\,\,&\dot y(t)=v(t)\,,\\
\,\,\,&\dot z(t)=\sqrt{u(t)^2+v(t)^2}\,,\\
\,\,\,&x(1)=x(0)\,,\,\,y(1)=y(0)\,,\,\,z(0)=0\,,\,\,z(1)=L\,,\\
\,\,\,&{\rm the\,\,\,curve}\,\,(x(\cdot),y(\cdot))\,\,{\rm is\,\,simple}\,,\\
\,\,&{\rm the\,\,\,functions}\,\,x(\cdot),\,y(\cdot),\,z(\cdot)\,\,
{\rm are\,\,AC}\,.\end{array}\right.
$$
\end{minipage}}}
\end{center}

\noindent and

\begin{center}
\fbox{\fbox{\begin{minipage}{11cm}
\null\vspace{1mm}
\begin{center}
{\bf THE ISOPERIMETRIC PROBLEM, VERSION IV}
\end{center}
$$
\left.\begin{array}{ll}
{\rm Given}&{\rm a\,\,number}\,\, L\,\,{\rm such\,\,that}\,\,L>0\,,\\
{\rm minimize}&\int_0^1\sqrt{u(t)^2+v(t)^2}\,dt\,,\\
{\rm subject\,\,\,to}&\dot x(t)=u(t)\,,\\
\,\,\,&\dot y(t)=v(t)\,,\\
\,\,\,&\dot z(t)=x(t)v(t)-y(t)u(t)\,,\\
\,\,\,&x(1)=x(0)\,,\,\,y(1)=y(0)\,,\,\,z(0)=0\,,\,\,z(1)=A\,,\\
\,\,\,&{\rm the\,\,\,curve}\,\,(x(\cdot),y(\cdot))\,\,{\rm is\,\,simple}\,,\\ 
\,\,\,&{\rm the\,\,\,functions}\,\,x(\cdot),\,y(\cdot),\,z(\cdot)\,\,
{\rm are\,\,AC}\,.\end{array}\right.
$$
\end{minipage}}}
\end{center}

As an alternative to Version III, we could reparametrize
the plane curve $t\mapsto(x(t),y(t))$ by arc length. If we do this,
then of course we can no longer take our curves to be defined
on the interval $[0,1]$, and have to work on $[0,L]$ instead.
The function $z(\cdot)$ becomes unnecessary, since now
$z(t)$ must be equal to $t$. 

In addition, it si clear that any translate of a curve
$\xi$ has the same length and encloses the same area as
$\xi$. We can therefore impose the extra restriction
that $x(0)=0$ and $y(0)=0$. We end up with 

\begin{center}
\fbox{\fbox{\begin{minipage}{11cm}
\null\vspace{1mm}
\begin{center}
{\bf THE ISOPERIMETRIC PROBLEM, VERSION V}
\end{center}
$$
\left.\begin{array}{ll}
{\rm Given}&{\rm a\,\,number}\,\, L\,\,{\rm such\,\,that}\,\,L>0\,,\\
{\rm maximize}&\int_0^L\Big(\,x(t)v(t)-y(t)u(t)\,\Big)\,dt\,,\\
{\rm subject\,\,\,to}&\dot x(t)=u(t)\,,\\
\,\,\,&\dot y(t)=v(t)\,,\\
\,\,\,&u(t)^2+v(t)^2=1\,,\\
\,\,\,&x(L)=x(0)=0\,,\,\,y(L)=y(0)=0\,,\\
\,\,\,&{\rm the\,\,\,curve}\,\,(x(\cdot),y(\cdot))\,\,{\rm is\,\,simple
\,\,and\,\,AC}\,.\end{array}\right.
$$
\end{minipage}}}
\end{center}

\noindent This is, again, a two-dimensional problem.  It is close to belonging
to the class of calculus of variations problems, but does not quite 
qualify, because the curves $(x(\cdot),y(\cdot))$ have to satisfy
the additional ``velocity constraint'' 
\mbox{$\dot x^2+\dot y^2=1$}. 

\subsection{Application of the maximum principle}

From now on, $L$ is a fixed positive real number. We let $\Gamma_L$  
be the set of all curves $\gamma:[0,L]\mapsto\rr^2$ such that

\begin{itemize}
\item[1.] $\gamma$ is absolutely continuous, 
\item[2.] $\|\dot\gamma(t)\|=1$ for almost all $t\in[0,L]$,
\item[3.] $\gamma(L)=\gamma(0)=(0,0)$,
\item[4.] $\gamma$ is simple (that is, the restriction of $\gamma$ to
the half-open interval $[0,L[$ is one-to-one),
\end{itemize}

For $\gamma\in\Gamma_L$, we define
$$
A(\gamma)\equaldef\frac{1}{2}\int_0^L(x(t)\dot y(t)-y(t)\dot x(t))\,dt\,,
$$
where we are writing $\gamma(t)=(x(t),y(t))$.

So the isoperimetric  problem (Version V)
can be restated as follows:

{\Large\em 
\begin{itemize}\item[]\begin{itemize}
\item[(IP.V)] Find a curve $\gamma_*\in\Gamma_L$ 
that maximizes
the functional $\Gamma_L\ni\gamma\mapsto A(\gamma)\in\rr$.
\end{itemize}\end{itemize}}

We associate to (IP.V) another problem involving a larger
class of curves. We let $\tilde\Gamma_L$ 
be the set of all curves $\gamma:[0,L]\mapsto\rr^2$ such that

\begin{itemize}
\item[1.] $\gamma$ is absolutely continuous, 
\item[2.] $\|\dot\gamma(t)\|\le 1$ for almost all $t\in[0,L]$,
\item[3.] $\gamma(L)=\gamma(0)=(0,0)$,
\end{itemize}
and consider the problem 
{\Large\em 
\begin{itemize}\item[]\begin{itemize}\item[]\begin{itemize}
\item[(IP.V.a)] Find a curve $\gamma_*$ belonging to $\tilde\Gamma_L$ 
that \mbox{maximizes}
the \mbox{functional} $\tilde\Gamma_L\ni\gamma\mapsto A(\gamma)\in\rr$.
\end{itemize}\end{itemize}\end{itemize}}

It is cleat that $\Gamma_L\subseteq\tilde\Gamma_L$. Therefore
\begin{proposition}
If (IP.V.a) has a solution $\gamma_*$ such that 
$\gamma_*\in\Gamma_L$, then $\gamma_*$ is a solution of 
(IP.V). \proofend
\end{proposition}

\subsection{Finding the solutions of (IP.V.a)}

To find the solutions of  (IP.V.a), we apply the necessary conditions
for optimality given by the maximum principle.  Suppose that
$$
[0,L]\ni t\mapsto\gamma_*(t)=(x(t),y(t))\in\rr^2
$$ 
is a solution of (IP''). It then follows from the conjecture on necessary
conditions for an optimum that there exists
an \mbox{absolutely} \mbox{continuous} ``momentum'' function
$$
[0,L]\ni t\mapsto\mu(t)=(m_x(t),m_y(t))\in\rr^2
$$ 
such that the Hamilton equations
\begin{eqnarray}
\dot m_x(t)&=&-\frac{\partial H}{\partial x}(\gamma_*(t),\mu(t),
\dot\gamma(t))\,,\label{hamil1}\\
\dot m_y(t)&=&-\frac{\partial H}{\partial y}(\gamma_*(t),\mu(t),
\dot\gamma(t))\,,\label{hamil2}
\end{eqnarray}
and the Hamiltonian maximization condition 
\begin{equation}\label{hamio}
H(\gamma_*(t),\mu(t),\dot\gamma_*(t))=
\max\Big\{H(\gamma_*(t),\mu(t),w):w\in\rr^2,\|w\|\le 1\Big\}
\end{equation}
hold for almost all $t\in[0,L]$. Here $H$ is the Hamiltonian,
that is, the function
\begin{equation}\label{hamio2}
\rr^6\ni(x,y,p_x,p_y,u,v)\mapsto H(x,y,p_x,p_y,u,v)\equaldef
p_xu+p_yv+xv-yu\,.
\end{equation}
In view of (\ref{hamio2}), we can rewrite 
(\ref{hamil1}), (\ref{hamil2}), as
\begin{eqnarray}
\dot m_x(t)&=&-\dot y(t)\,,\label{hamil3}\\
\dot m_y(t)&=&\phantom{-}\dot x(t)\,.\label{hamil4}
\end{eqnarray}

Define
$$
\alpha(t)=m_x(t)-y(t)\,,\qquad\beta(t)=m_y(t)+x(t)\,.
$$
Then 
\begin{eqnarray}
\dot\alpha(t)&=&-2\dot y(t)\,,\label{hamil5}\\
\dot\beta(t)&=&\phantom{-}2\dot x(t)\,,\label{hamil6}
\end{eqnarray}
and
\begin{equation}\label{hamix}
H(\gamma_*(t),\mu(t),u,v)=\alpha(t)u+\beta(t)v\,\,.
\end{equation}
Therefore, if we define 
\begin{equation}\label{haminu}
\nu(t)=\sqrt{\alpha(t)^2+\beta(t)^2}\,,
\end{equation}
(\ref{hamio}) says that
\begin{equation}\label{hamioux}
\left.\begin{array}{c}
\dot x(t)=\frac{\alpha(t)}{\nu(t)}\,,\\
\,\,\,\\
\dot y(t)=\frac{\beta(t)}{\nu(t)}\,,\end{array}\right\}\quad{\rm if}\quad
\nu(t)\neq 0\,.
\end{equation}
If $\nu(t)=0$ then  (\ref{hamio}) allows the vector 
$\dot\gamma_*(t)$ to be an arbitrary member of
$\{w\in\rr^2:\|w\|\le 1\}$. We can combine both cases into the following
restatement of (\ref{hamioux}) which is also valid when $\nu(t)=0$:
\begin{equation}\label{hamiouxx}
\nu(t)\dot x(t)=\alpha(t)\qquad{\rm and}\qquad
\nu(t)\dot y(t)=\beta(t)\,.
\end{equation}
A simple calculation then shows that
$$
\frac{d\phantom{t}}{dt}(\nu^2)=
\frac{d\phantom{t}}{dt}(\alpha^2+\beta^2)=
2\alpha\dot\alpha+2\beta\dot\beta=
2\nu\dot x\cdot(-2\dot y)+
2\nu\dot y\cdot 2\dot x=0\,.
$$
Therefore {\em the function $\nu$ is constant.} Let $\bar\nu$ be the 
constant value of $\nu(t)$. If $\bar\nu=0$ then 
$\alpha(t)\equiv\beta(t)\equiv 0$, so 
$\dot\alpha(t)\equiv\dot\beta(t)\equiv 0$, and then 
(\ref{hamil5}) and (\ref{hamil6}) imply that 
$\dot x(t)\equiv\dot y(t)\equiv 0$, so $\gamma_*$ is a constant
curve and $A(\gamma_*)=0$. It is clear that there do exist curves
$\gamma\in\tilde\Gamma_L$ for which $A(\gamma)>0$. Therefore 
the fact that $A(\gamma_*)=0$ contradicts the assumption that
$\gamma_*$ is a solution of (IP''). So we can exclude the possibility 
that $\bar\nu=0$.

From now on we assume that $\bar\nu>0$. Then (\ref{hamioux}) implies that
$\dot\gamma$ is continuous, and 
\begin{equation}\label{hamiouxi}
\|\dot\gamma(t)\|=1\qquad {\rm for\,\,\,all}\,\,\,t\in[0,L]\,.
\end{equation}

Write 
$$
\xi(t)=\dot x(t)\,,\qquad\eta(t)=\dot y(t)\,.
$$
Then, using (\ref{hamil5}), (\ref{hamil6}), and (\ref{hamioux})
we can conclude that
$$
\dot\xi\equiv-\frac{2\eta}{\bar\nu} \quad{\rm and}\quad
\dot\eta\equiv\frac{2\xi}{\bar\nu}\,.
$$
Therefore 
$$
\ddot\xi+\frac{4\xi}{\bar\nu^2}\equiv\ddot\eta+\frac{4\eta}{\bar\nu^2}\equiv
0\,.
$$
This implies that 
$$
\eta(t)=c_1\cos\Big(\frac{2t}{\bar\nu}\Big)+c_2\sin\Big(\frac{2t}{\bar\nu}\Big)
$$
for some constants $c_1$, $c_2$. If we choose $t_0\in\rr$ and
$r\in[0,\infty[$ such that 
$$
c_1=r\cos\Big(\frac{2t_0}{\bar\nu}\Big)\,,\qquad
c_2=r\sin\Big(\frac{2t_0}{\bar\nu}\Big)\,,
$$
we find that
$$
\eta(t)=r\cos\Big(\frac{2(t-t_0)}{\bar\nu}\Big)\,.
$$
Then 
$$
\dot\xi(t)=\frac{\bar\nu\eta(t)}{2}=-r\sin\Big(\frac{2(t-t_0)}{\bar\nu}\Big)\,.
$$
Since $\|\dot\gamma(t)\|=1$, we have $r=1$, and we have shown that 
\begin{equation}\label{hamiouxio}
\xi(t)=-\sin\Big(\frac{2(t-t_0)}{\bar\nu}\Big)\qquad
{\rm and}\qquad
\eta(t)=\cos\Big(\frac{2(t-t_0)}{\bar\nu}\Big)\,.
\end{equation}
The fact that $\gamma(L)=\gamma(0)$ implies that $\bar\nu=\frac{L}{\pi k}$ 
for some nonzero integer $k$. Therefore 
\begin{equation}\label{hamiouxioo}
\xi(t)=-\sin\Big(\frac{2\pi k(t-t_0)}{L}\Big)\qquad
{\rm and}\qquad
\eta(t)=\cos\Big(\frac{2\pi k(t-t_0)}{L}\Big)\,.
\end{equation}
We then get, by integration, 
\begin{eqnarray*}
x(t)&=&\bar x+\frac{L}{2\pi k}\cos\Big(\frac{2\pi k(t-t_0)}{L}\Big)\,,\\
y(t)&=&\bar y+\frac{L}{2\pi k}\sin\Big(\frac{2\pi k(t-t_0)}{L}\Big)\,,
\end{eqnarray*}
where 
\begin{eqnarray*}
\bar x&=&\frac{-L}{2\pi k}\cos\Big(\frac{2\pi kt_0}{L}\Big)\,,\\
\bar y&=&\frac{L}{2\pi k}\sin\Big(\frac{2\pi kt_0}{L}\Big)\,.
\end{eqnarray*}
The above formulas define a family $\Big\{\gamma^{t_0,k}\Big\}_{t_0\in\rr,
k\in\zz}$ of curves belonging to $\tilde\Gamma_L$, \mbox{depending} on the real
parameter $t_0$ and the nonzero integer parameter $k$. If we let 
$\bar\gamma^{t_0,k}$ be the point with coordinates 
$\frac{-L}{2\pi k}\cos\Big(\frac{2\pi kt_0}{L}\Big)$, 
$\frac{L}{2\pi k}\sin\Big(\frac{2\pi kt_0}{L}\Big)$, then 
the curves $\gamma^{t_0,k}$ satisfy 
$$
\Big\|\gamma^{t_0,k}(t)-\bar\gamma^{t_0,k}\Big\|\equiv
\frac{L}{2\pi k}\,.
$$
Therefore {\em the locus $|\gamma^{t_0,k}|$ of $\gamma^{t_0,k}$ is a circle 
of radius $\frac{L}{2\pi k}$}. It is easy to verify that
{\em the map $[0,L]\ni t\mapsto\gamma^{t_0,k}(t)\in\rr^2$ is 
parametrized by arc-length and 
covers the circle $|\gamma^{t_0,k}|$ $|k|$ times.} Moreover,
{\em the orientation derived from this map is counterclockwise if
$k>0$ and clockwise if $k<0$.}

Let us now compute the numbers $A(\gamma^{t_0,k})$. We have
\begin{eqnarray*}
A(\gamma^{t_0,k})&=&\int_0^L(x(t)(\dot y(t)-y(t)\dot x(t))\,dt\\
&=&\int_0^L
\Big(\bar x+\frac{L}{2\pi k}\cos\Big(\frac{2\pi k(t-t_0)}{L}\Big)\Big)
\cos\Big(\frac{2\pi k(t-t_0)}{L}\Big)\,dt\\
&\,&-\int_0^L
\Big(\bar y+\frac{L}{2\pi k}\sin\Big(\frac{2\pi k(t-t_0)}{L}\Big)\Big)
\Big(-\sin\Big(\frac{2\pi k(t-t_0)}{L}\Big)\Big)\,dt\\
&=&\frac{L^2}{2\pi k}\,.
\end{eqnarray*}
The numbers $A(\gamma^{t_0,k})$ are therefore negative 
for $k<0$ (corresponding to the fact that for such values of $k$ the 
curve $\gamma^{t_0,k}$ is oriented clockwise) and positive for 
$k>0$. Furthermore, the largest of these values is 
$\frac{L^2}{2\pi}$, which is attained  for $k=1$. 

In addition, when $k=1$ the curves $\gamma^{t_0,k}$ belong to $\Gamma_L$, 
because they are simple, parametrized by arc-length, and 
oriented counterclockwise.  It follows that 
{\em the curves $\gamma^{t_0,1}$} are solutions 
of (IP') if they are solutions of (IP''). Moreover, if 
a solution of (IP'') exists, then it has to be 
one of the $\gamma^{t_0,1}$, because we have shown that 
the $\gamma^{t_0,k}$ are the only members of $\tilde\Gamma$ that satisfy 
the necessary conditions for an optimum. In that case, then all 
the $\gamma^{t_0,1}$ are solutions, because the functional $A$ has the 
same value for all of them.

Summarizing, we have shown
\begin{proposition}\label{comb1}
Assume that (IP'') has a solution. Then the solutions of (IP'') are 
the circles $\gamma^{t_0,1}$, for $t_0\in\rr$. Moreover, these circles
are also the solutions of the isoperimetric problem (IP'). In particular,
the area enclosed by the area-maximizing circles is 
$\frac{L^2}{4\pi}$. \proofend
\end{proposition}

\subsection{Existence of area-maximizers}

To conclude our complete solution of (IP'), we have to prove that
(IP'') has a solution. The proof is quite simple. The set 
$\tilde\Gamma_L$ is a subset of $C^0([0,L],\rr^2)$ of continuous
functions $\gamma:[0,L]\mapsto\rr^2$, which is a Banach space
in a well-known waym with the sup norm. We first prove the following 
result. 

\begin{proposition}
$\tilde\Gamma_L$ is a compact subset of $C^0([0,L],\rr^2)$.
\end{proposition}

\medbreak\noindent PROOF.\ Let 
$\{\gamma_j\}_{j=1}^\infty$ be a sequence of members 
of $\tilde\Gamma_L$. We want to extract a subsequence
$\{\gamma_{j(k)}\}_{k=1}^\infty$ that converges uniformly
to a limit $\gamma\in\tilde\Gamma_L$. 

Since every member $\gamma$ of $\tilde\Gamma_L$ is absolutely continuous
and satisfies $\|\dot\gamma(t)\|\le 1$ for a.e.~$t$, and 
$\gamma(0)=(0,0)$, it is clear that the sequence 
$\{\gamma_j\}_{j=1}^\infty$ is uniformly bounded and equicontinuous. 
So there exists a subsequence
$\{\gamma_{j(k)}\}_{k=1}^\infty$ that converges uniformly
to a limit $\gamma$, and our conclusion will follow if we prove that 
$\gamma\in\tilde\Gamma_L$. Since $\gamma_j(0)=\gamma_j(L)=(0,0)$ for every 
$j$, it is clear that $\gamma(0)=\gamma(L)=(0,0)$. Since 
$\gamma_j$ is absolutely continuous and $\|\dot\gamma_j(t)\|\le 1$ for
almost all $j$, it follows that 
$$
\|\gamma_j(t)-\gamma_j(s)\|=\|\int_s^t\dot\gamma_j(r)\,dr\|\le|t-s|
$$
for all $j$ and all $t,s\in[0,L]$.  Therefore 
$$
\|\gamma(t)-\gamma(s)\|\le|t-s|\quad{\rm for\,\,\,all}\quad
t,s\in[0,L]\,.
$$
This says that $\gamma$ is Lispchitz with constant $1$, so 
$\gamma$ is absolutely continuous and
$\|\dot\gamma(t)\|\le 1$ for a.e.~$t$. Therefore 
$\gamma\in\tilde\Gamma_L$. \proofend

\medbreak
We now prove the continuity of $A$. 
\begin{proposition}
The functional $A$ is continuous on $\tilde\Gamma_L$.
\end{proposition}

\medbreak\noindent PROOF.\ Let 
$\{\gamma_j\}_{j=1}^\infty$ be a sequence of members 
of $\tilde\Gamma_L$ that converges uniformly to
a limit $\gamma_\infty\in\tilde\Gamma_L$. We want to prove 
that 
\begin{equation}
\lim_{j\to\infty}\Big(A(\gamma_j)-A(\gamma_\infty)\Big)=0\,.
\end{equation}
Write 
$$
\gamma_j(t)=(x_j(t),y_j(t))\quad{\rm for}\quad j\in\{1,2,\ldots\}\cup
\{\infty\}\,,\,t\in[0,L]\,.
$$
Then 
$$
A(\gamma_j)-A(\gamma_\infty)=I_j^1-I_j^2\,,
$$
where
\begin{eqnarray*}
I_j^1&=&\int_0^L\Big(x_j(t)\dot y_j(t)-x_\infty(t)\dot y_\infty(t)\Big)\,dt\,,\\
I_j^2&=&\int_0^L\Big(y_j(t)\dot x_j(t)-y_\infty(t)\dot x_\infty(t)\Big)\,dt\,.
\end{eqnarray*}
We must therefore show that 
$$
\lim_{j\to\infty}I_j^1=\lim_{j\to\infty}I_j^2=0\,.
$$
Let us prove that $\lim_{j\to\infty}I_j^1=0$. (The proof that 
$\lim_{j\to\infty}I_j^2=0$ is similar.) Write
$$
I_j^1=\hat I_j+\tilde I_j\,,
$$
where 
\begin{eqnarray*}
\hat I_j&=&\int_0^L\Big(x_j(t)-x_\infty(t)\Big)\dot y_j\,dt\,,\\
\tilde I_j&=&\int_0^Lx_\infty(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt\,.
\end{eqnarray*}
We will show that 
$$
\lim_{j\to\infty}\hat I_j=\lim_{j\to\infty}\tilde I_j=0\,.
$$
The fact that 
$\lim_{j\to\infty}\hat I_j=0$ is trivial, because
$x_j\to x_\infty$ uniformly, and the functions $\dot y_j$ are uniformly 
bounded, since $|\dot y_j(t)|\le\|\dot\gamma_j(t)\|\le 1$.

To prove that $\lim_{j\to\infty}\tilde I_j=0$, we pick a real number $\eps$
such that $\eps>0$, and find $J$ such that $|\tilde I_j|\le\eps$
whenever $j\ge J$. Let $h$ be a function of class $C^1$ on $[0,L]$ such
that
$$
|x_\infty(t)-h(t)|\le\frac{\eps}{4L}\quad{\rm whenever}\quad t\in[0,L]\,.
$$
(The existence of such an $h$ follows, for example, from teh
Stone-Weierstrass theorem, since $x_\infty$ is continuous.) Then 
$$
\tilde I_j=\int_0^L\Big(x_\infty(t)-h(t)\Big)
\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt
+\int_0^Lh(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt\,.
$$
The absolute value of the first integral is bounded 
by $\frac{\eps}{2}$, because 
$|x_\infty(t)-h(t)|\le\frac{\eps}{4L}$ 
and $|\dot y_j(t)-\dot y_\infty(t)|\le 2$. To estimate the second integral,
we first rewrite it using integrationby parts:
\begin{eqnarray*}
\int_0^Lh(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt
&=&h(L)\Big(y_j(L)-y_\infty(L)\Big)-h(0)\Big(y_j(0)-y_\infty(0)\Big)\\
&\,&\,\,\,-\int_0^L\dot h(t)\Big(y_j(t)-y_\infty(t)\Big)\,dt\\
&=&-\int_0^L\dot h(t)\Big(y_j(t)-y_\infty(t)\Big)\,dt\,,
\end{eqnarray*}
using the fact that $y_j(0)=y_\infty(0)=y_j(L)=y_\infty(L)=0$. 
Since $y_j\to y_\infty$ uniformly, and $\dot h$ is a fixed 
bounded function, the limit of 
$\int_0^Lh(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt$ as $j\to\infty$ is
zero. Therefore we can find $J$ such that 
$$
\Big|\int_0^Lh(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt\Big|\le
\frac{\eps}{2}\quad{\rm whenever}\quad j\ge J\,.
$$
It then follows that 
$$
|\tilde I_j|\le\eps\quad{\rm whenever}\quad j\ge J\,,
$$
completing our proof. \proofend

\medbreak
It then follows from Propositions 3 and 4 that
\begin{proposition}\label{comb2}
The maximization problem (IP'') has a solution. \proofend
\end{proposition}


\subsection{The final result}

By combining Propositions \ref{comb1} and \ref{comb2}, we get

\begin{theorem}\label{comb3}
The solutions of the modified isoperimteric problem (IP'') are the circles
$\gamma^{t_0,1}$, for $t_0\in\rr$. \mbox{Moreover}, these circles are also
the solutions of the original isoperimetric isoperimetric problem (IP'). In
\mbox{particular}, the area \mbox{enclosed} by the area-maximizing circles is
$\frac{L^2}{4\pi}$. \proofend
\end{theorem}

\subsection{How to make the solution of the isoperimetric problem rigorous}

\gnu{This section will eventually be greatly expanded and 
possibly moved to
a chapter on the issue of mathematical rigor. The expanded section
will include short but completely self-contained
proofs of the Jordan curve theorem
(following the beautiful presentation in Dugundji's book),
and of Green's formula for simple rectifiable loops.

At the very end of this draft we include our notes on the isoperimetric 
problem. What remains to be done is to incorporate them 
into the text and merge them with what we already have.


We will also include A. Hurwitz's 1902 proof based on Wirtinger's
inequality. Furthermore, Wirtinger's inequality itself
offers a wonderful example of how to apply optimal control
or calculus of variations to prove inequalities. We will include
the optimal control proof together with the standard Fourier 
series proof.}

\gnu{As it stands, the section contains half-finished paragraphs,
several of which correspond to an earlier approach that we are now
planning to drop.}


We use $\mathbb{S}^1$ to denote the unit circle 
$\{(x,y)\in\rr^2:x^2+y^2=1\}$.  Alternatively, if we think of $\rr^2$ as the
complex plane $\mathbb{C}$, then 
$\mathbb{S}^1=\{z\in\mathbb{C}:|z|=1\}$.  

We fix a positive real number $L$, and
let $\kappa_{_L}$ be the map 
$$
\rr\ni\theta\mapsto \kappa_{_L}(\theta)\equaldef
e^{\frac{2\pi i}{L}\theta}\in\mathbb{S}^1\,.
$$

A {\em loop} (or {\em closed curve}) in  $\rr^2$ is a continuous map
$\xi:\mathbb{S}^1\mapsto\rr^2$.  Then $C^0(\mathbb{S}^1,\rr^2)$ 
is the set of all loops in $\rr^2$.
The  {\em carrier}, or
{\em locus}, of the loop $\xi$ is the set $\{\xi(s):s\in\mathbb{C}^1\}$. 
Clearly, $|\xi|$ is compact.

If $L>1$, then $C^0(\mathbb{S}^1,\rr^2)$ can be identified in an
obvious way with $C_{per,L}^0(\rr,\rr^2)$, the set of all continuous
maps $\gamma:\rr\mapsto\rr^2$ that are $L$-periodic, i.e., such that
$\gamma(\theta+L)=\gamma(\theta)$ for all $\theta\in\rr$. Naturally,
the identification is made by assigning to each $\xi\in
C^0(\mathbb{S}^1,\rr^2)$ the map $\xi\circ\kappa_{_L}\in
C_{per,L}^0(\rr,\rr^2)$.

If $\xi:\mathbb{S}^1\mapsto\mathbb{C}$ is a loop,
and $p\in\mathbb{C}\backslash|\xi|$, then the {\em winding number} of
$\xi$ about $p$ in the integer $\nu(\xi,p)$ defined as follows:
pick any positive $L$, and choose a continuous function
$f:\rr\mapsto\mathbb{C}$ such that
\mbox{$e^{f(t)}=(\xi\circ\kappa_{_L})(t)-p$} for all $t\in\rr$. 
(The existence of $f$ is 
trivial locally, because  the map 
\mbox{$\mathbb{C}\ni z\mapsto e^z\in\mathbb{C}\backslash\{0\}$} is surjective
and has a continuous right inverse near every $s\in\mathbb{C}\backslash\{0\}$.
Given this, the global existence  of $f$ follows easily by patching up local
\mbox{versions}.) Then the number
$\frac{f(t+L)-f(t)}{2\pi i}$ necessarily has to be an integer, because 
\mbox{$e^{f(t+L)}=(\xi\circ\kappa_{_L})
(t+L)-p=(\xi\circ\kappa_{_L})(t)-p=e^{f(t)}$}. Clearly,
this integer depends continuously
on $t$. Furthermore, if $f_1,f_2:\rr\mapsto\mathbb{C}$ are
any two continuous such that
$e^{f_1(t)}=e^{f_2(t)}=(\xi\circ\kappa_{_L})(t)-p$ for all $t\in\rr$
must differ by a constant, since 
$e^{f_1(t)-f_2(t)}=1$, so that $f_1(t)-f_2(t)=2\pi ik(t)$, where
$k(t)$ is an integer that depends continuously on $t$. 
So $\frac{f(t+L)-f(t)}{2\pi i}$ is in fact independent
of $t$ and $f$, and it is easy to see that it is also 
independent of $L$. We define 
$\nu(\xi,p)\equaldef\frac{f(t+L)-f(t)}{2\pi i}$ for any $t\in\rr$,
where $L$ and $f$ are as above. 

A {\em reparametrization} of a loop $\xi:\mathbb{S}^1\mapsto\mathbb{C}$ 
is a loop of the form $\xi\circ\psi$, where 
$\psi:\mathbb{S}^1\mapsto\mathbb{S}^1$ is an orientation-preserving 
homeomorphism.
(``Orientation-preserving'' means, for example, that if $\bar t$ is such
that $\psi(e^{2\pi i\bar t})=1$, then the imaginary part of
$\psi(e^{2\pi i(\bar t+\eps)})$ is positive for all sufficiently small
positive $\eps$.) It is clear that reparametrization preserves
carriers and winding numbers. Precisely, 
\begin{itemize}
\item[(*)] {\em if 
$\eta$ is a reparametrization of $\xi$, then 
$|\eta|=|\xi|$ and $\nu(\eta,p)=\nu(\xi,p)$ for all 
$p\in\mathbb{C}\backslash|\xi|$.}
\end{itemize}
(Proof: the fact that 
$|\eta|=|\xi|$ is trivial. Let $f:\rr\mapsto\mathbb{C}$ be continuous
and such that $e^{f(t)}=\eta(e^{2\pi i t})$ for all $t$. Let 
$\psi:\mathbb{S}^1\mapsto\mathbb{S}^1$ be an orientation-preserving 
homeomorphism such that $\eta=\xi\circ\psi$. Then there exist
$a\in\rr$ and a homeomorphism
$\tilde\psi:[0,1]\mapsto[a,a+1]$ such that 
\begin{equation}\label{newway}
e^{2\pi i\tilde\psi(t)}=\psi(e^{2\pi it})
\end{equation}
for all $t\in[0,1]$, $\tilde\psi(0)=a$ and $\tilde\psi(1)=a+1$. Extend
$\tilde\psi$ to the whole real line by defining
$\tilde\psi(k+t)=k+\psi(t)$ for $t\in[0,1]$ and $k\in\zz$. Let
$g=f\circ\tilde\psi$. Then (\ref{newway}) holds for all $t\in\rr$. 
Moreover,
$$
e^{g(t)}=e^{f(\tilde\psi(t))}=\eta(e^{2\pi i\tilde\psi(t)})
=\eta\Big(\psi(e^{2\pi it})\Big)=(\eta\circ\psi)(e^{2\pi it})=
\xi(e^{2\pi it})\,.
$$
Therefore $\nu(\xi,p)=\frac{g(1)-g(0)}{2\pi i}=
\frac{f(\tilde\psi(1))-f(\tilde\psi(0))}{2\pi i}=
\frac{f(a+1)-f(a)}{2\pi i}=\nu(\eta,p)$.)

A loop $\xi\in C^0(\mathbb{S}^1,\rr^2)$ is {\em absolutely continuous}
if for some (and hence every) $L$ the function 
$\gamma=\xi\circ\kappa_{_L}$ is 
locally absolutely continuous. If $\xi$ is 
absolutely continuous and $\gamma=\xi\circ\kappa_{_L}$, 
then we can choose $f$ by letting
$$
f(t)=\bar f+\int_0^t\frac{\dot\gamma(s)}{\gamma(s)-p}\,ds\,,
$$ 
where $\bar f\in\mathbb{C}$ is such that
$e^{\bar f}=\gamma(0)-p$. Indeed, if we define $g:\rr\mapsto\mathbb{C}$
by letting $g(t)=(\gamma(t)-p).e^{-f(t)}$, we have 
\begin{eqnarray*}
\dot g(t)&=&-(\gamma(t)-p).\dot f(t).e^{-f(t)}+\dot\gamma(t).e^{-f(t)}\\
&=&-(\gamma(t)-p).\frac{\dot\gamma(t)}{\gamma(t)-p}.e^{-f(t)}
+\dot\gamma(t).e^{-f(t)}\\
&=&-\dot\gamma(t).e^{-f(t)}+\dot\gamma(t).e^{-f(t)}=0\,.
\end{eqnarray*}
Therefore $g(t)\equiv{\rm constant}$. Since $g(0)=1$, we have
$g(t)\equiv 1$ for all $t$, 
so $e^{f(t)}\equiv\gamma(t)-p$. We then get the formula
$$
\nu(\xi,p)=\frac{1}{2\pi i}
\int_0^L\frac{\dot\gamma(s)}{\gamma(s)-p}\,ds\,\,\,{\rm whenever}
\,\,\,p\notin|\xi|.
$$

A loop $\xi\in C^0(\mathbb{S}^1,\rr^2)$ is said to be {\em simple} if
it is a one-to-one map, i.e., if 
$\xi(z_1)\neq\xi(z_2)$ whenever $z_1,z_2\in\mathbb{S}^1$ 
and $z_1\neq z_2$.  

If $n$ is a positive integer, a
{\em plane $n$-polygonal loop} is a loop $\xi$ in $\rr^2$ such that 
there exist times
$t_1,t_2,\ldots,t_{n}$ for which
\mbox{$0=t_1<t_2<\ldots<t_n<1$} having the property that
for $j=1,\ldots,n$ the map $[t_{j},t_{j+1}]\ni t\mapsto
\xi(e^{2\pi it})$ is linear. A plane $n$-polygonal loop 
is completely determined by the 
times $t_1,t_2,\ldots,t_{n}$ and the $n$ points 
$A_1,\ldots,A_n$ defined by letting 
$A_j=\xi(e^{2\pi it_j})$  for $j=1,\ldots,n$.
Furthermore, changing the choices of $t_1,t_2,\ldots,t_{n}$ results
in a reparametrization of $\xi$, which does not change 
the carrier $|\xi|$ nor the winding numbers $\nu(\xi,p)$ for 
$p\notin|\xi|$. We can therefore talk unambiguously 
about the carrier and the winding numbers of the 
``polygonal with vertices $A_1,\ldots,A_n$.''

A {\em plane triangle} is a plane $3$-polygonal loop.

The {\em vertices}
of a triangle 


A triangle $\xi$ is {\em nondegenerate}
if $|\xi|$ is not entirely contained in a straight line. 
It is easy to see that a triangle $\xi$ with vertices
$A_1,A_2,A_3$ is nondegenerate if and only
if the vectors $A_2-A_1$ and $A_3-A_1$ are linearly independent. 
If $\xi$ is a nondegenerate triangle, we say that
$\xi$ {\em oriented counterclockwise} if the determinant
of the $2$ by $2$ suqare matrix with columns
$A_2-A_1$ and $A_3-A_1$ is positive. 


The {\em Jordan curve theorem} says that if $\xi$ is a simple 
loop then $\rr^2\backslash|\xi|$ has exactly two connected
components. In other words, 
$\rr^2\backslash|\xi|$ is the union of two 
disjoint connected open subsets $\Omega_1$, $\Omega_2$ of $\rr^2$.
If $R\in\rr$ is such that $R>0$ and $\|\xi(s)\|\le R$ for all 
$s\in\mathbb{S}^!$, then the set
$S=\{q\in\rr^2:\|q\|>R\}$ is open, connected, and disjoint from
$|\xi|$. Therefore $S$ is contained in one of the components
$\Omega_1$, $\Omega_2$. The other component
must then be contained in 
$\{q\in\rr^2:\|q\|\le R\}$. So one of the components is unbounded
and the other one is bounded.  We use 
${\rm Ins}(\xi)$, ${\rm Out}(\xi)$ (the ``inside'' and the
``outside'' of $\xi$) to denote, respectively,
the bounded and the unbounded connected component
of $\rr^2\backslash|\xi|$. We refer to 
${\rm Ins}(\xi)$ as {\em the region enclosed by the 
simple loop $\xi$}.  We use $A(\xi)$ to denote the area 
of ${\rm Ins}(\xi)$.  

A loop $\xi\in C^0(\mathbb{S}^1,\rr^2)$ is said to be {\em
rectifiable} if the map $\xi\circ\kappa_L$ is locally absolutely
continuous for some positive $L$, in which case it follows on 
easily that $\xi\circ\kappa_L$ is locally absolutely
continuous for every positive $L$.

\begin{lemma}
Let $\xi:\mathbb{S}^1\mapsto\rr^2=\mathbb{C}$ be a simple 
loop. Then the funcion $\mathbb{C}\backslash|\xi|\ni p\mapsto
\nu(\xi,p)\in\zz$ is constant on ${\rm Out}(\xi)$ and  
on ${\rm Ins}(\xi)$, and has the value $0$
on ${\rm Out}(\xi)$ and the value $1$ or $-1$ on 
${\rm Ins}(\xi)$.
\end{lemma}
\medbreak\noindent PROOF.\ The winding number $\nu(\xi,p)$ clearly
depends continuously on $p$ as long as $p$ varies on
$\mathbb{C}\backslash|\xi|$. (Let $p\in\mathbb{C}\backslash|\xi|$, and
let $\{p_j\}_{j=1}^\infty$ be a sequence
in $\mathbb{C}\backslash|\xi|$ converging to $p$. Let 
$f:\rr\mapsto\mathbb{C}$ be a continuous map with the property
that $e^{f(t)}=\xi(e^{2\pi it})-p\equaldef h(t)$ for all $t\in\rr$. 
Let $h_j(t)=\xi(e^{2\pi it})-p_j$. Then $h_j\to h$ as $j\to\infty$, 
uniformly on compact sets, so  $h_j\to h$ 
uniformly, since the $h_j$ are $1$-periodic. Since $h(t)\neq 0$ for 
all $t$, the functions $g_j\equaldef\frac{h_j}{h}$ converge uniformly
to the constant
$1$ as $j\to\infty$. Let $D$ be the open disc $\{z\in\mathbb{C}:
\|z-1\|<1\}$. Let $\lambda:D\mapsto\mathbb{C}$ be the 
standard branch of the natural logarithm, so that 
$\lambda(1+s)=\sum_{k=1}^\infty(-1)^{k+1}\frac{s^k}{k}$
whenever $|s|<1$. Then $\lambda(1)=0$, $\lambda$ is continuous, and 
$e^{\lambda(z)}=z$ whenever $z\in D$. Fix $J\in\nn$ such that
$g_j(t)\in D$ whenever $t\in\rr$ and $j\ge J$. 
Define
$$
f_j(t)=f(t)+\lambda\Big(g_j(t)\Big)\,\,\,{\rm for}
\,\,\,t\in\rr\,,\,j\ge J.
$$
Then 
$$e^{f_j(t)}=e^{f(t)}.e^{\lambda(g_j(t))}
=h(t).g_j(t)=h_j(t)=\xi(e^{2\pi it})-p_j\,.
$$
Therefore 
\begin{eqnarray*}
\nu(\xi,p_j)&=&\frac{1}{2\pi i}(f_j(1)-f_j(0))\\
&=&\frac{1}{2\pi i}\Big(f(1)-f(0)+\lambda(g_j(1))-\lambda(g_j(0))\Big)\\
&=&\frac{1}{2\pi i}\Big(f(1)-f(0)\Big)\\
&=&\nu(\xi,p)
\end{eqnarray*}
whenever $j\ge J$, since $\lambda(g_j(1))=\lambda(g_j(0))$, because
the function $g_j$ is 
\mbox{$1$-periodic}.)  \mbox{Therefore}
$\mathbb{C}\backslash|\xi|\ni p\mapsto
\nu(\xi,p)\in\zz$ is constant on ${\rm Out}(\xi)$ and  
on ${\rm Ins}(\xi)$, as stated.   

Let $R\in\rr$ be such that $R>0$ and $|\xi(e^{2\pi it})|\le R$ for all 
$t\in R$. Let $p$ be such that $|p|>R$. Then the complex
number $1-\frac{\xi(e^{2\pi it})}{p}$ belongs to $D$ for every
$t\in\rr$, so we can define a continuous function 
$\sigma:\rr\mapsto\mathbb{C}$ by letting 
$$
\sigma(t)=\lambda\Big(1-\frac{\xi(e^{2\pi it})}{p}\Big)\,\,{\rm for}
\,\,t\in\rr\,.
$$
Let $\bar f\in\mathbb{C}$ be such that $e^{\bar f}=-p$. Define
$f(t)=\bar f+\sigma(t)$. Then $f$ is continuous and
$$
e^{f(t)}=e^{\bar f}.e^{\sigma(t)}=(-p).\Big(1-\frac{\xi(e^{2\pi it})}{p}\Big)
=\xi(e^{2\pi it})-p\,.
$$
Therefore
$\nu(\xi,p)=\frac{1}{2\pi i}(f(1)-f(0))=0$, since $f$ is clearly
$1$-periodic. So $\nu(\xi,p)=0$ for one point $p$ of ${\rm Out}(\xi)$, and 
then $\nu(\xi,p)=0$ for all the points $p$ of ${\rm Out}(\xi)$.

Now choose a point $p\in{\rm Ins}(\xi)$. After a translation of the 
coordinate axes of the plane, we may assume that $p=0$. Let 
$\bar q$ be a point of $|\xi|$ closest to $0$. Then $\bar q\neq 0$ and,
after a rotation of the coordinate axes, we may assume that 
$\bar q$ is a positive real number $\bar r$. Fix a $\bar t\in\rr$ such that,
$\xi(e^{2\pi i\bar t})=\bar r$. Fix a number $\bar\eps$
such that $0<\bar\eps<\frac{1}{2}$,  and two continuous
functions $\rho:[\bar t-\eps,\bar t+\eps]\mapsto]0,\infty[$, 
$\varphi:[\bar t-\eps,\bar t+\eps]\mapsto\rr$, such that
$\xi(e^{2\pi it})=\rho(t)e^{2\pi i\varphi(t)}$ 
whenever $t\in[\bar t-\eps,\bar t+\eps]$, and
$\varphi(\bar t)=0$. Then $\rho(t)\ge\bar r$ for all
$\in[\bar t-\eps,\bar t+\eps]$.

Then, if $\eps>0$, it cannot be the case
that $\varphi(t)=0$ for all $t\in[\bar t-\eps,\bar t+\eps]$. (Indeed,
if $\varphi(t)=0$ whenever $t\in[\bar t-\eps,\bar t+\eps]$, 
then $\xi(e^{2\pi it})=\rho(t)\in\rr$ for $t\in[\bar t-\eps,\bar t+\eps]$.
Then the function $[\bar t-\eps,\bar t+\eps]\ni t\mapsto
\xi(e^{2\pi it})$ would be real-valued and would have a local 
minimum at $\bar t$, contradicting the fact that $\xi$ is one-to-one
on some neighborhood of $\bar t$.) Therefore there exists
a sequence $\{\tau_j\}_{j=1}^\infty$ in $\rr$ such that
$\tau_j\to\bar t$ as $j\to\infty$ and $\varphi(\tau_j)\neq 0$ for all $j$.
If we let $L_j$ be the closed half-line $\{re^{2\pi i\varphi(\tau_j)}:
r\ge 0\}$, then $L_j\cap|\xi|$ is a compact set.
Furthermore,  $L_j\cap|\xi|$ is nonempty, because
$\rho(t_j)e^{2\pi i\varphi(t_j)}\in L_j\cap|\xi|$. 
So $L_j\cap|\xi|$ has a point $q_j$ closest to $0$. 
Write $q_j=\xi(e^{2\pi i t_j})$, 
$t_j\in[\bar t-\frac{1}{2},\bar t+\frac{1}{2}]$. 
Also, let $q_j=r_je^{2\pi i\varphi(\tau_j)}$,
so $0<r_j\le\rho(\tau_j)$. 

We claim that $\lim_{j\to\infty}t_j=\bar t$. Indeed, if this was not so
there would exist a subsequence $\{t_{j(\ell)}\}_{\ell=1}^\infty$
of $\{t_{j}\}_{j=1}^\infty$ that converges to a 
$\tilde t\in[\bar t-\frac{1}{2},\bar t+\frac{1}{2}]\backslash\{\bar t\}$.
Let $\tilde q=\xi(e^{2\pi i\tilde t})$. Then 
$\tilde q=\lim_{\ell\to\infty}p_{j(\ell)}$.
Since $e^{2\pi i\varphi(\tau_j)}\to 1$ as $j\to\infty$, 
we conclude that $\tilde q$ is a positive real number
$\tilde r$, and $\tilde r=\lim_{\ell\to\infty}r_{j(\ell)}$.
Since $0<r_{j(\ell)}\le\rho(\tau_{j(\ell)})$, we have
$0<\tilde r\le\rho(\bar t)=\bar r$. Since $\bar r$ is a point 
of $|\xi$ closest to $0$, we must have $\tilde r=\bar r$, and then 
$\tilde q=\bar q$. Since $|\tilde t-\bar t|\le\frac{1}{2}$, the fact that
$\xi(e^{2\pi i\tilde t})=\xi(e^{2\pi i\bar t})$ implies $\tilde t=\bar t$, 
which is a contradiction, proving our claim.

By passing to a subsequence we may assume that $|t_j-\bar t|<\bar\eps$
for all $j$, and either $t_j<\bar t$ for all $j$ or $t_j>\bar t$ for 
all $j$. Then $q_j=\rho(t_j)e^{2\pi i\varphi(t_j)}$---so 
$r_j=\rho(t_j)$ and $\varphi(t_j)=\varphi(\tau_j)$---and 
$re^{2\pi i\varphi(t_j)}\notin|\xi|$ whenever $0\le r<r_j$. 
By passing to a subsequence,  we assume that 
either $\varphi(t_j)<0$ for all $j$ or $\varphi(t_j)>0$
for all $j$.

Let us consider the case when 
$t_j<\bar t$ and $\varphi(t_j)<0$ for all $j$. (The other
three cases are similar.)  Write $\varphi_j=-\varphi(t_j)$,
so $\varphi_j>0$. Fix a number $\delta$ such that 
$0<\delta<\bar r={\rm dist}(0,|\xi|)$. For each $j$, let
$S_j$ be the sector $\{re^{2\pi i\theta}:r\ge 0\wedge
-\varphi_j\le \theta\le 0\}$. Let $\Delta$ be the closed
disc $\{z\in\mathbb{C}:|z|\le\delta\}$. Let  $T_j=S_j\cup\Delta$.
Let $\eta_j:[0,3]\mapsto\mathbb{C}$ be the curve given by
$$
\eta_j(u)=\left\{\begin{array}{lcl}
\Big((1-u)r_j+u\delta\Big)e^{-2\pi i\varphi_j}&{\rm if}&
0\le u\le 1\,,\\
\delta e^{-2\pi i((2-u)\varphi_j+1-u)}&{\rm if}&1\le u\le 2\,,\\
(u-2)\bar r+(3-u)\delta&{\rm if}&2\le u\le 3\,.\end{array}\right.
$$
Then, as $u$ goes from $0$ to $3$, $\eta_j(u)$ starts at
the point $\xi(e^{2\pi i t_j})=r_je^{-2\pi i\varphi_j}$,
moves along the straight-line segment $M_j^1$ from 
$r_je^{-2\pi i\varphi_j}$ to $\delta e^{-2\pi i\varphi_j}$, 
then follows an arc $M_j^2$ of length 
$\delta.(2\pi-\varphi(t_j))$ of the circle $\{z:|z|=\delta\}$, clockwise
from $\delta e^{-2\pi i\varphi_j}$ to $\delta$, 
and finally moves along the straight-line segment $M_3$ from 
$\delta$ to $\bar r$, ending at $\bar r=\xi(e^{2\pi i\bar t})$. 
It is clear that $\eta_j$ is a one-to-one map. Furthermore,
$\eta_j(u)\neq|\xi|$ whenever $0<u<3$. 

Define $\hat\zeta_j:[\bar t-\frac{1}{2},\bar t+\frac{1}{2}]
\mapsto\mathbb{C}$  by letting
$$
\hat\zeta_j(t)=\left\{\begin{array}{lcl}
\xi(e^{2\pi it})&{\rm if}&t\in
[\bar t-\frac{1}{2},\bar t+\frac{1}{2}]\backslash
[\bar t-\bar\eps,\bar t+\bar\eps]\,,\\
\eta_j\Big(\frac{3t-3\bar t+3\bar\eps}{2\bar\eps}\Big)
&{\rm if}&t\in[\bar t-\bar\eps,\bar t+\bar\eps]\,.\end{array}\right.
$$
Then $\hat\zeta_j$ is continuous, and
$\hat\zeta_j(\bar t+\frac{1}{2})=\hat\zeta_j(\bar t+\frac{1}{2})$,
so we can extend $\hat\zeta_j$ to a $1$-periodic map $\zeta_j$ 
from $\rr$ to $\mathbb{C}$. Write $\zeta_j(t)=\omega_j(e^{2\pi it})$
for $t\in\rr$. Then $\omega$ is a simple loop. 

Let $\Omega_j={\rm Ins}(\xi)\backslash T_j$. We claim that 
Then $\Omega$ is open and bounded. 
\begin{equation}\label{isins}
\Omega_j={\rm Ins}(\omega_j)\,.
\end{equation}
To prove (\ref{isins}, we first observe that $\Omega_j$ is clearly
open and bounded.   Let us show that $\Omega_j$ is connected.
Let $p_0,p_1$ be points of $\Omega_j$. Then there exists 
a continuous path $\alpha:[0,1]\mapsto{\rm Ins}(\xi)$ such that
$\alpha(0)=p_0$ and $\alpha(1)=p_1$, since 
${\rm Ins}(\xi)$ is path-connected, because it is open and connected.

\proofend



We use $C^1(\mathbb{S}^1,\rr^2)$ to denote the 
the set of all loops $\xi\in C^0(\mathbb{S}^1,\rr^2)$ 
that are of class $C^1$. Equivalently, a loop 
$\xi\in C^0(\mathbb{S}^1,\rr^2)$ belongs to 
$C^1(\mathbb{S}^1,\rr^2)$ if and only if $\xi\circ\kappa$ is
continuously differentiable. Clearly, $C^1(\mathbb{S}^1,\rr^2)$ 
can be identified, via the map $\xi\mapsto\xi\circ\kappa$,
with $C_{per,2\pi}^1(\rr,\rr^2)$, the set of all continuously 
differentiable  $2\pi$-periodic maps 
$\gamma:\rr\mapsto\rr^2$. If $\xi\in C^0(\mathbb{S}^1,\rr^2)$,
we use $\dot\xi$ to denote the unique 
map $\zeta:\mathbb{S}^1\mapsto\rr^2$ such that 
$\zeta\circ\kappa$ is the derivative of the function
$\xi\circ\kappa$.



A loop $\xi\in C^1(\mathbb{S}^1,\rr^2)$ is 
{\em parametrized by arc-length} (abbr. PAL) if 
$\|\dot\xi(z)\|=1$ for all $z\in\mathbb{S}^1$.


\ssection{Example III: the catenary}\label{subsec-catenary}

Assume that we are given a flexible rope or chain having length
$\Lambda$ and a constant mass density per unit length.  The rope is to
hang, its endpoints being held fixed at points $A$ and $B$. We want to
determine the resulting shape, from the fact that {\em the center of
mass should be as low as possible}.

Let $P$ be a vertical plane containing $A$ and $B$, and let us choose
Cartesian coordinates $x$, $y$ on $P$ so that the $x$ axis is
horizontal, the $y$ axis is vertical and points upwards, and $A$, $B$
have coordinates $(a,\alpha)$, $(b,\beta)$, respectively. Let us
assume that $a<b$.  (This amounts to excluding the possibility that
$A$ and $B$ may lie on the same vertical line. The solution in this
case is easy to find directly.)  Let us make the {\em Ansatz} that the
shape of the hanging rope is given
by a function $y=y(x)$ belonging to the space 
$W^{1,1}([a,b],\rr)$ of all absolutely continuous functions 
$[a,b]\ni x\to y(x)\in\rr$.  

Then the coordinates $x_c,y_c$ of the center of mass are 
\begin{eqnarray*}
x_c&=&\frac{1}{\Lambda}\int_a^bx\,ds=  
\frac{1}{\Lambda}\int_a^bx\sqrt{1+y'(x)^2}\,dx\,,\\
y_c&=&\frac{1}{\Lambda}\int_a^by\,ds=\frac{1}{\Lambda}
\int_a^by(x)\sqrt{1+y'(x)^2}\,dx\,,
\end{eqnarray*}
where $ds$ is the element of arc length. Since $\Lambda$ is fixed, 
our goal is to minimize the integral 
$$
I=\int_a^by(x)\sqrt{1+y'(x)^2}\,dx\,,
$$ 
in the set $\mathcal{Y}$ of all curves $y(\cdot)\in W^{1,1}([a,b],\rr)$ that
satisfy the constraints
\begin{eqnarray*}
y(a)&=&\alpha\,,\\
y(b)&=&\beta\,,\\
\int_a^b\sqrt{1+y'(x)^2}\,dx&=&\Lambda\,.
\end{eqnarray*} 
We observe that the distance $\Delta$ from $A$ to $B$ is given by
$$
\Delta=\sqrt{(b-a)^2+(\beta-\alpha)^2}\,,
$$
so the  set $\mathcal{Y}$ is nonempty if and only if
\begin{equation}
\Lambda\ge\Delta=\sqrt{(b-a)^2+(\beta-\alpha)^2}\,,
\end{equation}
Let us introduce a new function $s$, the arc-length, defined by 
$$
s(x)=\int_a^x\sqrt{1+y'(r)^2}\,dr\,.
$$
Then our curves $y(\cdot)\in\mathcal{Y}$ are in an obvious one-to-one 
correspondence with the members of the set $\Xi$ of all
absolutely continuous curves 
$$
[a,b]\ni x\to\xi(x)=(y(x),s(x))\in\rr^2
$$
in $y,s$ space that satisfy 
$$
\xi(a)=(\alpha,0)\,,\,\,\,\xi(b)=(\beta,\Lambda)\,,
$$
as well as
$$
\xi'(x)\in U\quad{\rm for\,\,\,a.e.}\quad x\in[a,b]\,,
$$
where $U$ is the subset of $\rr^2$ given  by
$$
U=\Big\{\,(u,v)\in\rr^2:v=\sqrt{1+u^2}\,\Big\}\,.
$$

\gnu{Put section number in GNU below}

We thus have a calculus of variations problem of the kind considered
in \S GNU, whose Lagrangian $\rr^2\times U\ni(y,s,u,v)\to L(y,s,u,v)$ 
is given by 
$$
L(y,s,u,v)=yv\,.
$$
Assume that $y_*\in\mathcal{Y}$ is a solution,
and let $\xi_*$ be the corresponding curve in $\Xi$, so 
$\xi_*(x)=(y_*(x),s_*(x))$, where $s_*(x)=\int_0^x\sqrt{1+y_*'(r)^2}\,dr$. 
Let $u_*=y_*'$, $v_*=s_*'$, so $v_*(x)=\sqrt{1+u_*(x)^2}$ for a.e. $x$. 

\gnu{Put conditions name in GNU below}

The necessary conditions GNU then assert that there exist an
absolutely continuous 
vector-valued function $[a,b]\ni x\to\pi(x)=(\pi_y(x),\pi_z(x))\in\rr^2$ 
and a nonnegative constant $\pi_0$ such that 
\begin{eqnarray}
\pi_y'(x) &=&\frac{\partial L}{\partial y}(\xi_*(x),\xi_*'(x))=
\pi_0 v_*(x)=\pi_0\sqrt{1+u_*(x)^2}\,,\label{finn-a}\\
\pi_s'(x)&=&\frac{\partial L}{\partial s}(\xi_*(x),\xi_*'(x))=0\,\label{finn-b}
\end{eqnarray}
for a.e. $x\in[a,b]$, $(\pi_y(x),\pi_s(x),\pi_0)\neq(0,0,0)$ for all $x$, and 
\begin{eqnarray*}
&\,&\pi_y(x)u_*(x) +\pi_s(x)v_*(x)-\pi_0y_*(x)v_*(x)\\
&\,&\,\,\qquad=\pi(x)\cdot\xi_*'(x)-\pi_0L(\xi_*(x),\xi_*'(x))\\
&\,&\,\,\qquad=\max\Big\{\,\pi_y(x)u+\pi_s(x)v-\pi_0L(\xi_*(x),w):
w\in U\,\Big\}\\
&\,&\,\,\qquad=\max\Big\{\,\pi_y(x)u+\pi_s(x)v-\pi_0y_*(x)v:(u,v)\in U\,\Big\}
\end{eqnarray*}
for almost all $x\in[a,b]$.
It then follows that 
\begin{eqnarray}
&\,&\pi_y(x)u_*(x) +(\pi_s(x)-\pi_0y_*(x))\sqrt{1+u_*(x)^2}\nonumber\\
&\,&\,\,\qquad=\max\Big\{\,\pi_y(x)u+(\pi_s(x)-\pi_0y_*(x))
\sqrt{1+u^2}:u\in\rr\,\Big\}\label{finn-c}
\end{eqnarray}
for almost all $x\in[a,b]$.

We now extract from the above conditions more detailed information
about the solutions of the optimization problem.  To begin with,
observe that (\ref{finn-b}) implies in particular that the number
$\pi_s(x)$ is in fact equal to a constant $\bar\pi_s$.

Assume first that $\pi_0=0$. Then (\ref{finn-a}) and (\ref{finn-b})
imply that the vector $\pi(x)$ is equal to a constant
$\bar\pi=(\bar\pi_y,\bar\pi_s)$, which is not equal to $(0,0)$. The
possibility that $\bar\pi_s=0$ is excluded, because the maximization
condition (\ref{finn-c}) would imply, if $\bar\pi_s=0$, that the function
$\rr\ni u\to\bar\pi_y u$ has a maximum, contradicting the fact that
$\bar\pi_y\neq 0$. So $\bar\pi_s\neq 0$. Then the $u$-derivative of the
right-hand side of (\ref{finn-c}) is 
$\bar\pi_y+\frac{\bar\pi_s u}{\sqrt{1+u^2}}$, which can only vanish if
\begin{equation}\label{finn-d}
\frac{u}{\sqrt{1+u^2}} = -\frac{\bar\pi_y}{\bar\pi_s}\,.
\end{equation}
The function $\rr\ni u\mapsto \frac{u}{\sqrt{1+u^2}}$ is strictly
increasing, so the solution $u$ of (\ref{finn-d}) is unique. Hence
$u_*$ is a constant. Thus $y_*$ is a linear function.

%So  $p=0$ is the {\em abnormal case}, when we get a line. 

We now study the case when $\pi_0\neq 0$. Then we may assume $\pi_0=1$.
Equation (\ref{finn-a}) then says that
$$
\pi_y'(x) = \sqrt{1+u_*(x)^2}=v_*(x)=s_*'(x)\,.
$$
Therefore $\pi_y(x)=\bar\pi_y+s_*(x)$ for some constant $\bar\pi_y$.  Let 
$$
Y(x)=y_*(x)-\bar\pi_s\,.
$$
Then (\ref{finn-c}) says that $u=u_*(x)$ maximizes the function
\begin{equation}\label{finn-cc}
u\to(\bar\pi_y+s_*(x))u-Y(x)\sqrt{1+u^2}\,,
\end{equation}
from which it follows that
$$
\frac{u_*(x)}{\sqrt{1+u_*(x)^2}} = -\,\frac{\bar\pi_y+s_*(x)}{Y(x)}\,,
$$
as long as $Y(x)\neq 0$. So 
$$
u_*(x)Y(x)=(\bar\pi_y+s_*(x))\sqrt{1+u_*(x)^2}\,,
$$
and then 
$$
Y(x)Y'(x)=S(x)S'(x)\,,
$$
where $S(x)=\bar\pi_y+s_*(x)$. It then follows that
\begin{equation}\label{finn-e}
Y(x)^2=S(x)^2+c\,,
\end{equation}
where $c$ is a constant.  So we have shown that the function
$Y^2-S^2$ is constant on every subinterval of $[0,1]$ where $Y$ has no zeros.
 

If $x_0\in[a,b]$ is such that $Y(x_0)=0$, then $S(x_0)$ must vanish
too, because the function $\rr\ni u\to S(x_0)u$ does not have a
maximum if $S(x_0)\neq 0$.  It follows that $s_*(x_0)=-\bar\pi_y$.  Since
$s_*$ is strictly increasing, the point $x_0$, if it exists, is
unique. Moreover, (\ref{finn-e}) holds on both sides of $x_0$, with
possibly different constants $c$. If we then let $x$ go to $x_0$ from
the left and from the right, we see that both constants are equal.

So we have shown that (\ref{finn-e}) holds throughout 
$[a,b]$ for some constant  $c$, even if $Y$ has zeros. 
On the other hand, $Y(x)$ cannot be negative, because if it was then 
the function
(\ref{finn-cc}) would not have a maximum. So $Y(x)\ge 0$ for all $x$, and then 
$$
Y(x)=\sqrt{S(x)^2+c}\,,
$$
where, as usual, $\sqrt{r}$ means ``the nonnegative square root of
$r$.''  In addition, the inequality $|S(x)|< Y(x)$ must hold for
almost all $x$, because if $|S(x)|\ge Y(x)$ the function
(\ref{finn-cc}) does not have a maximum.  Therefore $c>0$, and we can
write $c=\kappa^2$, and then
$$
Y(x)=\sqrt{S(x)^2+\kappa^2}\,,\quad\kappa>0\,.
$$
Define a function $\tau:[a,b]\to\rr$ by letting 
$$
S(x)=\kappa.\sinh(\tau(x))\,.
$$
Then 
\begin{eqnarray*}
Y(x)&=&\kappa.\cosh(\tau(x))\,,\\
Y'(x)&=&\kappa.\tau'(x).\sinh(\tau(x))\,,\\
S'(x)&=&\kappa.\tau'(x).\cosh(\tau(x))\,.
\end{eqnarray*}
The equations $Y'=u_*$ and $S'=\sqrt{1+u_*^2}$ then imply 
$S'=\sqrt{1+(Y')^2}$, from which it follows that $(S')^2=1+(Y')^2$, and then
\begin{equation}\label{finn-f}
\kappa^2\tau'(x)^2\cosh^2(\tau(x))=
1+\kappa^2\tau'(x)^2\sinh^2(\tau(x))\,.
\end{equation}
Since $\cosh^2r=1+\sinh^2r$ for all $r$, (\ref{finn-f}) implies
$$
\kappa^2\tau'(x)^2=1\,.
$$
Moreover, $S$ is strictly increasing, because $s_*$ is, so $\tau'(x)>0$. 
This implies that 
$$
\tau'(x)\equiv\frac{1}{\kappa}\,,
$$
so
$$
\tau(x)=\frac{x-x_0}{\kappa}
$$ 
for some constant $x_0$, and then 
\begin{equation}\label{tipper}
y_*(x)=\gamma+\kappa \cosh\Biggl(\frac{x-x_0}{\kappa}\Biggr)\,,
\end{equation}
where the constant $\gamma$ is equal to $\bar\pi_y$. 

To determine the numbers $\gamma$, $\kappa$, $x_0$ in terms of the data
$a$, $b$, $\alpha$, $\beta$, $\Lambda$, we first use (\ref{tipper}) to 
conclude that 
\begin{eqnarray*}
\alpha&=&y_*(a)=\gamma+\kappa \cosh{\mathcal{A}}\,,\\
\beta&=&y_*(b)=\gamma+\kappa \cosh{\mathcal{B}}\,,\\
\frac{\beta-\alpha}{\kappa}&=&\cosh{\mathcal{B}}-\cosh{\mathcal{A}}\\
&=&2\sinh{\frac{\mathcal{B}+\mathcal{A}}{2}}
\sinh{\frac{\mathcal{B}-\mathcal{A}}{2}}\\
&=&2\sinh{\mathcal{C}}\sinh{\mathcal{D}}\,,
\end{eqnarray*}
where 
$$
\mathcal{A}=\frac{a-x_0}{\kappa}\,,\qquad
\mathcal{B}=\frac{b-x_0}{\kappa}\,,\qquad
\mathcal{C}=\frac{a+b-2x_0}{2\kappa}\,,\qquad
\mathcal{D}=\frac{b-a}{2\kappa}\,.
$$
On the other hand, $y'_*(x)=\sinh{\frac{x-x_0}{\kappa}}$, 
so  $\sqrt{1+y'_*(x)^2}=\cosh{\frac{x-x_0}{\kappa}}$, 
and then
$$
\Lambda=\int_a^b\sqrt{1+y'_*(x)^2}dx=
\kappa(\sinh{\mathcal{B}}-\sinh{\mathcal{a}})\,,
$$
so
\begin{eqnarray*}
\frac{\Lambda}{\kappa}&=&\sinh{\mathcal{B}}-\sinh{\mathcal{A}}\\
&=&2\sinh{\frac{\mathcal{B}-\mathcal{A}}{2}}
\cosh{\frac{\mathcal{B}+\mathcal{A}}{2}}\\
&=&2\sinh{\mathcal{D}}\cosh{\mathcal{C}}\,.
\end{eqnarray*}
Then
$$
\frac{\Lambda^2}{\kappa^2}=
4\sinh^2{\mathcal{D}}\cosh^2{\mathcal{C}}\,,
$$
and
$$
\frac{(\beta-\alpha)^2}{\kappa^2}
=4\sinh^2{\mathcal{C}}\sinh^2{\mathcal{D}}\,,
$$
so 
$$
\frac{\Lambda^2-(\beta-\alpha)^2}{\kappa^2}
=4\sinh^2{\mathcal{D}}\,,
$$
and then
$$
2\kappa\sinh{\mathcal{D}}=\sqrt{\Lambda^2-(\beta-\alpha)^2}\,,
$$
so 
\begin{equation}\label{tipper-a}
\frac{\sinh{\mathcal{D}}}{\mathcal{D}}=
\frac{\sqrt{\Lambda^2-(\beta-\alpha)^2}}{b-a}\,.
\end{equation}
The function $[0,+\infty[\ni u\to\frac{\sinh{u}}{u}\in\rr$
is strictly increasing, and takes the value $1$ when $u=0$. If we assume
that $\Lambda>\Delta$, then the right-hand side of 
(\ref{tipper-a}) is $>1$, so there exists a unique positive 
solution $\mathcal{D}$ of (\ref{tipper-a}).  Then $\kappa$ is 
determined by 
\begin{equation}\label{tipper-b}
\kappa=\frac{b-a}{2\mathcal{D}}\,.
\end{equation}
In addition,
\begin{equation}\label{tipper-c}
\tanh{\mathcal{C}}=\frac{\beta-\alpha}{\Lambda}\,,
\end{equation}
and the assumption that $\Lambda>\Delta$ implies that the 
right-hand side of (\ref{tipper-c}) belongs to the interval
$\,]-1,1\,[\,$, so there exists a unique real
solution $\mathcal{C}$ of (\ref{tipper-c}).  
Then $x_0$ is determined by 
\begin{equation}\label{tipper-d}
x_0=\frac{a+b}{2}-\kappa\mathcal{C}
=\frac{a+b}{2}-\frac{(b-a)\mathcal{C}}{2\mathcal{D}}\,.
\end{equation}
Finally, we determine $\gamma$ by 
\begin{equation}\label{tipper-e}
\gamma=\alpha-\kappa\cosh{\mathcal{A}}=
\alpha-\frac{b-a}{2\mathcal{D}}\cosh(\mathcal{C}-\mathcal{D})\,.
\end{equation}
This leads to the formula
\begin{equation}\label{tipper-f}
y_*(x)=\alpha+\frac{b-a}{2\mathcal{D}}\Biggl(
\cosh\Big(\mathcal{C}+\frac{\mathcal{D}(2x-a-b)}{b-a}\Big)
-\cosh(\mathcal{C}-\mathcal{D}\Biggr)\,,
\end{equation}
where $\mathcal{C}$ and $\mathcal{D}$ are determined
by (\ref{tipper-a}) and (\ref{tipper-c}).

All this was done under the assumption that $\Lambda>\Delta$. 
Let us now examine the limit of the right-hand side of 
(\ref{tipper-f}) as $\Lambda\downarrow\Delta$. For this purpose,
we rewrite (\ref{tipper-f}) as 
\begin{eqnarray*}
y_*(x)&=&\alpha+\frac{b-a}{\mathcal{D}}\Biggl(
\sinh\Big(\mathcal{C}+\frac{\mathcal{D}(x-b)}{b-a}\Big)
\sinh\Big(\frac{\mathcal{D}(x-a)}{b-a}\Big)\Biggr)\\
&=&\alpha+(x-a)\frac{\sinh\Big(\frac{\mathcal{D}(x-a)}{b-a}\Big)}
{\frac{\mathcal{D}(x-a)}{b-a}}
\sinh\Big(\mathcal{C}+\frac{\mathcal{D}(x-b)}{b-a}\Big)\,,
\end{eqnarray*}
and observe that $\mathcal{D}\to 0$ and 
$\mathcal{C}\to\mathcal{C}_0$ as $\Lambda\downarrow\Delta$, 
where $\mathcal{C}_0$ is characterized by
\begin{equation}\label{tipper-gg}
\tanh{\mathcal{C}_0}=\frac{\beta-\alpha}{\Delta}\,.
\end{equation}
It follows that, as $\Lambda\downarrow\Delta$,
$y_*(x)$ converges to
\begin{equation}\label{tipper-g}
\bar y_*(x)=\alpha+(\sinh{\mathcal{C}_0})(x-a)\,.
\end{equation}
Equation (\ref{tipper-gg}) easily implies 
$\sinh{\mathcal{C}_0}=\frac{\beta-\alpha}{b-a}$, so the final expression
for $\bar y_*(x)$ is
\begin{equation}\label{tipper-gu}
\bar y_*(x)=\alpha+\frac{\beta-\alpha}{b-a}\cdot(x-a)\,,
\end{equation}
which is, as expected, the unique member of $\mathcal{Y}$ for the
limiting case $\Lambda=\Delta$. 

\gnu{Must put in remarks on how in the ``singular case''
$\Lambda=\Delta$ the catenary problem gives rise to an abnormal extremal.}


\cchapter[$\quad$The $\,$control $\,$Hamiltonian $\,$and $\,$the 
$\,$Hamiltonian $\,$formulation $\,$of\\
\null$\quad$relativistic mechanics]{The control Hamiltonian and 
the Hamiltonian 
formulation of relativistic mechanics\label{rela} }

\noindent In our discussion of Weierstrass' failure to discover the 
Hamiltonian 
interpretation of his side conditions and its profound implications, we
pointed out that Lagrangians that are positively homogeneous of degree
one are particularly unsuited to a Hamiltonian formulation of the
necessary conditions for optimality in terms of the classical
Hamiltonian, so that for this class of Lagrangians the superiority 
of the control Hamiltonian is apparent.   

Such Lagrangians are in fact quite important in applications.  They occur,
for example, in relativistic mechanics, where we look at paths in
space-time and there is no natural ``time function'' to parametrize
space-like curves.

For example, the Lagrangian for the motion of a charged particle in an
electromagnetic field is
\begin{equation}\label{relatlagrangian}
L=-mc||\dot q||+\frac{\rho}{c} A(q)\cdot\dot q\,.
\end{equation}
Here 
\bi
\item[1.] the variable $q$ takes values in 4-dimensional Minkowski
space-time $Q$, 
\item[2.] $||\ldots||$ is length with respect to the
Lorentz-Minkowski metric, so that 
\begin{equation}
||v||=\sqrt{\langle v,v\rangle}\,,
\end{equation}
where $\langle\cdot,\cdot\rangle$ is the Minkowski inner product, given,
with respect to any Lorentz frame,  by
\begin{equation}
\langle v,w\rangle =\sum_{i=0}^3g_iv_iw_i\,,{\rm \,\,\, with
\,\,\,}g_0=c^2\,,\,\,\,g_1=g_2=g_3=-1\,,
\end{equation}
\item[3.] $q\mapsto A(q)$ is the electromagnetic potential, thought of as a 
field of covectors on $Q$, 
\item[4.]  $m$ is the rest mass, $\rho$ is the charge, and $c$
is the speed of light, 
\item[5.] $w\cdot v$ is the ordinary pairing between
the covector $w$ and the vector $v$, given in components by
\begin{equation}
w\cdot v=w_0v_0+w_1v_1+w_2v_2+w_3v_3\,.
\end{equation}
\item[6.] $L(q,\dot q)$ is defined for $q\in Q$ and $\dot q\in C$, where 
$C$ is the open cone of forward time-like vectors.
\ei

\begin{remark}{\rm 
Formula (\ref{relatlagrangian}) can be ``derived'' (or, more
precisely, shown to be mathematically natural) by the following elementary
invariance arguments.  The action functional $S=\int_a^bL(q(s),\dot
q(s))ds$ ought to be defined for a curve $s\mapsto q(s)$ independently
of the choice of parametrization, which forces $L$ to be positively
homogeneous of degree $1$ with respect to $\dot q$. For a free
particle, the Lagrangian should be translation-invariant and therefore
must be a function of $\dot q$ alone.  Lorentz invariance then dictates 
that it should be a function of $||\dot q||$ alone, and homogeneity 
then forces it to be proportional to $||\dot q||$.   The choice of 
a constant of proportionality equal to $\alpha mc$, for some dimensionless
scalar $\alpha$, is then needed for dimensional reasons.  

Finally, 
the fact that $\alpha=-1$ is necessary to have the appropriate 
nonrelativistic limit for small velocities.  
Indeed, let us  write $q=(q_0,q_{\#})$, relative to some Lorentz frame, 
where $q_{\#}$ is a 3-dimensional vector and $q_0$ is time, 
and use the frame-dependent physical time $t=q_0$ to parametrize our 
curves. Then 
\begin{equation}
\alpha mc||\dot q||=\alpha mc\sqrt{c^2-||\dot q_{\#}||^2}\,,
\end{equation}
where $||\dot q_{\#}||$
is now the ordinary Euclidean length of the 3-vector $\dot q_{\#}$. 
Therefore 
\begin{equation}
\alpha mc||\dot q||=
\alpha mc^2\sqrt{1-\frac{||\dot q_{\#}||^2}{c^2}}\,, 
\end{equation}
which behaves like $\alpha mc^2-\frac{\alpha m}{2}||\dot q_{\#}||^2$ 
as $\dot q_{\#}\to 0$.  Modulo an
additive constant---which in any case makes no contribution to the
Euler-Lagrange equation---this should look like the classical
kinetic energy  $\frac{ m||\dot q_{\#}||^2}{2}$ as $\dot q_{\#}\to 0$, 
so $\alpha$ must be equal to $-1$.  

As for the other term, 
given that the electromagnetic potential $A$ is a field of covectors, 
the simplest way to associate to 
$A$ a function of $q$ and $\dot q$ which is  positively homogeneous
of degree $1$ with respect to $\dot q$ is to take it proportional 
to $A(q)\cdot \dot q$, and the choice of $\frac{\rho}{c}$ for the 
constant is once again natural for dimensional reasons.}\proofend
\end{remark}

It is well known in physics that there are nontrivial difficulties when one
attempts to formulate the relativistic equations of motion in
Hamiltonian form.  For example, the textbook \cite{goldstein}, by
H. Goldstein, contains the following remarkable assertion, that 
we quote adding our own emphasis: 

\bqrm for Lagrangians homogeneous in the
$\dot q_i$'s in the first degree the usual Hamiltonian procedure does
not work.  A number of alternate ways of formulating the Hamiltonian
in the ``homogeneous problem'' have been devised. All take into
account explicitly the constraint implied by the vanishing of the
Jacobian determinant, but they are all complicated, and no one has
been accepted as the standard technique.  {\bem It appears that if
at all possible the homogeneous ``noninvertible'' Lagrangian should be
avoided like the plague.}  
\eqrm
For a not very convincing example of one of the ``alternate ways,''
the reader may consult Lanczos \cite{lanczos}.

On the other hand, 
a truly succesful approach was proposed by Tulczyjew in
1977 in \cite{tul1} (cf. also \cite{tul2}), using symplectic geometry.

{}From the control perspective, is it easy to see why there appears to
be a problem with the classical Hamiltonian formulation, since the
transformation (\ref{pimpl}) cannot  be inverted. On the other hand, 
{\bem the control Hamiltonian $H$ makes perfect sense, and can
be used to write the equations of motion, which turn out to be the 
usual relativistic equations of motion.}

To see this, write 
\begin{equation}
H=p\cdot u-L=
p\cdot u+mc||u||-\frac{\rho}{c} A(q)\cdot u\,,
\end{equation}
so that $H=\pi\cdot u+mc||u||$, where $\pi=
p-\frac{\rho}{c} A(q)$. 

Then $\frac{\partial H}{\partial u}=0$ yields 
\begin{equation}\label{releq1}
\pi_0=-mc^3\frac{u_0}{||u||}\,,\,\,\,\,\,
\pi_i=mc\frac{u_i}{||u||}\,\,\,\,{\rm for}\,\,\,\,i=1,2,3\,,
\end{equation}
while the other Hamilton equations say that 
\begin{equation}\label{releq2}
\frac{dq_i}{ds}=u_i\,,\,\,\,\,\,
\frac{dp_i}{ds}=\frac{\rho}{c}\sum_j\frac{\partial A_j}{\partial q_i}(q)\,u_j\,.
\end{equation}
Naturally, the ``time'' parameter $s$ here is not physical time.
Physical time is one of the state variables, namely, the component
$q_0$ of the vector $q$, and depends on the choice of a Lorentz frame.  
The quantity  $s$ is just a parameter for the curve $q(\cdot)$. Notice that 
the system of equations (\ref{releq1}), (\ref{releq2}) is invariant under
arbitrary reparametrizations of the curve.   Notice also that 
the 4-momentum covector $\pi$ satisfies
\begin{equation}
||\pi||^2=m^2c^2\,,
\end{equation}
where $||\pi||$ is the norm $\sqrt{\langle \pi,\pi\rangle}$ 
of $||\pi||$ arising from the inner product 
\begin{equation}
\langle  p,p'\rangle=\frac{p_0p'_0}{c^2}-\sum_{i=1}^3p_ip'_i
\end{equation}
of covectors. This in particular 
implies the famous formula 
\begin{equation}
E=mc^2
\end{equation}
for the ``rest energy,'' i.e.,  the $0$-th component $E\equaldef\pi_0$ of
$\pi$ relative to a a frame with respect to which the particle is at rest.

{}From (\ref{releq2}) we get 
\begin{equation}\label{force}
\frac{d\pi_i}{ds}=\frac{\rho}{c}\sum_j\Biggl(
\frac{\partial A_j}{\partial q_i}(q)-\frac{\partial A_i}{\partial q_j}(q)\Biggr)
\,u_j\,,
\end{equation}
i.e.,
\begin{equation}
\frac{d\pi_i}{ds}=\frac{\rho}{c}\sum_j  F_{ij}(q)u_j\,,
\end{equation}
where
\begin{equation}
F_{ij}=\frac{\partial A_j}{\partial
q_i}-\frac{\partial A_i}{\partial q_j}\,.
\end{equation}
If we now choose a Lorentz frame, use the time 
$q_0$ as the curve parameter, and write 
$v$ to denote the vector $(\dot q_1,\dot q_2,\dot q_3)$,
we get $\dot q_0=1$,  and
$||\dot q||=\sqrt{c^2-||v||^2}$.  (The dots now denote derivatives 
with respect to $q_0$.)  So
\begin{equation}
\pi_i=\frac{mv_i}{\sqrt{1-\frac{||v||^2}{c^2}}}\,,
\end{equation}
showing that $\pi_1$, $\pi_2$, $\pi_3$ are the components of the relativistic 
momentum of our particle.  Equation (\ref{force}) then 
gives us the time derivative of the momentum, i.e., the force acting on 
the particle. Setting $cE_i=F_{0i}$ for $i=1,2,3$, and 
$B_1=F_{23}$, $B_2=F_{31}$, $B_3=F_{12}$, as is customary, we find 
\begin{equation}
{\rm Force}=\rho\Big(E+\frac{1}{c}(v\times B)\Big)\,,
\end{equation}
which is the usual relativistic law of motion of a charged particle
in an electromagnetic field.

Clearly, the above argument involves no {\em ad hoc} considerations,
and is no more than a straightforward appplication of the 
Hamiltonian formalism, in ``control'' form. 

\cchapter[$\quad$The value function and the comeback of Hamilton's great
idea]{The value function and the comeback of Hamilton's great
idea\label{valuefun}}

\noindent Recall that, according to J.~L.~Synge's account of Hamilton's
``principal'' or ``characteristic'' function $V$ in \S \ref{synge-quot},
page \pageref{synge-quot}, 
one defines $V(A',A)$, for two points $A'$, $A$, to be the
minimum of the values of the integral (\ref{syngeint}) for all curves
joining $A'$ to $A$.  For this to make sense, one needs the ``rash''
assumption that there always is a curve that gives a minimum value to the
integral.

In modern terms, and without resorting to this ``rash assumption,''
we can define the {\sl value function} $V:Q\times Q\mapsto\bar{\rr}$,
for a controlled dynamics $\dot q=f(q,u)\,,\,\,q\in Q$ and 
Lagrangian $L(q,u)$, by letting $V(\bar q,\hat q)$ be, for a pair of
points $\bar q,\hat q\in Q$, the infimum of the integrals
$\int_a^bL(q(t),u(t))dt$, taken over all choices of $a$, $b$ and all
solutions $t\mapsto (q(t),u(t))$ of $\dot q(t)=f(q(t),u(t))$,
$t\in[a,b]$, such that $q(a)=\bar q$, $q(b)=\hat q$.  (Here 
$\bar{\rr}=\rr\cup\{-\infty,+\infty\}$.   The infimum is 
$+\infty$ if there are no trajectories of the system going from 
$\bar q$ to $\hat q$.) 

If we fix a ``target set'' $\mathcal{T}\subseteq Q$, then we can also
consider the value function $q\mapsto V_{\mathcal{T}}(q)$, defined by
letting 
\begin{equation}
V_{\mathcal{T}}(q)=\inf\{V(q,q'):q'\in\mathcal{T}\}\,.  
\end{equation}
It is clear that $V_{\mathcal{T}}$ is very closely related to 
Hamilton's principal function.  

Optimal control theory has fully vindicated Synge's view of the
importance of Hamilton's idea of the principal function, by making
this function (or, rather, its close relative $V_{\mathcal{T}}$) the
central object of the method of ``dynamic programming,'' one of the
theory's main tools.

According to this method, one can determine the value function 
$V_{\mathcal{T}}$ by solving a certain partial differential 
equation---the ``Hamilton-Jacobi-Bellman (HJB) equation''---with appropriate 
boundary conditions. One can then use the value function 
to find the optimal controls in ``feedback form.''

More precisely,  the HJB equation says  
\begin{equation}\label{hjbeq}
\inf\{\langle\nabla V(q),f(q,u)\rangle+L(q,u):u\in U\}=0\,.
\end{equation}
and it can be proved, under appropriate technical conditions,
that the function $V_{\mathcal{T}}$ satisfies (\ref{hjbeq})
and is the unique solution that satisfies a certain boundary condition.

However, $V_{\mathcal{T}}$ in general fails to be continuously 
differentiable, so (\ref{hjbeq}) has to be interpreted in a weak sense,
for example as a ``viscosity solution''   (cf. \cite{fleming} 
for details).   On the other hand, at those points $q$ where
$V_{\mathcal{T}}$ is differentiable and the infimum of (\ref{hjbeq}) 
is a true minimum, attained by a unique value $u(q)$ of 
of $u$, this value has a natural interpretation:  {\em $u(q)$ is the
initial value of the control that gives rise to an optimal 
trajectory from $q$ to the target.}  

For a simple example, consider the minimum-time optimal control
problem in $\rr^2$ with equations 
\begin{equation}\label{minpla}
\dot x=\frac{u}{1+y^2}\,,\,\,\,\dot y=v\,, 
\end{equation} 
with control constraint
$|u|+|v|\le 1$, and target $\{(0,0)\}$.  It is easy to see
intuitively, and to prove rigorously, that the optimal trajectory from
an initial point $(x,y)$ to the origin consists of a vertical segment
from $(x,y)$ to $(x,0)$, with $v=\pm 1$, followed by a horizontal
segment from $(x,0)$ to the origin, with $u=\pm 1$. The value function
$V$ is given by 
\begin{equation}
V(x,y)=|x|+|y|\,.  
\end{equation}
Notice that $V$ is not
continuously differentiable.  However, $V$ is continuously
differentiable on the set $S=\{(x,y):xy\neq 0\}$, and 
\begin{equation}\label{nabv} 
\nabla V(x,y)=({\rm sgn}(x),{\rm sgn}(y))\,\,\,\,{\rm for}\,\,\,\, 
(x,y)\in S\,.  
\end{equation}
The HJB equation says 
\begin{equation}
{\rm inf}\Bigl\{\frac{u.{\rm sgn}(x)}{1+y^2}+v.{\rm sgn}(y):|u|+|v|\le 1\Bigl\}
=-1\,,
\end{equation}
from which we get $v=-{\rm sgn}(y)$ and $u=0$ for 
$xy\neq 0$.  Notice that at the points of differentiability of 
$V$ we completely recover the optimal feedback from the function $V$.

On the other hand, it turns out that {\em the points of nondifferentiability 
of $V$ play a crucial role}, even though they form a small set.
(For example, they form a set of measure zero, and in fact 
of Hausdorff codimension 1.) Indeed, it is at these points that the optimal 
control ``turns'' and becomes horizontal rather than vertical.

This turns out to be a general feature: the value function is in
general not of class $C^1$, and the points of nondifferentiability
play a crucial role in determining the structure of the optimal
controls.  (For an even simpler example with a clear geometrical
interpretation, let $Q$ be the unit sphere in $\rr^3$, let the cost be
arc-length, and let the target be the South pole. Then $V(q)$ is the
distance along the sphere from $q$ to the South pole. Clearly, $V$ is
continuously differentiable everywhere except at the South and North
poles.  The HJB equation says that $||\nabla V||=1$.  Given any point
$q$ which is not one of the poles, the direction $u$ that minimizes
the inner product $\langle\nabla V(q),u\rangle$ is exactly the
direction of the meridian through $q$, pointing down, i.e., the
direction of the minimizing arc from $q$ to the South pole.  The
nondifferentiability of $V$ at the North pole is a reflection of the
fact that at this point the minimizing arc going to the South pole is
not unique.)

Therefore, while Synge was right in claiming that mathematicians
should have paid more attention to Hamilton's principal 
function, it is also true that their reluctance to proceed 
as if this function had nice regularity properties, was justified.
What was truly needed to do justice to Hamilton's idea was theory 
that would face, rather than avoid, the issue if nonsmoothness,
and would make it possible to deal with the value function 
as a truly nonsmooth object, as modern optimal control does. 

\ssection{The true meaning of the momentum variable}

\noindent The value function has turned out
to have many important properties that we will not enumerate here.  In
the remainder of this section, we will limit ourselves to describing
how the value function provides a nice interpretation of the meaning of the
momentum vector $p$.

In our outline of the evolution leading to the maximum principle,
we have seen how the meaning of the ``adjoint vector'' $p$---i.e., the
``momentum''---has changed.  Originally defined by the equation
\begin{equation}\label{newdefofp}
p(t)=\frac{\partial L}{\partial u}(q_*(t),\dot q_*(t),t)\,,
\end{equation}
(cf. (\ref{defofp})), the momentum was thought of as a time-varying
vector uniquely associated to a given curve $t\mapsto q_*(t)$.  
The Weierstrass side
condition can then be formulated in terms of this quantity, and says that
the function $u\mapsto\langle p(t),u\rangle -L(q_*(t),u)$, with 
$p(t)$ defined by (\ref{newdefofp}), i.e., by 
\begin{equation}\label{newdefofpnot}
p(t)=\frac{\partial L}{\partial u}(q_*(t),\dot q_*(t))\,,
\end{equation}
is maximized by the choice of $u=\dot q_*(t)$. 
(Recall that in Weierstrass' setting $L$ is supposed not to depend on $t$),

On the
other hand, without requiring {\it a priori} that $p(t)$ be given by
(\ref{newdefofp}), one could simply say that 
\bi
\item[(A)] {\em $p(t)$ is such that $u\mapsto\langle p,u\rangle -L(q(t),u,t)$ is
maximized by $u=\dot q(t)$}.
\ei
Since (A) implies (\ref{newdefofp}) anyhow, it would appear that
nothing is gained by substituting (A) for (\ref{newdefofp}). This
appearance, however, is quite deceptive, as was explained in
Chapter \ref{fromto}.    We saw there that 
{\em the gain achieved by  
using (A) instead of (\ref{newdefofp}) is in fact enormous,}
because (A) implies (\ref{newdefofp})
under the usual conditions of the classical calculus of variations,
but not otherwise. Hence, as soon as the ``usual conditions'' fail to be
satisfied, it turns out that (A) is still correct but (\ref{newdefofp}) 
is not.  (We have seen examples of this in Chapter \ref{fromto}. In Example 
\ref{toy1} $L$ fails to be differentiable 
with respect to $u$ at $u=\dot q(t)$, so (\ref{newdefofp}) does not make
sense.    In Example \ref{toy2} there is an inequality constraint 
on $\dot q(t)$, so (A) does not imply (\ref{newdefofp}) when 
the maximum is attained on the boundary of the set of permissible values
of $\dot q(t)$.)   

So (A) is much more fundamental than (\ref{newdefofp}) as a
characterization of ``the momentum along a curve.''  The substitution
of (A) for (\ref{newdefofp}) has a cost, however: while
(\ref{newdefofp}) clearly defines $p(t)$ uniquely, Condition (A) does
not, since there may be more than one value of the vector $p$ for
which (A) holds.  So we are now no longer allowed to talk about
{\bf\em the} momentum vector corresponding to a given curve, but only
of {\bf\em a} momentum vector.\Ft{with apologies to foreign readers whose
native language fails to make a precise distinction between definite
and indefinite articles.  Here the distinction is crucial.}  And in
fact the maximum principle then says that a necessary condition for
optimality of a trajectory is that there exist a momentum function
$t\mapsto p(t)$ that satisfies certain properties.

It turns out, however, that there is another identity about $p$ that
remains valid under much more general conditions, and provides
a much better interpretation of the ``true'' meaning of the momentum.
Precisely, it can be proved under very general conditions that 
{\bf\em the momentum $p$ is the negative of the
gradient of the value function}, as we now
explain.

We first illustrate the general situation  with a simple example.
\begin{example}{\rm 
Consider the minimum-time optimal control problem in $\rr^2$
discussed earlier, with equations (\ref{minpla}), 
control constraint $|u|+|v|\le 1$, and target $\{(0,0)\}$.  
As explained before, the value function $V$ is given by 
the formula $V(x,y)=|x|+|y|$, so 
$V$ is continuously differentiable on the set $S=\{(x,y):xy\neq 0\}$, and
(\ref{nabv}) holds. 

Pick a point $(\bar x,\bar y)\in S$, and let $\xi:[0,T]\mapsto\rr^2$ 
be the optimal trajectory from $(\bar x,\bar y)$ to $(0,0)$
(so that $T=|\bar x|+|\bar y|$).  Write
$\xi(t)=(x(t),y(t))$, and 
let $t\mapsto (u^\xi(t),v^\xi(t))$ be the
corresponding control. It then follows that
$u^\xi(t)=0$ and $v^\xi(t)=-{\rm sgn}(\bar y)$
for $0\le t\le|\bar y|$, and $v^\xi(t)=0$,  
$u^\xi(t)=-{\rm sgn}(\bar x)$ for $|\bar y|<t\le|\bar x|+|\bar y|$.  
The Hamiltonian is given by
\begin{equation}
H=\frac{p_xu}{1+y^2}+p_yv-p_0\,,
\end{equation}
where $p_x,p_y$ are the components of the adjoint covector $p$.

\def\pxxi{p_x^\xi}
\def\pyxi{p_y^\xi}
\def\dpxxi{\dot{p}_x^\xi}
\def\dpyxi{\dot{p}_y^\xi}

Using $\pxxi$, $\pyxi$ to denote the
components of the momentum along $\xi$, we get the equations
\begin{equation}
\dpxxi =0\,,\,\,\,\dpyxi =-\frac{2y\pxxi{}u^\xi}{(1+y^2)^2}\,,
\end{equation}
so that $\dpxxi\equiv\dpyxi\equiv 0$,  since
$y(t)u^\xi(t)\equiv 0$.  The maximum value of the Hamiltonian 
is 
\begin{equation}
M(t)=\tilde M(t)-p_0\,,\,\,\,{\rm where}\,\,\,\,
\tilde M(t)=\max\Biggl(\frac{|\pxxi(t)|}{1+y(t)^2},|\pyxi(t)|\Biggr)\,.
\end{equation}
Clearly, $\tilde M(t)\neq 0$ unless both $\pxxi(t)$ and $\pyxi(t)$
vanish. The possibility that 
$\pxxi(t)=\pyxi(t)=0$ is excluded because it implies $p_0=0$,
contradicting the nontriviality condition of the maximum principle. 
So  $\tilde M(t)>0$, and then $p_0>0$, so we may normalize the adjoint
vector and assume that 
$p_0=1$.  To maximize the Hamiltonian we must take 
\begin{equation}
u(t)={\rm sgn}(\pxxi(t))\,,\,\,\,v(t)=0\,\,\,\,{\rm if\,\,\,\,}
\frac{|\pxxi(t)|}{1+y(t)^2}>|\pyxi(t)|\,,
\end{equation}
and
\begin{equation}
u(t)=0\,,\,\,\,v(t)={\rm sgn}(\pyxi(t))\,\,\,\,{\rm if\,\,\,\,}
\frac{|\pxxi(t)|}{1+y(t)^2}<|\pyxi(t)|\,.
\end{equation}
It follows that at time $t=|y|$ the equality 
$|\pxxi(t)|=|\pyxi(t)|$ holds, from which it easily follows 
that $\pxxi(t)={\rm sgn}(\bar x)$ and $p_\xi^y(t)={\rm sgn}(\bar y)$.

This shows that 
\begin{equation}
(\pxxi(0),\pyxi(0))=-\nabla V(\bar x,\bar y)\,\,\,\,{\rm if}\,\,\,\,
\bar x\neq 0\,,\,\,\bar y\neq 0\,.
\end{equation}
In other words, {\em at every point $\bar q\in Q$ 
where the value function is differentiable,
if $\xi\!:\![0,T]\!\mapsto\! Q$ is an optimal trajectory from $\bar q$ to the
target, and $t\!\mapsto\! p(t)$ is the corresponding momentum vector given by
the maximum principle, normalized so that $p_0=1$, then the identity
\begin{equation}
p(0)=-\nabla V(\bar q)
\end{equation}
holds.}} \proofend
\end{example}
What has been shown in the above example turns out to be true under fairly
general conditions, at points where $V$ is of class $C^1$.   Moreover, 
the result identifying $p$ with $-\nabla V$ has been generalized 
to the case when $V$ is Lipschitz but not $C^1$, using the tools and
concepts of nonsmooth analysis (cf. 
\cite{clarkemethods,clarvin,fleming,subbotina} for details).   

\begin{remark}{\rm 
Often, the maximum principle is written using a Hamiltonian 
$H=\langle p,f(q,u,t)\rangle+p_0L(q,u,t)$, and requiring that
$u_*(t)$ minimize---rather than maximize---the function
$u\mapsto H(q_*(t),u,p(t),t)$.   This corresponds to a choice of 
$p$ equal to the negative of the $p$ we have been using here.
When this version of the maximum principle is used, the relationship 
between $p$ and $V$ is that $p(0)=\nabla V(\bar q)$.} \proofend
\end{remark}

\cchapter{From principle to theorem\label{frompri}}

\ssection{Principles and versions} Our discussion so far
has only dealt with the formal aspect of 
the necessary conditions for optimality. 
The results of the previous sections, from the Euler-Lagrange equation to
the maximum principle, should be regarded as {\em principles} rather than
theorems. For us, a {\em principle} is a generator of theorems, a not yet
completely precise statement that can be made into a theorem by filling in
the technical details and making all the definitions and conditions
completely precise.  The resulting theorems are {\em versions} of the
principle.  Usually, the choice of technical conditions can be made in more
than one way, so a ``principle'' has more than one version.  For example,
the statement that ``if the derivative of a function $f$ vanishes then $f$
is a constant'' is a ``principle'' in our sense.  One version is obtained
by taking $f$ to be defined on a subinterval $I$ of $\rr$, and requiring
that $f'(x)$ exist and be equal to $0$ for all $x\in I$. Another one may
just require $f$ to be continuous on $I$, and $f'(x)$ to exist and equal
zero for all $x$, except possibly for a finite or countable set.  A third
one requires $f$ to be absolutely continuous and $f'(x)$ to exist and equal
zero almost everywhere. A fourth one requires $f$ to be locally integrable,
and $f'$ to vanish in the sense of distributions. In the first three
versions, the conclusion is that $f(x)=0$ for all $x$, but in the fourth
one the conclusion only says that $f(x)=0$ almost everywhere.  One can
produce many more versions, e.g. by taking $f$ to be a distribution rather
than a function, or by going to higher-dimensional spaces, or to manifolds.

In some cases, a ``principle'' becomes identified in the minds of
mathematicians with its first published rigorous version. This has happened
to some extent in the case of the maximum principle, because the book
\cite{pontrybook}, where the result was first presented, already contains a
rigorous version. We will show below that this version does
not exhaust the full power of the principle, and the work of stating and
proving stronger and more general versions is still very much in progress.

In order to turn the principles of the previous sections into real
mathematical theorems, we have to be accurate as to the technical
assumptions on $L$ and, in the optimal control case, on $f$ and $U$. We
have to be rigorous as to the problem statement and the meaning of the
conclusions.  This requires, among other things, that the concept of an
``arbitrary function'' be replaced by a precise specification of the kind
of function involved.  Even for a simple dynamical law such as $\dot x=u$,
it would not make sense to solve for $x$ for a function $t\mapsto u(t)$
which is truly ``arbitrary'' in the sense of 20th century mathematics,
i.e., not even required to be measurable.  One needs at least to ask that
$u(\cdot)$ be integrable in the sense of some reasonable integration theory
such as those of Riemann, Lebesgue or Perron.  

These considerations would have been alien to 18th century mathematicians,
for whom the concept of function had not yet crystalized into a notion
susceptible of rigorous treatment.  For example, Lagrange wrote
(cf. \cite{struik}, pp.~388-9) that ``We define as a {\em function} of one
or several quantities any mathematical expression in which those quantities
appear in any manner.''  But then he went on to say that if $f(x)$ was a
function of one variable, then ``by the theory of series'' we can expand
$f(x+i)$, ``$i$ being an arbitrary quantity,'' in a series of the form
$f(x)+pi+qi^2+ri^3+\ldots...$, in which $p,q,r,\ldots$ are functions of
$x$. One wonders whether Lagrange would have accepted $f(x)=|x|$ as a
``function''---as he certainly should have, given his own definition of
function---and how he would have written the power series expansion of
$f(0+i)$ in this case.

Regarding the necessary conditions for optimality, while the discovery of
new and more general formal conditions progressed, rigorous versions of the
formal results were derived at various stages of the process, using in each
case the mathematical tools available at the time.  For example,
Weierstrass used the concepts of calculus in a perfectly rigorous and
modern way, and proved theorems that pass the most stringent 20th century
tests for rigor and precision. But the tools of 20th century measure and
integration theory were not yet available to him, so in many cases his
theorems make unnecessarily restrictive assumptions, such as that a
function $f$ is piecewise continuously differentiable, even though the
truly natural assumption would be that $f$ is absolutely continuous.

The story of how rigor emerged from the confusion of 17th and 18th century
mathematics is quite interesting in its own right, but we will not tell it
here in full detail.  We will only give a brief outline, concentrating only
on a few points that are either relevant for the final outcome---namely,
the maximum principle and its various versions---or particularly
remarkable in their own right.  We will then give a much more detailed
discussion of the work of translating the maximum principle into rigorous
mathematics.  This is still very much in progress, and we will attempt to
convey the flavor of some of the current research on the subject.

\ssection{Euler} It is well known that Euler was able to
perform amazing calculations that 
yielded correct results but by today's standards would be totally
unacceptable. (See, for example, the description in Stillwell
\cite{stillwell} of how he found the glorious formula
$\sum_{n=1}^\infty\frac{1}{n^2}=\frac{\pi^2}{6}$.)  For curve minimization
problems, his method was based on discretizing time, making finite
difference approximations for all the derivatives involved, obtaining a
finite difference equation for the solution of the approximating problem,
and then passing to the limit.  The integral of a function was approximated
by the area under a polygonal curve approximating the function's graph.

More accurately, one should keep in mind that
expressions such as ``passing to the limit'' are from a later
period. What Euler actually did was use the language of ``infinitely small
quantities'' and resort very often to ``geometric'' considerations: he
would divide the interval of definition of a function into ``infinitely''
many subintervals, and draw a picture (which, of course, only showed a
subdivision into finitely many intervals).  

\ssection{Lagrange}  While Euler's work was full of 
``geometric'' arguments, and contained many pictures,
Lagrange used no pictures.  (Newton's 1687 {\it Principia} had 250 figures.
Lagrange proudly stated in the preface of his 1788 {\it M\'ecanique
Analytique} that ``no diagrams will be found in this work.'')  
He introduced, by
analogy with the usual differentials $dx$, $dy$, another class of
differential-like objects, the ``variations''---for which he used
notations such as $\delta y$---which were thought of as infinitesimally
small functions, or infinitesimal functional increments of a function,
exactly as the ordinary $dx$, $dy$ are to be thought of as infinitesimal
changes of numerical variables.

So, for example, to solve a ``standard''  problem  (\ref{standard})
by Lagrange's method, we would set  $\delta I=0$, and then write
\begin{equation}\label{derivlag}
0=\delta I=\delta\int_a^b L\,dt=
\int_a^b(\delta L)\,dt=\int_a^b (M\,\delta q+N\,\delta\dot q)dt\,,
\end{equation}
where $M\!=\!\frac{\partial L}{\partial q}$, $N\!=\!\frac{\partial
L}{\partial\dot q}$. Using the commutation rule 
$\delta\dot q\,dt\!=\!\delta dq\!=\!d\delta q$, together with the
integration by parts rule 
$\int_a^b y\,dx= yx|_a^b-\int_a^b x\,dy$, we get 
$$
\int_a^b N\delta\dot q\,dt=\int_a^b N\,d\delta q=N\delta q\biggl|_a^b-
\int_a^b(\delta q)dN=N\delta q\biggl|_a^b-\int_a^b(\delta q)\dot N\,dt\,.
$$
So 
$0=\int_a^b(M-\dot N)\delta q\,dt+N\delta q\biggl|_a^b$.  Since $\delta q$
vanishes at $a$ and $b$,  because our variations have to preserve the
endpoint  conditions $q(a)=\bar q$, $q(b)=\hat q$,  we get 
$\int_a^b(M-\dot N)\delta q\,dt=0$. Since $\delta q$ is arbitrary,
we end up with $M=\dot N$, i.e., with (\ref{eulagfirst}). 

\begin{remark}\label{berkeley-rem}{\rm 
The precise meaning of Lagrange's ``variations'' was not clear to 
Lagrange or to his contemporaries. When Lagrange 
wrote there was a lot of confusion about the meaning of
``infinitesimal quantities,'' ``fluxions\Ft{This was Newton's term
for ``velocities,'' or ``celerities,''
of ``decrease or increase'' of ``fluents.''  
Newton's ``fluents,'' or ``flowing quantities,''  were ``quantities which 
I [i.e., Newton] consider as gradually and indefinitely increasing,'' and
were represented by letters such as $v$, $w$, $x$, $y$ and $z$, while 
``the velocities by which every fluent is increased 
by its generating motion (which I may call \textit{fluxions}, or
simply velocities, or  celerities), I shall represent by the same letters
pointed, thus $\dot v$, $\dot x$, $\dot y$, $\dot z$.'' (Quoted from
Cajori \cite{cajori}, p. 194.)},'' derivatives, and differentials.
It was clear that there were  rules for their manipulation of these
mysterious entities, and that these rules somehow appeared to lead to
consistent results. But the justification of the rules themselves
was lacking, as was a truly coherent account of the rules themselves. 
Mathematicians following these rules had to know when to apply them and
when to refrain to use certain arguments that the rules appeared to permit 
in order to avoid reaching a contradiction. For example, 
if $y=x^2$, and they wanted to compute $dy/dx$, they could write
\begin{equation}\label{need-a}
dy=(x+dx)^2-x^2=2x\,dx+(dx)^2\,.
\end{equation}
It was then permissible to divide both sides by $dx$, presumably by
invoking some kind of ``division rule,'' according to which
``if $a\cdot c=b\cdot c$ and $c\neq 0$ then 
$a=b$.''  This yields 
\begin{equation}\label{need-b}
dy/dx=2x+dx\,.
\end{equation}
They would then have to contradict themselves and arbitrarily set $dx=0$,
thereby obtaining  the desired foimula 
\begin{equation}\label{need-c}
dy/dx=2x\,.
\end{equation}
The problem here is, of course, that $dx\neq 0$ is neded
to go from (\ref{need-a}) to (\ref{need-b}), 
and $dx=0$ to go from (\ref{need-b}) to (\ref{need-c}).  So in fact 
the ``division rule'' would also have to be applicable in some cases
when $c=0$. But naturally, it could not be {\em always} applicable
when $c=0$, for this would have led to such arguments as
``$1\cdot 0=2\cdot 0$, therefore $1=2$.''  The precise 
conditions of applicability of the division rule were 
thus clouded in mystery, and the best that could be said was 
that the rule could be applied as long as it did not lead to a contradiction 
or an impossibility. 

With such methods of reasoning, it is not at all surprising that 
some mathematicians, in particular Nieuwentijt\Ft{The Dutch 
Bernhard Nieuwentijt (1654-1718) was a physician-burgomaster of
Purmerend, near Amsterdam.} as early as 1694, and, later,  Bishop George
Berkeley (1685-1753), the eminent philosopher,  criticized the foundations of
differential  calculus.  Berkeley was perfectly justified when he wrote in the
{\em Analyst}, in 1734,
that instantaneous rates of change were ``neither finite quantities, nor
quantities infinitely small, nor yet nothing.'' They were but ``the ghosts
of departed quantities.  Certainly $\ldots$ he who can digest a second or
third fluxion $\ldots$ need not, methinks, be squeamish about any points of
Divinity.''\Ft{According to Cajori 
(\cite{cajori}, p. 219), ``Berkeley's object in
writing the \textit{Analyst} was to show that the principles 
of fluxions are no clearer than those of Christianity. He referred 
to an `infidel mathematician'  (Edmund Halley), of whom the story is told
that, when he jested concerning theological questions, he was repulsed
by Newton with the remark, `I have studied those things, you have not.' 
A friend of Berkeley, when on a bed of sickness, 
refused spiritual consolation because the great mathematician Halley had 
convinced him of the inconceivability of the doctrines of
Christianity. This induced Berkeley to write 
the \textit{Analyst}.''}

If such was the situation regarding ordinary differentials, it was even
more so for variations, which were treated in a purely formal 
way.\Ft{Cf. \cite{goldstine}, p.~112, for a more detailed discussion of this
point, and also for the observation that Euler in 1771 ``imagined a family
of comparison curves $y=y(x,t)$ and took for his variation $\delta y$ the
quantity $dt(dy/dt)$,'' evaluated along the curve, and that Lagrange
pursued a similar course in his later work.}

Both Euler and Lagrange understood the criticisms perfectly well, agreed
with the critics, and accepted that the Calculus was unsound, but they
believed that it gave correct results anyhow because of cancellation of
errors. And their uncanny intuition enabled them to navigate 
in a sea of conceptual muddle and focus on the obstacles
that truly mattered (see, for example, footnote \ref{footlurk} in page
\pageref{footlurk}), without being distracted by those foundational issues
that later  generations would eventually settle in a completely rigorous 
way.} \proofend
\end{remark}


\ssection{The modern justification of Lagrange's derivation; the
Lavrentiev phenomenon}\label{lavrephen}  Lagrange's derivation 
of (\ref{eulagfirst}),
properly reformulated in the vector case, is exactly what we would still do
today on the formal level to get the basic equations of the calculus of
variations.  On the other hand, a rigorous mathematical proof given today
would assign a more precise meaning to the ``variations.'' We would begin
by specifying a space $F$ of $Q$-valued functions on $[a,b]$, for example
the space $C^1([a,b],Q)$ of $C^1$ maps from $[a,b]$ to $Q$.  We would then
define a ``variation'' (with fixed endpoints) of an arc $q_*$ to be a curve
$[-\bar\eps,\bar\eps]\ni\eps\mapsto q^\eps$---for some $\bar\eps>0$---in 
the space $F$, consisting of arcs $q^\eps$ such that 
the endpoint conditions 
$q^\eps(a)=\bar q$, $q^\eps(b)=\hat q$ always hold.  We would then observe 
that, if $q_*$ is a solution of our minimization problem, then
for very variation $[-\bar\eps,\bar\eps]\ni\eps\mapsto q^\eps$
the function $\eps\mapsto I(q^\eps)$ must have a minimum at $\eps=0$.
Therefore, if $\frac{d\,\,}{d\eps}\Bigl|_{\eps=0}I(q^\eps)$ exists, then 
it must vanish.   Writing $q(t,\eps)$ rather than $q^\eps(t)$, 
we would then let $\delta q$, $\delta\dot q$ stand for 
$\frac{\partial q}{\partial \eps}(t,0)$,  $\frac{\partial\,\,\,}{\partial
\eps}\frac{\partial q}{\partial t}(t,0)$,  respectively.
Then the commutation rule expresses the equality of the mixed 
derivatives: 
\begin{equation}
\frac{\partial\,\,\,}{\partial \eps}\frac{\partial q}{\partial t}=
\frac{\partial\,\,\,}{\partial t}\frac{\partial q}{\partial\eps}\,.
\end{equation}
(So $\delta q$ is an ``infinitesimal variation'' of $q_*$, and
$\delta\dot q$ can be thought of as the corresponding 
``infinitesimal variation'' of $\dot q_*$, or---thanks to the commutation
rule---as the time derivative of $\delta q$.)

To avoid unnecessary technical complications, we can just limit ourselves to
variations of the form $q(t,\eps)=q_*(t)+\eps r(t)$, where $r\in F$
is such that $r(a)=r(b)=0$.  In this case, the commutation rule 
is valid if, for example, $q_*$ and $r$ are of class $C^1$. 

Then the precise meaning of the formal calculations of (\ref{derivlag}) is, of
course, that we are differentiating the integral with respect to the
parameter $\eps$ at $\eps=0$: 
\begin{eqnarray}
&\frac{d\,\,}{d\eps}\Biggl|_{\eps=0}\int_a^b L\Big(q_*(t)+\eps r(t),\dot
q_*(t)+\eps\dot
r(t),t\Big)\,dt\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,
\,\,\,\nonumber\\
&\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,\,
=\int_a^b\biggl[\frac{\partial
L}{\partial q}(q_*(t),\dot q_*(t),t)\cdot r(t) +\frac{\partial
L}{\partial\dot q}(q_*(t),\dot q_*(t),t)\cdot\dot
r(t)\biggr]\,dt\,.\label{difunder}\\ \nonumber 
\end{eqnarray}
\noindent Clearly, this step needs careful justification.

Once this step is justified, we end up having rigorously proved
that $\int_a^b(Mr+N\dot r)=0$. An integration by parts---which also
needs to be properly justified---then yields
$\int_a^b(M-\dot N)r=0$.  Finally, since this 
is true for all $r$, we should be able to conclude that 
$M=\dot N$, i.e., the that Euler-Lagrange equation holds.

So there are three steps that need rigorous justification, namely, 
\bi
\item[(i)] the differentiation under the integral sign (\ref{difunder}), 
\item[(ii)] the integration by parts, 
\item[(iii)] the passage from ``$\int_a^b F.r=0$ for all $r$'' to $F\equiv 0$. 
\ei

The proper justification of (i) and (ii) are delicate problems requiring
precise technical hypotheses. The reader should have no difficulty
verifying that, under appropriate smoothness conditions on $L$ (for
example, if $L$ is of class $C^1$). Step~(i)~can be rigorously justified if
$q_*$ and $r$ are Lipschitz continuous, but cannot be justified if $q_*$ is
only known to be absolutely continuous, even if $r$ is very nice, e.g.  a
polynomial.

One can also justify (\ref{difunder}) for more general Lagrangians,
using the Lebesgue dominated convergence theorem. This requires
integral bounds. It suffices to assume, for example, that 

\medskip
\bi
\item[(H1)] {\sl $L(q,u,t)$
is of class $C^1$ with respect to $q$ and $u$ for each fixed $t$,}
\item[(H2)] {\sl $L(q,u,t)$ is measurable with respect to $t$ for each
fixed $(q,u)$,}
\ei
and
\bi
\item[(H3)] {\sl there exist $\alpha\!>\!0$ and $\varphi\!\in\!L^1$ such
that}
\vskip -0.15cm
\begin{equation}\label{intobo}
\!\!\!\!\!\!\!\!\!\!\!\!
\Biggl\|\frac{\partial L}{\partial q}(q,u,t)\Biggr\|
\!+\!\Biggl\|\frac{\partial L}{\partial u}(q,u,t)\Biggr\|\!\le\!\varphi(t){\rm \,\,\,
for}\,\,\,||q\!-\!q_*(t)||\!\le\alpha\,,\,\,||u\!-\!\dot
q_*(t)||\!\le\!\alpha\,.  
\end{equation}
\ei

\medskip\noindent 
Notice that these conditions automatically hold if $L$ is $C^1$ with
respect to $q,u,t$ and $q_*$ is Lipschitz, but need not hold,
even if $L$ is a polynomial in $(q,u,t)$, when $q_*$ is absolutely
continuous but not Lipschitz, since in this case $\dot q_*$ could be
unbounded and then (\ref{intobo}) does not follow.  

For example, even if $L$ is very smooth---say a polynomial in all 
three variables---it can happen that the 
differentiation under the integral sign {\em cannot} be justified  
for a  solution
$q_*(\cdot)$ of the minimization problem in the class $AC$ of absolutely
continuous curves and in fact {\em this solution 
does not satisfy the Euler-Lagrange equation}. This is
known as the ``Lavrentiev phenomenon,'' discovered in 1926
(cf. \cite{lavre}, \cite{cesari} \S18.5, \cite{lavreball}). 

Step (ii)---the integration by parts---also poses a delicate problem.  The
simplest way to justify it is to assume that $L$ is of class $C^2$ with
respect to all its variables and the curve $q_*$ is of class $C^2$, 
in which case the function 
$t\mapsto \frac{\partial L}{\partial\dot q}(q_*(t),\dot q_*(t),t)$
is a composite of functions of calss $C^1$ and hence of class $C^1$.  

Finally, to carry out Step (iii), and 
infer that $M\equiv \dot N$ from the fact that 
$\int_a^b(M-\dot N)r\,dt=0$ ``for arbitrary $r$,''
one has to be precise about the meaning of ``arbitrary $r$,''
and then one has to give a rigorous proof. For example, if 
``arbitrary $r$'' means ``continuously differentiable
$r$ vanishing at the boundary of the interval $[a,b]$,''
then one can use some result such as the
``duBois-Raymond Lemma,'' better known today as the ``fundamental lemma of
the theory of distributions,'' according to which, 
if a locally integrable function $\varphi:[a,b]\mapsto\rr$ 
satisfies $\int_a^b\varphi(t)\psi(t)dt=0$ for all 
$\psi\in C^1([a,b],\rr)$ such that $\psi(a)=\psi(b)=0$, then 
$\varphi(t)=0$ for almost all $t$. 


A more sophisticated analysis, using needle variations, shows that many of
the above hypotheses are not needed, as will be shown below.
The truly important conditions are,
first of all, that $L$ be of class $C^1$ with respect to $q$ and
$||\frac{\partial L}{\partial q}(q,u,t)||$ satisfy an integral bound.
Rather surprisingly, {\em the integral bound for $\frac{\partial
L}{\partial u}(q,u,t)||$ and even the differentiability of $L$ with respect
to $u$, can be dispensed with altogether,} provided that one reinterprets
(\ref{eulagfirst}) appropriately, as was already explained in 
Chapter \ref{fromto}.
This may seem fairly surprising at first, since the elementary derivation
of (\ref{eulagfirst}) that we have suggested appears to require the
existence and boundedness of $\frac{\partial L}{\partial u}(q,u,t)$ in a
crucial way. (Actually, it may even appear that $\frac{\partial L}{\partial
u}$ itself has to be differentiable, since its derivatives are needed in
order to be able to write the left-hand side of (\ref{eulagfirst}).) The
true explanation of this strange fact is that Conjecture M2, in
Chapter \ref{fromto}, is true, and the statement of Conjecture M2 contains no
derivatives of $L$ with respect to $u$.


\ssection{Legendre}\  Legendre used the same formalism of
variations, and derived his 
second-order necessary condition by looking at the ``second variation''
$\delta^2I$.  We now present his derivation, mainly to persuade
the reader that it is not at all simple, and contains at least one argument
which is rather ingenious but somewhat artificial and has a 
minor\Ft{Relative to the standards of rigor of the time.  
We are of course ignoring the lack of rigor of the language of
infinitesimal quantities, differentials  and variations.} flaw.   

Let us proceed formally, using  
Lagrange's $\delta$ notation, and compute $\delta^2I$, at a curve 
that already satisfies $\delta I=0$, setting
$$
P=\frac{\partial^2 L}{\partial q^2}\,,\,\,\,
Q=\frac{\partial^2 L}{\partial q\partial\dot q}\,,\,\,\,
R=\frac{\partial^2 L}{\partial\dot q^2}\,.
$$ 
We get 
$$
\delta^2I=\int_a^b(M\,\delta^2q+N\,\delta^2\dot q+P(\delta q)^2+
2Q\,\delta q\delta\dot q+R(\delta\dot q)^2)\,.
$$
Since $M\delta^2q+N\delta^2\dot q$ is the derivative of $N\delta^2q$
(because $M=\dot N$, since $\delta I=0$), and $\delta q$ vanishes at the 
endpoints,  we get $\int_a^b(M\,\delta^2q+N\,\delta^2\dot q)=0$, so 
\begin{equation}\label{write}
\delta^2I=\int_a^b(P(\delta q)^2+2Q\,\delta q\delta\dot q+R(\delta\dot q)^2)\,.
\end{equation}
Legendre then had to deal with the problem of inferring 
that $R\ge 0$
{}from the fact that $\delta^2I\ge 0$ for ``all variations'' $\delta q$. 
\begin{remark}{\rm \label{remneedle}
Today, we would take care of this using ``needle variations'':
take $q(t,\eps)=q(t)+u_\eps$, where $u_\eps$ is a continuous function
supported in an  interval $[t,t+2\eps]$, with value $\sqrt{\eps}$ at
$t+\eps$, and linear on   $[t,t+\eps]$ and on $[t+\eps,t+2\eps]$.  An easy
calculation then shows that, 
as $\eps\downarrow 0$, $\delta^2I$ behaves like  $2R(q(t),\dot q(t),t)$.
So $\delta^2I\ge 0$ implies $R\ge 0$.

The point of this argument is that, by allowing $u_{\eps}$ to 
be supported in an interval of length proportional to $\eps$, 
we are guaranteeing that $\dot u_{\eps}$ will be much larger than
$u_{\eps}$ as $\eps\to 0$, thus making sure that the last term of the 
right-hand side of (\ref{write}) dominates the other ones. \proofend
}\end{remark}
This method  was not available to Legendre, so he used another, very 
ingenious  trick.  Observe that if $\alpha$ is an arbitrary function,
then the integral $\int_a^b(\dot\alpha(\delta q)^2+2\alpha\,\delta
q\,\delta\dot q)$ is equal to $0$, 
since the integrand is the derivative of $\alpha(\delta q)^2$, and 
$\delta q=0$ at the endpoints.  So we can rewrite (\ref{write}) as
\begin{equation}\label{rewrite}
\delta^2I=\int_a^b((P+\dot\alpha)(\delta q)^2+2(Q+\alpha)\,\delta q\delta
\dot q+R(\delta\dot q)^2)\,.
\end{equation}
Now suppose we can choose $\alpha$ to be a solution of the differential
equation  
\begin{equation}\label{legeq}
(P+\dot\alpha)R=(Q+\alpha)^2\,.
\end{equation}
Then $\delta^2I=\int_a^bR^{-1}((Q+\alpha)\delta q+R\delta\dot q)^2$, which is 
$<0$ if $R<0$. So $R$ has to be $\ge 0$.   

This proof is correct except for the fact, pointed out by Lagrange in 1797,
that (\ref{legeq}) need not have a solution defined on the whole interval
$[a,b]$. The reader will recognize (\ref{legeq}) as a Riccati differential
equation.\Ft{Here we see the first mention of the Riccati differential
equation in relation to problems of the optimal control type. It wasn't the
last!  Matrix versions of this equation have since the 1960s come to play
a central role in systems and control theory. Stationary points of this
quadratic differential equation are determined by quadratic matrix
equations, which are called Algebraic Riccati Equations.  Finding
conditions for the existence of real solutions of this algebraic matrix
equation is like finding generalizations of the discriminant condition for
quadratic polynomials.  Matrix Riccati equations yield very effective
algorithms for computing optimal (feedback) gains in linear quadratic
control problems, in Kalman filtering, and in $H^{\infty}$ problems.}  It
is well known that such equations can have explosions in finite time.
Lagrange, who used $v$ to denote $-\alpha$, and $x$ for the independent
variable, wrote that ``it would be necessary for the correctness of this
result that one be able to make sure that the value of $v$ could not become
infinite for a value of $x$ lying between the given values of $a$ and $b$,
which will most often be impossible to do because of the difficultly in
finding the function $v(x)$.''\Ft{\label{footlurk}Lagrange obviously 
understood that a solution could fail to exist globally on the full
interval. Yet, one wonders what differences 
between his picture of the mathematical universe and our own might lurk
behind his reference to ``the difficultly in finding the function'' rather
than to ``the possibility that the function might not exist.''
But what makes Lagrange's criticism of Legendre's argument truly amazing 
is how, at a time when the calculus was rife with confusion and 
poorly justified reasoning, Lagrange was able to zero in on a real
mathematical difficulty (namely, that a certain initial value problem
for a differential equation may fail to have a global solution), 
while he appeared to have no problem using arguments that he 
knew were unsound when working with infinitesimal quantities, cf. Remark
\ref{berkeley-rem} in page \pageref{berkeley-rem}.} 

This, however, is a rather minor flaw, as was shown by Weierstrass in
1879. All one has to do to fix this aspect of Legendre's proof is to
observe that given any $t\in[a,b]$ there is a small interval $J$ containing
$t$ in its interior and such that a solution of (\ref{legeq}) exists on
$J$.  Since the restriction to $J$ of a solution of (\ref{standard}) is a
solution of the minimization problem on $J$, we can apply Legendre's
argument to this restriction, and conclude that $R\ge 0$ on $J$.  Since
this is true on a neighborhood of every point of $[a,b]$, Legendre's
conclusion follows.

\ssection{The needle variations approach}  

\gnu{{\bf This section still has to be written.}}

\ssection{The classical version of the maximum
principle}\ The first rigorous version of the maximum principle
appears in the book \cite{pontrybook}. This ``classical'' version was
then improved by other authors, e.g. L. Cesari \cite{cesari},
M. Hestenes \cite{hestenesbook}, E. B. Lee and L. Markus
\cite{leemarkus}, and L. D. Berkovitz \cite{berkovitz}.  We choose to
quote a version appearing in Berkovitz's 1974 book. 

We let $f^1,\ldots,f^n$ be the components of $f$, and write $f^0$ for $L$.
It is assumed that the $f^i$, for $i=0,\ldots,m$, 
are defined on $Q\times U_0\times[a,b]$,  
where $Q$, $U_0$ are open subsets of $\rr^n$, $\rr^m$, respectively.
Moreover, each function $q\mapsto f^i(q,u,t)$ is required to be of class
$C^1$ with respect to $q$ for each $(u,t)\in U_0\times[a,b]$, and each map
$(u,t)\mapsto f^i(q,u,t)$ has to be Borel measurable for each 
fixed $q\in Q$. The set $U$ is a subset of $U_0$. 
An {\em admissible control} is a map $[a,b]\!\mapsto\!u(t)\!\in\!U$ such that 
for every compact subset $K$ 
of $Q$  there is an integrable function 
$t\mapsto \varphi_K(t)$ such that the bound 
$||f^i(q,u(t),t)||+||\frac{\partial f^i}{\partial q}(q,u(t),t)||
\le\varphi_K(t)$ holds for all $(q,t)\in K\times[a,b]$ and all
$i=0,\ldots,m$.
For a general class $\mathcal{ U}$ of $U$-valued functions on $[a,b]$, 
and $\bar q,\hat q\in Q$,  let us
use  $\mathcal{ C}(\mathcal{ U},\bar q,\hat q)$  to denote the set of all pairs 
$(q(\cdot),u(\cdot))$ such that $u(\cdot)\in\mathcal{ U}$, 
$q(\cdot)$ is a solution of (\ref{dyna}) (i.e., $q(\cdot)$ is an absolutely 
continuous curve $[a,b]\mapsto Q$ such that (\ref{dyna}) holds for almost
every~$t$),  $q(a)=\bar q$, and $q(b) =\hat q$.
Use $\mathcal{ U}_{adm}$ to denote the class of 
all admissible controls. Then the 
optimization problem is that of minimizing 
the integral $I\!=\!\int_a^bL(q(t),u(t),t)\,dt$ in the class 
$\mathcal{ C}(\mathcal{ U}_{adm},\bar q,\hat q)$.
The conclusion of the theorem is that 
of (MP), with $p_*$ absolutely continuous, and 
the adjoint equation and the maximization condition 
holding almost everywhere.  

One could also weaken this, and work with a more restricted class $\mathcal{
U}$ of controls. (For example, we may want $\mathcal{ U}$ to be
the class of all piecewise constant maps from $[a,b]$ to $U$, or the class
of all piecewise continuous maps.)  In order to state the result in this
case, let us use the notation $u\#_Ev$ to denote, if $u,v$ are maps
defined on $[a,b]$, and $E$ is a subset of $[a,b]$, the map
$w$ on $[a,b]$ defined by ``substituting $v$ for $u$ on $E$,''
that is, $w(t)=u(t)$ for $t\notin E$, $w(t)=v(t)$ for $t\in E$. 

With this terminology, the ``classical'' result is also valid for the
minimization problem in $\mathcal{ C}(\mathcal{ U},\bar q,\hat q)$, where $\mathcal{
U}$ is any class of controls such that {\em (a)~$\mathcal{ U}\subseteq\mathcal{
U}_{adm}$, (b)~$\mathcal{ U}$ is closed under interval substitutions (that is,
if $u(\cdot)\in\mathcal{ U}$, $v(\cdot)\in\mathcal{ U}$, and $J$ is a subinterval
of $[a,b]$, then $u\#_Jv\in\mathcal{ U}$), and (c)~all constants belong to
$\mathcal{{U}}$.}  The conclusion is the same as in the classical result.
\begin{remark}{\rm 
If $\mathcal{ U}\!=\!\mathcal{ U}_{adm}$, then $\mathcal{ U}$ satisfies 
(a) and  (b) but not necessarily~(c).   So, strictly speaking, the
``classical'' result does not follow from the extension discussed in the
previous paragraph. It is easy, however, to find a hypothesis 
that includes both. What is needed is to substitute for (c) the weaker
requirement (c')~{\em for every $u(\cdot)\!\in\!\mathcal{ U}$ and every 
constant map $v:[a,b]\!\mapsto\!U$ there exist compact subsets $E_j$ of
$[a,b]$ such that (i)~$u\#_{E_j}v\in\mathcal{ U}$ for all $j$, 
and (ii)~${\rm meas}([a,b]\backslash E_j)\to 0$ as $j\to\infty$.}

One can easily verify that (c') always holds when
$\mathcal{{U}}=\mathcal{{U}}_{adm}$, and it is trivial that  (c) implies
(c'). \proofend 
}\end{remark}
\begin{remark}{\rm 
The hypothesis that $U$ is a subset of $\rr^m$, often found in early works 
on the maximum principle, is not needed. One can just take $U$ to be a
completely arbitrary separable metric space, and nothing changes.

\gnu{Rewrite the paragraph below.}

There is even a more general version where $U$ is an arbitrary set, with no
structure whatsoever, but in this case one has to be more careful about the
technical hypotheses and the conclusion.  Full details will be given in
\cite{suss5,suss6,suss7}.  \proofend
}\end{remark} 
\begin{remark}{\rm  
Notice that with the above assumptions it need not be true, for a general
calculus of variations problem with a Lagrangian $L(q,u,t)$ which is very
smooth---e.g. polynomial---in all three variables, that every integrable
function $u:[a,b]\mapsto Q=\rr^n$ is an admissible control.  (Just consider
the trivial example of $f(x,u,t)=u^2$, with $Q=U=\rr$, $a=0$, $b=1$, and
take $u(t)=\frac{1}{\sqrt{t}}$.)
This makes it at least understandable why the ``Lavrentiev phenomenon''
mentioned earlier can occur. \proofend
}\end{remark}

The proof of this first version of the maximum principle 
is rather long, and we will not
present it in full or even sketch it here.  But we will mention a couple of
ideas that go into this proof and are crucial for the understanding of
generalizations to be discussed later.

First of all,  one observes that the reference control $u_*(\cdot)$ gives rise
to a {\em flow}, i.e., a family of maps $\Phi_{t',t}$, defined 
by letting $\Phi_{t',t}(q)=\xi_{q,t}(t')$, where 
we use $\xi_{q,t}(\cdot)$ to denote the
solution of $\dot\xi(\tau)=f(\xi(\tau),u(\tau),\tau)$, $\xi(t)=q$. 
These flow maps are of class $C^1$, and their differentials 
$D\Phi_{t',t}(q)$ can be computed by solving the variational equation 
$\dot y(\tau)=\frac{\partial f}{\partial q}(\xi_{q,t}(\tau),u(\tau),\tau)\cdot
y(\tau)$. The value $D\Phi_{t',t}(q).v$ is then $y_{q,t,v}(t')$, where 
$y_{q,t,v}$ is the solution $y$ of the variational equation for which 
$y(t)=v$.  

One then makes ``needle variations,'' in which the reference
control $u_*$ is replaced, over a finite union
$[t_1,t_1\!+\!s_1]\cup\ldots[t_\nu,t_\nu\!+\!s_\nu]$ of intervals, by 
a control $u$ with constant values $u_i$ on $[t_i,t_i\!+\!s_i]$.
This gives rise, for a fixed ${\bf t}\!=\!(t_1,\ldots,t_\nu)$ and 
${\bf u}\!=\!(u_1,\ldots,u_\nu)$, to a
$\nu$-parameter family $\{u_{{\bf s}}\}_{{\bf s}\in\rr_+^\nu(\sigma)}$ 
of controls, for some $\sigma>0$.
(Here $\rr_+^\nu$ is the nonnegative orthant in $\rr^\nu$, and 
$\rr_+^\nu(\sigma)\!=\!\{s\!\in\!\rr_+^\nu\!:\!s_1\!+\!\ldots\!+\!s_\nu\le\sigma\}$.)
The $u_{{\bf s}}$ in turn determine 
a $\nu$-parameter family $\{q_{{\bf s}}\}_{{\bf s}\in\rr_+^\nu(\sigma)}$ 
of trajectories---keeping the
initial condition $q_{{\bf s}}(a)\!=\!\bar q$ fixed---and then a map
${\bf s}\!\mapsto\!q_{{\bf s}}(b)\!=\!\gamma({\bf s})\in Q$. 

One then computes the differentials of these maps at 
${\bf s}=0$, and the differential of the corresponding cost.
The computation is done by means of the chain rule.   For example,
$$
\frac{\partial \gamma}{\partial s_i}(0)
=\frac{d\,\,}{ds}\Biggr|_{s=0}\Big(\Phi_{b,t_i}
\Phi_{t_i,t_i+s}\Phi_{t_i+s,t_i}^{u_i}\Phi_{t_i,a}(\bar q)\Big)\,,
$$
where $\Phi_{t',t}^u$ are the flow maps corresponding to the constant
control $u$.  Since 
\begin{equation}\label{needjust}
\Phi_{t_i,t_i+s}\Phi_{t_i+s,t_i}^{u_i}(q)
=q+s\Big(f(q,u_i,t_i)-f(q,u_*(t_i),t_i)\Big)+o(s)\,,
\end{equation}
we see that $\frac{\partial \gamma}{\partial s_i}(0)$ is obtained by
solving the  
variational equation with initial condition $y(t_i)=
f(q,u_i,t_i)-f(q,u_*(t_i),t_i)$, and evaluating the solution at time $b$.
The justification of (\ref{needjust}) when $f(q,u_i,t)$ and $f(q,u(t),t)$
are jointly continuous in $q$ and $t$ is trivial. In the general case,
it is a much more a delicate point, and the conclusion is not valid for all 
possible choices of the $t_i$, but only for some special $t_i$'s. However, 
these
$t_i$'s form a set of full measure, so the ``almost everywhere'' conclusion
of the theorem is still valid.


With a lot of work, these basic facts lead to the versions of 
(MP) described above, which are essentially those of
\cite{pontrybook} and \cite{berkovitz}.  

\ssection{Recent versions of the maximum principle: the nonsmooth connection\label{nonsmoothcon}}

\noindent Stronger versions have been obtained by weakening the
hypothesis of the first version, or strengthening the conclusions, or both.

\ssection{The nonsmooth analysis approach} 
One important improvement of the classical version resulted from the
use of {\em nonsmooth analysis} (cf. Clarke
\cite{clarkeminimal,clarkebook}).  Clarke replaced the continuous
differentiability of $f$ and $L$ with respect to $q$ by the weaker
requirement that $f$ and $L$ be Lipschitz with respect to $q$.   
(Other than that, the hypotheses are exactly as above.  Naturally,
in all the integral bounds the term 
$||\frac{\partial f^i}{\partial q}(q,u(t),t)||$ has to be replaced by a 
Lipschitz constant for $f^i(\cdot,u(t),t)$ in the appropriate 
set $K$.)

This of course renders the meaning of the adjoint equation problematic,
since 
\begin{equation}
\frac{\partial H}{\partial q}=\langle p,
\frac{\partial f}{\partial q}\rangle-\frac{\partial L}{\partial q}\,,
\end{equation}
which in principle does not make sense unless $f$ and $L$ are differentiable 
with respect to $q$.   Clarke, however, introduced the ``generalized
Jacobian'' $\partial F(q)$ of a Lipschitz map $F$ at a point $q$, which is a
{\em set} of linear maps rather than a single linear map. Using  this 
object, one can still assign a meaning to the adjoint equation in the
Lipschitz case, except that one now obtains an {\em adjoint differential
inclusion} rather than an equation.   

More recently, S. Lojasiewicz Jr., in work still unpublished, observed that
in the construction of needle variations one never uses the derivatives
with respect to $q$ of the functions $f^i(q,u_j,t)$, and that the only
place where these derivatives appear is in the adjoint equation, which only
involves the reference control $u_*$. This enabled him to extend the
maximum principle even further, by dispensing with the hypothesis that
$f(q,u,t)$ is Lipschitz in $q$ for all $u$, $t$, and requiring only that
$f(q,u_*(t),t)$ be Lipschitz in $q$ for each $t$---with a Lipschitz
constant $C(t)$ integrable with respect to $t$---while $f(q,u,t)$ for
general $u,t$ is only required to be continuous with respect to $q$.

In addition, a second direction developed in which (MP) is formulated not 
for controlled differential equations $\dot q=f(q,u,t)$, but for 
{\em differential inclusions} $\dot q\in F(q,t)$, where $F$ is a set-valued
map (cf. for example \cite{clarkebook}).   

\ssection{High-order necessary conditions} While the ``nonsmooth'' 
generalizations were being developed, other
authors pursued a different direction, for very smooth systems.
They observed that one could get
stronger results by allowing a class of variations richer than 
that used in the classical proof. 
One can then obtain ``high-order necessary conditions for optimality,'' in
which the inequality $\langle p(t),v\rangle\ge 0$ must hold not only for
$v$ of the form $f(q_*(t),u_*(t),t)-f(q_*(t),u,t)$, as in (MP), but also
for other 
vectors resulting from more elaborate variations involving Lie brackets.
(We are assuming here for simplicity that $L\equiv 1$, i.e., that we are
dealing with minimum-time control.)  This is a vast subject, and we refer 
the reader to \cite{belljac,gabakir,goh,kelley} for results and surveys 
of work done in the 1960s, and to
\cite{agra1,agra2,agra3,bress1,knobloch,krener} for later, more Lie-bracket
oriented, work. 

\ssection{The ``combined systems'' problem} The results discussed above
are proved by different methods and cannot be combined into a single
theorem.

At the risk of belaboring the obvious, we point out that 
{\bem extremality is a property which is not local with respect to time}.
That is, if 
the domain $I=[a,b]$ of definition of a 
trajectory-control pair $\gamma_*=(q_*,u_*)$ can be covered by intervals
$I_j$ such that the restriction $\gamma_{*,j}$
of $\gamma_*$ to each $I_j$ is an extremal, 
it does not follow that $\gamma_*$ itself is an extremal.   The reason is,
simply, that if there exist minimizing adjoint vectors $p_j$ for each 
$\gamma_{*,j}$, then these $p_j$ need not match, so it may be impossible to
use them to construct a minimizing adjoint vector for $\gamma_*$.  

\begin{remark}{\rm 
The classical calculus of variations is in this respect
exceptional and misleading, because its Hamiltonian is
$\langle p,u\rangle-L(q,u,t)$, so (MC) implies that $p(t)=\frac{\partial
L}{\partial u}(q_*(t),u_*(t),t)$, showing that $p$ is unique, and then the
various $p_j$ corresponding to the $I_j$ have to match.  So in this case
extremality is a local property. \proofend
}\end{remark} 
Because being an extremal
is not a local property, the following curious situation can arise.
Suppose we are dealing with a system $\Sigma$ (i.e., a dynamical law $f$ and
a Lagrangian $L$) which is ``hybrid,'' in the sense that $\Sigma$ satisfies
the hypotheses of the classical version of (MP) in some region $Q_1$, while
in some other region $Q_2$ it is very smooth, so that one can make
high-order variations involving Lie brackets, and in some other region
$Q_3$ it satisfies the conditions of Clarke's nonsmooth MP, while, finally,
there is a region $Q_4$ where $\Sigma$ is given by a differential
inclusion.  Suppose $q_*$ is an optimal trajectory
such that $q_*$ visits all four regions. Then we can apply each of the
four theorems to each part of $q_*$ that stays in one region, and
conclude that any such part is an extremal, but {\em we cannot combine them
all to get one minimizing adjoint vector $p_*$ on the whole interval of
definition of $q_*$ that satisfies in each part the corresponding
conclusion.}

We will not even attempt to explain why this is so, because to do it
we would have to discuss in detail the proofs of these theorems, showing
that in each case one uses a different construction, and these
constructions cannot be combined  into a single one valid on the whole
interval.  But it is a fact that, due to this incompatibility 
of the various proofs, a single theorem covering all cases and 
combining them---that is, applying to ``hybrid'' problems as 
above---appeared until a few years ago to be beyond reach.

\gnu{JAN: the following section was written before
I found the Generalized Differential Quotients and the 
Path-Integral Generalized Differentials. For a skecth of where
I am now, see my French paper. (Item No. 58 of the 
``papers avaliable in the Web'' link of my Web page.}



\ssection{A unified nonsmooth approach} Recently one of us (Sussmann
\cite{suss1,suss2,suss3,suss4,suss5,suss6,suss7}) has obtained a general
version of (MP) that 
contains all the above results, applies to some new cases as well, and 
actually covers the ``hybrid'' case. 

The key point of this work is that the main ideas of the classical proof of
\cite{pontrybook} apply under much more general conditions, provided only
that a suitable theory of ``generalized differentials'' is used.  This
theory need not be---and, in fact, {\em cannot be}---that of classical
differentials or that of Clarke's generalized Jacobian.  What is truly
needed is a definition of ``generalized differentials'' for which there is
a good calculus (and in particular a chain rule) and a nice open mapping
theorem.\Ft{The classical open mapping theorem says that if a continuous
map {\mit F} from an open subset {\mit U} of a finite-dimensional
real linear space {\mit X} to another such space {\mit Y} is such
that {\mit F} is differentiable at a point {\mit q} and the
differential {\mit DF(q)} is a surjective linear map, then {\mit
F} is {\mit open at q}, in the sense that {\mit F} maps
neighborhoods of {\mit q} to neighborhoods of {\mit F(q)}.  What
we need for a proof of the maximum principle under general conditions is
a generalization of this to a large enough class of maps,
including many  maps that are  not necessarily
differentiable in the ordinary sense, and not Lipschitz or even
single-valued (since, for example, flow maps generated by continuous vector
fields should be in the class, and these flow maps are in general
set-valued). The main point of \cite{suss1,suss2,suss3,suss4,suss5,suss6,suss7}
is to single out  
a class of maps which is large enough to contain all the maps that are
needed for the proof of the maximum principle, and at the same time 
such that a concept of ``generalized differential,'' with good 
properties---in particular, a chain rule and an open mapping 
theorem---exists for that class.}

In \cite{suss1,suss2,suss3,suss4,suss5,suss6,suss7} it is proposed to use the
theories of ``semidifferentials,'' ``multidifferentials,'' and
``generalized differential quotients. 
Basically, a continuous map $f$ is
``semidifferentiable'' at a point $x$ if it can be approximated to first
order near $x$ by a Lipschitz map.  (For example, the function
$f(x)=|x|+|x|^{3/2}\sin{\frac{1}{x}}$ is semidifferentiable at $0$, because
$f(x)=|x|+o(|x|)$. This function  does not, however, 
have a Clarke generalized Jacobian at $0$, because 
it is not Lipschitz near $0$, and it is clear that it does not have a classical
differential at $0$ either.)  This theory can also be extended to define
``semidifferentiability'' of certain set-valued maps, needed to handle
``needle variations'' when $f(q,u,t)$ is only continuous in $q$, so 
that the
flow maps $\Phi_{t',t}^u$ are set-valued.  It turns out that, using
semidifferentiable maps, one can prove a general version of (MP) that
contains and combines the classical, nonsmooth and Lojasiewicz versions.
In \cite{suss4,suss5,suss6,suss7,suss8} we propose two even more general
differentiability theories---the ``multidifferentials'' and the 
``generalized differential quotients''---which make it possible to extend (MP)
farther into a version that also covers the differential inclusions case.

\begin{remark}{\rm 
The conventional wisdom of the subject holds that differential inclusions
are more general than controlled dynamical systems, because a set-valued 
function usually does not have continuous selections.  While this is true,
A. Bressan has shown 
(cf. \cite{bress2,bress3,bress4,bress5,bress6,bress7})
that lower semicontinuous set-valued maps
admit ``conically continuous selections,''  and that these selections are as
good as continuous ones as far as the properties of the corresponding flows
are concerned.  Adapting Bressan's theory, it is shown in 
\cite{suss3,suss4,suss5} that the general differential inclusion case can
be reduced to 
the case of a controlled dynamical system, and one ends up with a result
that covers all cases. \proofend
}\end{remark}


%GUNGU

\cchapter{The proof of the maximum principle}\label{chapproofmax}

\ssection{Introduction and overview}

The purpose of this chapter is to present a detailed and  completely 
rigorous proof of a very general 
version of the Maximum Principle, giving a necessary condition for a 
trajectory-control pair to be a minimizer for an optimal control problem 
with endpoint constraints.  

Following  the approach of the book \cite{pontrybook}, we will
``geometrize'' the problem of necessary conditions for  optimal control,
by \mbox{forgetting} about the optimization, and 
working instead with a {\bf\em separation problem}, 
in which we seek necessary conditions for a set $\mathcal{S}$
of possible endpoint pairs 
to be separated (in sense to be made precise below,
cf. Def. \ref{locsep})  from the set $\evset(\Sigma)$ 
of endpoint pairs arising from a given system $\Sigma$
at a given.

Furthermore, the result for the separation problem will be proved
by \mbox{using} a \mbox{construction} similar to that utilized in
\cite{pontrybook}, based on making
``needle \mbox{variations}'' and combining them into ``packets of needle
variations.'' 

\begin{remark}
{\rm The idea of needle variations goes back to Weierstrass.}\proofend
\end{remark} 

\begin{remark}
{\rm It would also be possible to use needle variations to give a direct
proof of the \mbox{maximum} principle
for optimal control, without going through the reduction
to the separation problem, but the geometric method presented
here is more elegant and leads to more general results.}\proofend
\end{remark}

\pparagraph{The general setting: control systems, optimal control, and 
separation problems}\label{optisetting}
The general situation to be considered in our various versions of the 
maximum principle will involve 
\begin{itemize}
\item[1.] a {\bf\em state space} $\Omega$;
\item[2.] a set $\mathcal{P}$ of {\bf\em paths} in $\Omega$ 
or a set $\mathcal{P}^{a}$ of {\bf\em augmented paths} in $\Omega$;
\item[3.] an {\bf\em endpoint constraint set} $\mathcal{S}$;
\item[4.] a {\bf\em reference path} $\xi_*$, 
or {\bf\em reference augmented path} $\Xi_*$.
\end{itemize}
The precise definitions are as follows. Suppose 
$\Omega$ is a set. 
\begin{itemize}
\item[1.] A {\bf\em path} in $\Omega$ is a function
$\xi:[a_\xi,b_\xi]\mapsto\Omega$, defined on some closed bounded
subinterval $\dom(\xi)=[a_\xi,b_\xi]$ of $\rr$, known as the 
{\bf\em domain} of $\xi$.
\item[2.] An {\bf\em augmented path} in $\Omega$ 
is a path in $\rr\times\Omega$. If $\Xi:[a,b]\mapsto\rr\times\Omega$ 
is an augmented path in $\Omega$, and we write 
$\Xi(t)=(\xi^0(t),\xi(t))$ for $t\in[a,b]$, then the map
$\xi:[a,b]\mapsto\Omega$ is the {\bf\em state space path}
associated to $\Xi$, and the map $\xi^0:[a,b]\mapsto\rr$ is the 
{\bf\em running cost} for $\Xi$.
\item[3.] An {\bf\em endpoint constraint set} for $\Omega$ is a 
subset $\mathcal{S}$ of $\Omega\times\Omega$. 
\item[4.] The {\bf\em cost functional} for a set 
$\,\,\mathcal{P}^{a}\,\,$ of augmented paths in $\,\,\Omega\,\,$ is 
the \mbox{function}
${\cal J}_{\mathcal{P}^{a}}:\mathcal{P}^{a}\mapsto\rr$ that assigns 
to each $\Xi=(\xi^0,\xi)\in{\cal J}_{\mathcal{P}^{a}}$ the number
${\cal J}_{\mathcal{P}^{a}}(\Xi)$ given by 
\begin{equation}\label{defcostfunct}
{\cal J}_{\mathcal{P}^{a}}(\Xi)\,\,\equaldef\,\,\xi^0(b_\Xi)-\xi^0(a_\Xi)\,.
\end{equation}
\item[5.] The {\bf\em boundary} (or  {\bf\em endpoint condition}, or  
{\bf\em boundary value}, or  {\bf\em boundary \mbox{condition}}) 
of an arc $\xi:[a,b]\mapsto\Omega$
is the ordered pair $\partial\xi\in\Omega\times\Omega$ given by 
\begin{equation}\label{defboun}
\partial\xi\,\,\equaldef\,\,(\xi(a),\xi(b))\,.
\end{equation}
\item[6.] If $\mathcal{P}$ is a set of arcs in $\Omega$, we use 
$\partial\,\mathcal{P}$ to denote the set
$\{\partial\xi:\xi\in\mathcal{P}\}$ of all boundaries
of all arcs $\xi\in\mathcal{P}$, and refer to 
$\partial\,\mathcal{P}$ as the {\bf\em boundary value set}
of $\partial\,\mathcal{P}$.
\item[7.] If $\mathcal{P}^a$ is a set of augmented arcs in $\Omega$, 
we use $\partial\,\mathcal{P}^a$ to denote the set $\partial
\,\mathcal{P}$, where 
$\,\,\mathcal{P}=\{\partial\xi:(\xi^0,\xi)\in\mathcal{P}^a\}\,\,$
for some $\xi^0$, so $\partial\,\mathcal{P}^a$ is the set 
of all \mbox{boundaries} of all state space paths of the augmented
arcs $\Xi\in\mathcal{P}^a$. We  refer to 
$\partial\,\mathcal{P}^a$ as the {\bf\em boundary value set}
of $\partial\,\mathcal{P}^a$.
\item[8.] Suppose that $\,\mathcal{P}^{a}\,$ is a set of augmented paths in 
$\,\Omega\,$, and $\,\mathcal{S}\subseteq\Omega\times\Omega\,$ is an 
endpoint \mbox{constraint} set for $\,\Omega\,$. \ A 
{\bf\em $\,\,\,(\mathcal{P}^{a},\mathcal{S})$-minimizer} \ is an 
\mbox{augmented} path \mbox{$\Xi_*=(\xi_*^0,\xi_*)\in\mathcal{P}^{a}$}
such that $\partial\xi_*\in\mathcal{S}$ and 
\begin{itemize}\item[]\begin{itemize}
\item[(MIN)$\,\,$] {\sl ${\cal J}_{\mathcal{P}^{a}}(\Xi_*)
\le{\cal J}_{\mathcal{P}^{a}}(\Xi)$ 
for all $\Xi=(\xi^0,\xi)\in\mathcal{P}^{a}$ 
such that $\partial\xi\in{\cal S}\,$,}
\end{itemize}\end{itemize}
i.e., $\Xi_*$ minimizes the cost ${\cal J}_{\mathcal{P}^{a}}(\Xi)$ 
among all augmented paths $\Xi=(\xi^0,\xi)$ of $\mathcal{P}^{a}$ 
that satisfy the boundary condition $\partial\xi\in{\cal S}$.
\end{itemize}

\pparagraph{The optimal control problem} We will try to solve the following
\medbreak\begin{center}
\fbox{\begin{minipage}{12cm}
\begin{center}
{\large\bf OPTIMAL CONTROL PROBLEM (OCP)}
\end{center}

{\bf Given} a set $\Omega$, a set $\mathcal{P}^a$ of augmented
paths in $\Omega$, an endpoint constraint set
${\cal S}\subseteq\Omega\times\Omega$, and a 
``reference augmented path'' $\Xi_*\in\mathcal{P}^a$,

{\bf find}: a necessary condition for $\Xi_*$ 
to be a $(\mathcal{P}^a,{\cal S})$-minimizer.
\end{minipage}}\end{center}

\pparagraph{The separation problem} 
If $A$, $B$ are subsets of a set $\Omega$, \
we say  that $A$ is {\bf\em separated} from $B$ at a 
point $m\in M$ if $A\cap B=\{m\}$.

\label{pagesepa}
\medbreak\begin{center}
\fbox{\begin{minipage}{12cm}
\begin{center}
{\large\bf SEPARATION PROBLEM (SP)}
\end{center}

{\bf Given}: a set $\Omega$, a set $\mathcal{P}$ of 
paths in $\Omega$, an endpoint constraint set
${\cal S}\subseteq\Omega\times\Omega$, and a 
``reference path'' $\xi_*\in\mathcal{P}$,

{\bf find}: a necessary condition for the boundary value 
set $\partial\,\mathcal{P}$ to be separated from 
${\cal S}$ at $\partial\xi_*$.
\end{minipage}}\end{center}

\pparagraph{Reduction of the optimal control problem to the separation
problem} Suppose we have solved the SP for a certain class
$\mathcal{C}$ of 4-tuples $(\Omega,\mathcal{P},{\cal S},\xi_*)$
of data. We will show how to derive as a corollary of this a
solution of the OCP for a \mbox{corresponding} class
$\mathcal{C}^a$ of 4-tuples $(\Omega,\mathcal{P}^a,{\cal S},\Xi_*)$.

Fix data $\,\,\Omega,\mathcal{P}^a,{\cal S}\,$, 
\ and suppose that the augmented path $\Xi_*=(\xi_*^0,\xi_*)$
is a  
\mbox{$(\mathcal{P}^a,{\cal S})$-minimizer}. Let $\Omega^a=\rr\times\Omega$.
Then $\mathcal{P}^a$ is a set of paths in $\Omega^a$.
Write  
\begin{eqnarray*}
a_*&=&a_{\Xi_*}\,\,,\qquad\phantom{\xi_*(a_{*})}\,b_*=b_{\Xi_*}\,\,,\\
x_*&=&\xi_*(a_{*})\,\,,
\qquad\phantom{a_{\Xi_*}}\,\tilde x_*=\xi_*(b_{*})\,\,,\\
x_*^0&=&\xi_*^0(a_{*})\,\,,\qquad\phantom{a_{\Xi_*}}
\tilde x_*^0=\xi_*^0(b_{*})\,\,,
\end{eqnarray*}
and define
$$
c_*=\tilde x_*^0-x_*^0=\mathcal{J}_{\mathcal{P}^a}(\Xi_*)\,.
$$
Let ${\cal S}^a$ to be the set of all ordered pairs
$\Big((x^0,x),(\tilde x^0,\tilde x)\Big)\in\Omega^a\times\Omega^a$
such that
\begin{itemize}
\item[(a)] $(x,\tilde x)\in{\cal S}\,$,
\item[(b)] $\tilde x^0-x^0\le c_*-\|x-x_*\|^2-\|\tilde x-\tilde x_*\|^2\,$,
\item[(c)] $x^0=x_*^0\,$.
\end{itemize}
Then it is clear that $\partial\Xi_*\in{\cal S}^a$.
Furthermore, if $Q\in\partial\mathcal{P}^a\cap\mathcal{S}^a$,
then there must exist a $\Xi\in\mathcal{P}^a$ such that 
$Q=\partial\Xi\in\mathcal{S}^a$.  \ If we write
$a=a_\Xi$, $b=b_\Xi$, \mbox{$\Xi=(\xi^0,\xi)$}, $x=\xi(a)$, 
$\tilde x=\xi(b)$, $x^0=\xi^0(a)$, $\tilde x^0=\xi^0(b)$, 
$c=\tilde x^0-x^0=\mathcal{J}_{\mathcal{P}^a}(\Xi)$, then the fact
that $\partial\Xi\in\mathcal{S}^a$ implies that
$\partial\xi\in\mathcal{S}$ and
\begin{equation}\label{jess}
c\le c_*-\|x-x_*\|^2-\|\tilde x-\tilde x_*\|^2\,.
\end{equation}
Since $\Xi_*$ is a \mbox{$(\mathcal{P}^a,{\cal S})$-minimizer},
and $\partial\xi\in\mathcal{S}$, we have 
\mbox{$\mathcal{J}_{\mathcal{P}^a}(\Xi_*)\le\mathcal{J}_{\mathcal{P}^a}
(\Xi)$}, i.e., $c_*\le c$. Therefore 
$c_*\le c\le c_*-\|x-x_*\|^2-\|\tilde x-\tilde x_*\|^2$,
from which it follows that $x=x_*$, $\tilde x=\tilde x_*$,  and $c=c_*$. 
Since $x^0=x_*^0$, we can conclude that $\tilde x^0=\tilde x_*^0$, so 
that $\partial\Xi=\partial\Xi_*$.   Hence $Q=\partial\Xi_*$. So 
we have shown that
$\partial\,\mathcal{P}^a\cap\mathcal{S}^a=\{\partial\Xi_*\}$.
It follows that $\mathcal{P}^a$ and $\mathcal{S}^a$ are 
separated at $\partial\Xi_*$.  If the \mbox{4-tuple}
$\,(\Omega^a,\mathcal{P}^a,\mathcal{S}^a,\Xi_*)\,$ 
\mbox{belongs} to the
class $\mathcal{C}$ for which we have a solution of the
SP,  then we can apply the necessary
condition for separation given by this solution, and obtain a
necessary condition for minimization for the OCP.

Naturally, the class $\mathcal{C}^a$ of 4-tuples for which this
reduction method produces a \mbox{necessary} condition for $\Xi_*$ to
be a $(\mathcal{P}^a,{\cal S})$-minimizer consists exactly of the
\mbox{4-tuples} $(\Omega,\mathcal{P}^a,{\cal S},\Xi_*)$ for which the 4-tuple
$(\Omega^a,\mathcal{P}^a,{\cal S}^a,\Xi_*)$ constructed above belongs
to the class $\mathcal{C}^a$.  Any time a specific set of technical
conditions characterizing a class $\mathcal{C}$, the above reduction
will yield a corresponding class $\mathcal{C}^a$, and then the
necessary condition for separation for $\mathcal{C}$ can be translated
into a necessary condition for optimality for $\mathcal{C}^a$.  This
will be carry out in detail later, for each of our necessary
conditions for separation.

\ssection{Proof of a  classical maximum principle}\label{secproofclass}

As explained in the previous section, the optimal control problem can be 
reduced to the separation problem. We will therefore focus our efforts 
from now on the SP. The first step will be to introduce some notations
and review various basic mathematical concepts that will be used in the
proof. 

\subsection{Notations, definitions, and background 
facts}\label{secnotations}

\pparagraph{Numbers, intervals, Euclidean spaces, 
matrices}\label{subsecnotationsnumbers}
We use $\zz$, $\rr$ to denote, respectively, the set
of all integers and the set of all real \mbox{numbers}, 
and write 
\begin{eqnarray*}
\zz_+&=&\{n\in\zz:n\ge 0\}\,\,,\qquad\phantom{\rr_+\cup\{+\infty\}}
\nn\phantom{_+}=\{n\in\zz:n\ge 1\}\,\,,\\
\overline{\nn}\phantom{_+}&=&\nn\cup\{+\infty\}\,\,,\qquad
\phantom{\{x\in\rr_+:x\le 0\}}\rr_+=\{x\in\rr:x\ge 0\}\,\,,\\
\rr_-&=&\{x\in\rr:x\le 0\}\,\,,\qquad\phantom{\rr_+\cup\{+\infty\}}
\rrinf\phantom{_+}=\rr\cup\{-\infty,+\infty\}\,\,,\\
\rrinf_+&=&\rr_+\cup\{+\infty\}\,\,,\qquad
\phantom{\{x\in\rr:x\le 0\}}\rrinf_-=\rr_-\cup\{-\infty\}\,\,.
\end{eqnarray*}

An {\bf\em extended real interval\/} is a subset $I$ of
$\rrinf$ such that whenever $a\in I$, $b\in I$, and $a<x<b$, 
it follows that $x\in I$. We use the notations 
$[a,b]$, $]a,b[\,$, $[a,b[\,$, $]a,b]$ for
the closed, open, left-closed right-open and right-closed left-open
intervals with endpoints $a,b$.  (The \mbox{expression} $(a,b)$ will always
denote the ordered pair whose components are $a$ and $b$.)  A {\bf\em real
interval} is an extended real interval $I$ such that $I\subseteq\rr$.
An extended real \mbox{interval}
is {\bf\em nontrivial} if it contains more than one point.

If $m,n\in\nn$, we use $\rr^{m\times n}$ to denote the space of all real 
{\bf\em matrices}
$$
A=\Big\{a_{j}^i\Big\}_{1\le i\le m,1\le i\le n}=
\left(\begin{array}{cccc}a_1^1&a_2^1&\cdots&a_n^1\\
\vdots&\vdots&\ddots&\vdots\\
a_1^m&a_2^m&\cdots&a_n^m\end{array}\right)
$$
with $m$ rows and $n$ columns.  The {\bf\em transpose} of a matrix
$A\in\rr^{m\times n}$ is the matrix $A^\dagger\in\rr^{n\times m}$ 
such that 
$A^\dagger=\Big\{b_{\ell}^k\Big\}_{1\le k\le n,1\le\ell\le m}$,
where $b_{\ell}^k=a_{k}^\ell$. 

We use $\rr^n$ to denote the space of all real
$n$-dimensional  {\bf\em column} vectors 
$$
x=\Big\{x^i\Big\}_{1\le i\le n}=
\left(\begin{array}{c}x^1\\x^2\\\vdots\\x^n\end{array}\right)
=(x^1,x^2,\cdots,x^n)^{\dagger}\,,
$$
so $\rr^n=\rr^{n\times 1}$.
Also, we use $\rr_n$ to denote the space of all real
$n$-dimensional {\bf\em covectors}---i.e. row
vectors---$p=(p_1,p_2,\ldots,p_n)$, so 
$\rr_n=\rr^{1\times n}$.

The {\bf\em matrix product} $A\cdot B$ of two matrices
$A\in\rr^{m\times n}$, $B\in\rr^{\mu\times\nu}$ is defined
if and only if $n=\mu$, and in that case, if 
$A=\Big\{a_{j}^i\Big\}_{1\le im,1\le j\le n}$ and
$B=\Big\{b_{\ell}^k\Big\}_{1\le k\le n,1\le\ell\le\nu}$, we
let $A\cdot B=C$, where $C\in\rr^{m\times\nu}$ is given by
$$
C=\Big\{c_{\ell}^i\Big\}_{1\le i\le m,1\le\ell\le\nu}\,,\quad
c_\ell^i=\sum_{j=1}^na_j^ib_\ell^j\,.
$$
The matrix product operation is associative:
if $A\in\rr^{m\times n}$, $B\in\rr^{n\times p}$, 
$C\in\rr^{p\times q}$, then $(A\cdot B)\cdot C=A\cdot(B\cdot C)$. 

If $A\in\rr^{m\times n}$, then $A$ induces a linear map
$\Lambda_A:\rr^n\mapsto\rr^m$, given by $\Lambda_A(x)=A\cdot x$.
It is then easy to see that if $A,B\in\rr^{m\times n}$, then $A=B$ 
if and only if $\Lambda_A=\Lambda_B$, and that 
if $A\in\rr^{m\times n}$ and $B\in\rr^{n\times p}$, then 
$\Lambda_{A\cdot B}=\Lambda_{A}\circ\Lambda_{B}$.

The {\bf\em inner product} of two 
vectors 
$$
x=(x^1,x^2,\cdots,x^n)^{\dagger}\in\rr^n\,, 
\qquad y=(y^1,y^2,\cdots,y^n)^{\dagger}\in\rr^n\,,
$$
is the number $\langle x,y\rangle=x^{\dagger}\cdot y$. 
(In other words, $\langle x,y\rangle=\sum_{j=1}^nx^jy^j$.)
The {\bf\em Euclidean \mbox{norm}} of a 
\mbox{vector} $x=(x^1,x^2,\cdots,x^n)^{\dagger}$ 
is the number $\|x\|=\sqrt{\langle x,x\rangle}$.
(That is, $\|x\|=\sqrt{\sum_{j=1}^n(x^j)^2}$.)
The inner \mbox{product} of two covectors $p=(p_1,p_2,\cdots,p_n)\in\rr_n$,
\mbox{$q=(q_1,q_2,\cdots,q_n)\in\rr_n$}  is defined in a \mbox{similar} way, 
to be the number
\mbox{$\langle p,q\rangle=p\cdot q^{\dagger}$}. 
The Euclidean norm of a covector $p=(p_1,p_2,\ldots,p_n)$ is the number 
\mbox{$\|p\|=\sqrt{\langle p,p\rangle}$}. 

\pparagraph{Convex sets, cones}\label{secconvex}
If $X$ is a real linear space, a subset $S$ of $X$ is {\bf\em convex}
if $rx+(1-r)x'\in S$ whenever $x\in S$, $x'\in S$, $r\in\rr$, and
$0\le r\le 1$. 

A {\bf\em cone} in a real linear space $X$ is a 
nonempty subset $C$ of $X$ such that $r\cdot v\in C$ whenever 
$v\in C$, $r\in\rr$, and $r\ge 0$.  It is then clear that 
{\em if $C$ is a cone in $X$ then automatically $0\in C$}. (Indeed,
we may pick a $v\in C$, because $C$ is nonempty, and then 
$0\cdot v\in C$ as well, so $0\in C$.)

It follows easily from the above definitions that 
\bfact\label{fa-charconvcone}
A subset $S$ of a real linear space $X$ is a convex cone if and only if 
$S$ is nonempty and closed under addition (that is, such that
$x+x'$ belongs to $S$ whenever $x,x'\in S$)  as well
as under multiplication by nonnegative reals 
(that is, such that $rx\in S$
whenever $x\in S$ and $r\in\rr_+$)$\,$. \proofend
\efact

The following trivial observation, which will play a crucial role later, 
is an immediate consequence of Fact \ref{fa-charconvcone}.
\bfact\label{fa-convcone-subspace}
Let $C$ be a convex cone in a real linear space $X$. Then 
$C$ is a linear subspace of $X$ if and only if 
$c\in C\Rightarrow -c\in C$ whenever $c\in C$.
\efact
\begin{proof}
The ``only if'' part is evidently true. To prove the ``if'' statement, 
assume that $C$ is a convex cone such that
$c\in C\Rightarrow -c\in C$ whenever $c\in C$. To prove that $C$ is a 
linear subspace we have to show that $C$ is closed under the \mbox{operations}
of addition and multiplication by real scalars.
Fact \ref{fa-charconvcone} tells us that $C$ is closed under the
addition and multiplication by nonnegative real scalars. The \mbox{assumption}
that \mbox{$c\in C\Rightarrow -c\in C\,$} whenever $c\in C$ then implies 
that $C$ is closed under \mbox{multiplication} by $-1$. From this it follows
easily that $C$ is closed under \mbox{multiplication} by negative real
scalars, completing the proof.\proofend
\end{proof}



If $\mathcal{C}$ is any set of subsets of a real linear space
$X$, then the intersection 
$$
\bigcap\mathcal{C}\equaldef
\Big\{\,x\in X:x\in C\,\,{\rm whenever}\,\,C\in\mathcal{C}\,\Big\}
$$
is a subset of $X$ that is contained in all the members of $\mathcal{C}$, 
and inherits from these sets some important properties. (Notice that 
$\bigcap\mathcal{C}=X$ if $\mathcal{C}$  is empty.) 
For example, {\em if all the members of $\mathcal{C}$ are convex, then 
$\bigcap\mathcal{C}$ is convex}. Also, 
{\em if all the members of $\mathcal{C}$ are cones, then 
$\bigcap\mathcal{C}$ is a cone}. It follows that 
{\em if all the members of $\mathcal{C}$ are convex cones, then 
$\bigcap\mathcal{C}$ is a convex cone}. 

In particular, given a subset $S$ of $X$, we can let 
$\mathcal{C}^{conv}(S)$, $\mathcal{C}^{cone}(S)$, $\mathcal{C}^{convcone}(S)$, 
be, respectively, the set of all cones in $X$ that contain $S$,
the set of all convex \mbox{subsets} of $X$ that contain $S$, and
the set of all convex cones in $X$ that contain $S$. Then
$\bigcap\mathcal{C}^{conv}(S)$, $\bigcap\mathcal{C}^{cone}(S)$, $\bigcap
\mathcal{C}^{convcone}(S)$, are, respectively, {\em the smallest 
convex \mbox{subset} of $X$ containing $S$}, {\em the smallest 
cone in $X$ containing $S$}, and {\em the smallest convex 
cone in $X$ containing $S$}. 


\pparagraph{Metric spaces, neighborhoods, open and 
closed sets}\label{secmetric}
If $X$ is a set, a  {\bf\em \mbox{pseudodistance}} (or 
{\bf\em pseudometric}) on $X$ is a 
function $d:X\times X\mapsto\rr$ such that 
\begin{itemize}
\item[(1)] {\em $d(x,x')=d(x',x)$ whenever $x,x'\in X$,}
\item[(2)] {\em $d(x,x'')\le d(x,x')+d(x',x'')$ whenever $x,x',x''\in X$,}
\item[(3)] {\em $d(x,x)=0$ whenever $x\in X$.}
\end{itemize}
A pseudodistance $d$ on $X$ is a {\bf\em distance} (or 
{\bf\em distance function}, or {\bf\em metric}) if it satisfies
\begin{itemize}
\item[(4)] {\em if $x,x'\in X$ and 
$d(x,x')=0$ then $x=x'$.}
\end{itemize}
A {\bf\em pseudometric space} is a set $X$ endowed with a pseudometric 
on $X$. A {\bf\em metric space} is a pseudometric space 
$(X,d)$ such that $d$ is a metric. 

If $X$ is a pseudometric space, we use \idx{$d_X$} $d_X$ to denote the
pseudodistance function on $X$. If $\bar x\in X$, we write $\bb_X(\bar x,r)$,
$\overline{\bb}_X(\bar x,r)$, to denote, respectively, the
{\bf\em open and closed balls with center $\bar x$ and radius $r$}, that is,
the sets
$$ 
\bb_X(\bar x,r)\,\,\equaldef\,\,\{x\in X:d_X(x,\bar x)<r\}\,,\qquad
\overline{\bb}_X(\bar x,r)\,\,\equaldef\,\,\{x\in X:d_X(x,\bar x)\le r\}\,.
$$

If $X$ is a pseudometric space, $\bar x\in X$, and $U$ is a subset $U$ of
$X$ we say that $U$ is a {\bf\em neighborhood} of $\bar x$---and that 
$\bar x$ is an {\bf\em interior point} of $U$---if
$\bb_X(\bar x,r)\subseteq U$ for some $r\in\rr$ such that $r>0$.
The {\bf\em interior of $U$ in $X$} is the set ${\rm Int}_X(U)$ of all
interior points of $U$. A subset $U$ of $X$ is {\bf\em open}
(or {\bf\em open in $X$}) if ${\rm Int}_X(U)=U$, i.e., if $U$ is a 
neighbourhood of every point of $U$.  If $S\subseteq X$, then 
${\rm Int}_X({\rm Int}_X(S))={\rm Int}_X(S)$, so ${\rm Int}_X(S)$ is 
open in $X$.  A subset $C$ of $X$ is 
{\bf\em closed in $X$} if $X\backslash C$ is open in $X$. The 
{\bf\em closure in $X$} of a subset $S$ of $X$ is the smallest closed 
subset of $X$ that contains $S$, i.e., the intersection of all closed
subsets of $X$ that contain $S$.  We use ${\rm Clos}_X(S)$ to denote 
the closure of $S$ in $X$. Clearly, then, 
${\rm Clos}_X(S)=X\backslash\Big({\rm Int}(X\backslash S)\Big)$.

If $S$ is a subset of a pseudometric space $X$, then $S$ is itself a 
pseudometric space with the distance function $d_S:S\times S$ given by
$d_S(x,x')=d_X(x,x')$ whenever $x,x'\in S$.  Clearly, if f
$d_X$ is a metric then $d_S$ is a metric as well.
 
If $X$ and $Y$ are pseudometric spaces, then the Cartesian product $X\times Y$
is a \mbox{pseudometric} space, equipped with the {\bf\em product pesudometric}
$d_{X\times Y}$, given by
$$
d_{X\times Y}\Big((x,y),(x',y')\Big)=d_X(x,x')+d_Y(y,y')\,\,\,\,\,{\rm if} 
\,\,\,\,\,\,x,x'\in X\,,\,\, y,y'\in Y\,.
$$ 

\pparagraph{Limits}\label{seclimit}
A {\bf\em sequence} is a mapping ${\bf x}$ defined on $\nn$.
When dealing with sequences, we will often use the ``family of objects''
notation, rather than the more common ``mapping'' notation, 
and write expressions such as ${\bf x}=\{x_j\}_{j\in\nn}$, 
or ${\bf x}=\{x_j\}_{j=1}^\infty$,   in
which case it will be understood that ``$x_j$'' denotes the value at $j$ of
the map ${\bf x}$, i.e., the object usually referred to as 
${\bf x}(j)$. 
If $S$ is a set, then a sequence ${\bf x}=\{x_j\}_{j\in\nn}$ is a 
{\bf\em sequence in $S$}, or a {\bf\em sequence of members of $S$}, 
if $x_j\in S$ for all $j\in\nn$. 

If ${\bf x}=\{x_j\}_{j\in\nn}$ is a sequence, and
$\sigma$ is a strictly increasing map from $\nn$ to $\nn$, then 
the composite map ${\bf x}\circ\sigma$ is a sequence. Any sequence
of the form ${\bf x}\circ\sigma$, with $\sigma:\nn\mapsto\nn$
strictly increasing, will be called a  {\bf\em subsequence} of ${\bf x}$.
In that case, we will also write $\{x_{\sigma(j)}\}_{j\in\nn}$, or
even $\{x_{\sigma_j}\}_{j\in\nn}$ since, after all, the map
$\sigma$ is itself a sequence, so we can write it in ``family of objects''
notation.  If $J$ is an infinite subset of $\nn$, then 
there exists a unique strictly increasing map $\sigma:\nn\mapsto\nn$ 
such that $\sigma(\nn)=J$. We use $\sigma^J$ to denote this map, so 
the sequence $\sigma^J=\{\sigma_j^J\}_{j\in\nn}$ is ``the sequence
of members of $J$, arranged in strictly increasing order.''  Clearly,
if $\sigma:\nn\mapsto\nn$ is strictly increasing, then $\sigma=\sigma^J$,
if we let $J=\sigma(\nn)$. It follows that a subsequence 
$\{x_{\sigma_j}\}_{j\in\nn}$ of ${\bf x}=\{x_j\}_{j\in\nn}$ is completely
determined by specifying the set $J=\sigma(\nn)$, and any infinite subset
$J$ of $\nn$ determines a subsequence. 

If $X$ is a pseudometric space, $\{x_j\}_{j\in\nn}$ is a sequence  in $X$,
and $\bar x\in X$, we say that {\bf\em the sequence  
$\{x_j\}_{j\in\nn}$ converges (or ``goes'') to $\bar x$},
or that {\bf\em the $x_j$ converge (or ``go'') to $\bar x$ as
$j$ goes to infinity} (and write 
``$x_j\to\bar x$ as $j\to\infty$'' or ``$x_j\to\bar x$'')  if for every 
positive $\eps\in\rr$ there exists an $N\in\nn$ such that 
$d_X(x_j,\bar x)<\eps$ whenever $j\in\nn$ and $j\ge N$. 
A sequence $\{x_j\}_{j\in\nn}$ is {\bf\em convergent} if
there exists $\bar x$ such that $x_j\to\bar x$ as $j\to\infty$.

It is clear that if $X$ is a {\em metric} space and 
$\{x_j\}_{j\in\nn}$ is a convergent sequence  in $X$,
then the limit $\bar x$ of $\{x_j\}_{j\in\nn}$ is unique. (The converse
implication is also true: if $X$ is a pseudometric space such that 
the limit of every convergent sequence$\{x_j\}_{j\in\nn}$ is
unique, then $X$ is metric.)  In that case,
we also write ``$\lim_{j\to\infty}x_j=\bar x$,'' or 
``$\lim_{j\to\infty}x_j=\bar x$.''

The following two facts are easily verified.
\bfact
If $\{x_j\}_{j\in\nn}$ is a sequence  in a pseudometric space $X$, and
$\bar x\in X$, then $\{x_j\}_{j\in\nn}$ converges to $\bar x$ if and only if
$\lim_{j\to\infty}d_X(x_j,\bar x)=0$.\proofend
\efact
\bfact
If $\{x_j\}_{j\in\nn}$ is a convergent sequence  in a pseudometric space $X$, 
then $\{x_j\}_{j\in\nn}$ has the {\bf\em Cauchy property}: 
\begin{itemize}
\item[(CP)] 
for every positive real number $\eps$ there exists an $N\in\nn$ such that 
\mbox{$d_X(x_j,x_k)<\eps$} whenever $j,k\in\nn$, $j\ge N$, and $k\ge N$. \proofend
\end{itemize}
\efact
A sequence that has the Cauchy property is called a {\bf\em Cauchy
sequence}. A pseudometric space $X$ is {\bf\em complete} if every Cauchy
sequence in $X$ is convergent. 

It is easy to verify that 
\bfact
The closure in $X$ of a subset $S$ of a psedudometric space $X$ is the set of 
all points $\bar x\in X$ such that there is a sequence in $S$ that 
converges to $\bar x$. In particular, a subset $S$ of a pseudometric space 
$X$ is closed if and only if whenever a sequence in $S$ converges 
to an $\bar x\in X$ it follows that $\bar x\in S$.\proofend
\efact

\bfact
A closed subset of a complete pseudometric space is a complete pseudometric 
space.\proofend
\efact

If $X$, $Y$ are pseudometric spaces, $S$ is a subset of $X$, 
$f$ is a mapping from $S$ to $Y$, $\bar x\in X$, and $\bar y\in Y$, 
we say that $\bar y$ is a {\bf\em limit of $f(x)$ as 
$x$ approaches $\bar x$} (or ``as $x$ approaches $\bar x$ along $S$,''
or ``as $x$ approaches $\bar x$ via $S$,'' 
or ``as $x$ approaches $\bar x$ via values in $S\,$''), 
and write ``$f(x)\to\bar y$ as $x\to_S\bar x\,$'') 
if for every positive $\eps\in\rr$
there exists a positive $\delta\in\rr$ such that 
$d_Y(f(x),\bar y)<\eps$ whenever $x\in S$ and 
$d_X(x,\bar x)<\delta$.  (In all the above notations, expressions
such as ``along $S$'' or ``via values in $S$'' are really unnecessary,
because $S$ is determined by $f$, since $S=\dom(f)$.  We will, however,
often use such expressions to emphasize the fact that $f$ is only 
intended to be defined on  $S$.)  
\begin{remark}
{\rm Let $\beta:\bar\nn\mapsto\rr$ be  the map given by 
$$
\beta(n)=2^{-n}\,\,\quad{\rm for}\,\quad\,n\in\nn\,,\qquad\quad
\,\,\,\beta(+\infty)=0\,.
$$
Define \mbox{$d_{\bar\nn}(\mu,\nu)=|\beta(\mu)-\beta(\nu)|$} for 
$\mu,\nu\in\bar\nn$. Then 
$d_{\bar\nn}$ is a metric on $\bar\nn$. It is then easy to verify 
that a sequence ${\bf x}=\{x_j\}_{j\in\nn}$ of members of a pseudometric
space $X$ converges to a point $\bar x\in X$ if and only if 
${\bf x}(j)\to\bar x$ as $j\to_{\nn}+\infty$. So the notion of limit of a sequence
is a special case of the more general concept of limit of a function
along a set.}\proofend
\end{remark}

If $Y$ is metric, and $\bar x\in{\rm Clos}_X(S)$, then the limit
$\bar y$ of $f(x)$ as $x\to_S\bar x$ is unique. (Indeed, if
$f(x)\to\bar y_1$ as $x\to_S\bar x$  and $f(x)\to\bar y_2$ as $x\to_S\bar x$,
then, given $\eps>0$, we can find a positive $\delta\in\rr$ such that 
$d_Y(f(x),\bar y_1)<\eps$ and $d_Y(f(x),\bar y_2)<\eps$ 
whenever $x\in S$ and $d_X(x,\bar x)<\delta$. Since $\bar x\in{\rm Clos}_X(S)$,
we can pick $x\in X$ such that \mbox{$d_X(x,\bar x)<\delta$}. Then 
$d_Y(\bar y_1,\bar y_2)<2\eps$. Since $\eps$ is arbitrary,
$d_Y(\bar y_1,\bar y_2)=0$. Since $Y$ is metric, we can conclude 
that $\bar y_1=\bar y_2$.)
In that  case we can talk about {\em the} limit of $f(x)$ as 
$x$ approaches $\bar x$, and use
$\lim_{x\to\bar x}f(x)$ (or $\lim_{x\to\bar x,x\in S}f(x)$) 
to denote this limit.

Notice that, with our definitions, if $\bar x\in S$ and $Y$ is metric, 
then $\lim_{x\to\bar x}f(x)=\bar y$ implies that $f(\bar x)=\bar y$, i.e., 
that $f$ is continuous at $\bar x$ along $S$
(cf. page \pageref{secconti}).   If we wish to have $f(x)$ 
``go to $\bar x\,$'' without allowing $x$ to take the value $\bar x$, we will 
take the limit $\lim_{x\to\bar x,x\in S\backslash\{\bar x\}}f(x)$, 
for which we will also use the notation 
$\lim_{x\to\bar x,x\in S,x\neq \bar x}f(x)$. 

It is easy to verify that
\bfact
If $X$, $Y$ are pseudometric spaces, $\bar x\in X$, $\bar y\in Y$,
$S\subseteq X$, and $f:S\mapsto Y$, then 
$f(x)\to\bar y$ as $x\to\bar x$ if and only if 
whenever $\{x_j\}_{j\in\nn}$ is a sequence in $S$ that converges 
to $\bar x$ it follows that the sequence 
$\{f(x_j)\}_{j\in\nn}$ converges to $\bar y$. \proofend
\efact

\pparagraph{The contraction mapping theorem}\label{seccontra}

Let $X$ be a metric space. A mapping $f:X\mapsto X$ is said to be a 
{\bf\em contraction} if there exists a real number $c$ such that 
$0\le c<1$ having the property that 
$d_X(f(x),f(x'))\le cd_X(x,x')$ whenever $x,x'\in X$. 

A {\bf\em fixed point} of a mapping $f:X\mapsto X$ is a point
$x\in X$ such that $f(x)=x$. 

The following well-known theorem will play a crucial role, so we 
include its statement and proof. 

\begin{theorem}\label{the-contraction}
Assume that $X$ is a complete metric space and $f:X\mapsto X$ is a
contraction. Then $f$ has a unique fixed point.
\end{theorem}
\begin{proof}
Uniqueness is trivial, because if $x$ and $x'$ are fixed points then
$$
d_X(x,x')=d_X(f(x),f(x'))\le c d_X(x,x')\,,
$$
which is only possible if $d_X(x,x')$, i.e., if $x=x'$. 

To prove existence, we pick a point $x_1\in X$, and define 
\mbox{$x_2=f(x_1)$}, \mbox{$x_3=f(x_2)$}, and, recursively, 
$x_k=f(x_{k-1})$ if $k\in\nn$, $k>1$. Then 
$$
d_X(x_{k+1},x_{k})=d_X(f(x_{k}),f(x_{k-1}))\le 
cd_X(x_{k},x_{k-1})\qquad{\rm if}\quad k>1\,,
$$
from which it follows by a trivial induction that 
$$
d_X(x_{k+1},x_{k})\le c^{k-1}d_X(x_2,x_1)\qquad{\rm whenever}\quad
k\ge 1\,.
$$ 
Then the triangle inequality implies that 
\begin{eqnarray*}
d_X(x_{k},x_{k+j})&\le& d_X(x_{k},x_{k+1})+
d_X(x_{k+1},x_{k+2})+\cdots+ d_X(x_{k+j-1},x_{k+j})\\
&\le&\Big(\,c^{k-1}+c^k+\cdots+c^{k+j-1}\,\Big) d_X(x_2,x_1)\\
&=&c^{k-1}\Big(\,1+c+\cdots+c^{j}\,\Big) d_X(x_2,x_1)\\
&\le&c^{k-1}\,\Big(\sum_{\ell=0}^\infty c^{\ell}\,\Big)d_X(x_2,x_1)\\
&\le&\frac{c^{k-1}}{1-c}d_X(x_2,x_1)\,,
\end{eqnarray*}
whenever $k,j\in\nn$.

Given a positive $\eps$, we can pick
$k\in\nn$ such that $\frac{c^{k-1}}{1-c}d_X(x_2,x_1)\le\eps$, 
and then $d_X(x_\ell,x_s)\le\eps$ whenever $\ell\ge k$ and $s\ge k$. 
So the sequence $\{x_k\}_{k\in\nn}$ satisfies the Cauchy condition.
Hence $\{x_k\}_{k\in\nn}$ converges to a point 
$\bar x\in X$. If $k\in\nn$, then $d_X(f(\bar x),x_{k+1})\le 
cd_X(\bar x,x_{k})$. So 
$$
d_X(f(\bar x),\bar x)\le d_X(f(\bar x),x_{k+1})+
d_X(x_{k+1},\bar x)\le cd_X(\bar x,x_{k})+d_X(x_{k+1},\bar x)\,.
$$
Since $cd_X(\bar x,x_{k})+d_X(x_{k+1},\bar x)\to 0$ as 
$k\to\infty$ (because $\{x_k\}_{k\in\nn}$ converges to $\bar x$),
we have $d_X(f(\bar x),\bar x)=0$. Hence
$f(\bar x)=\bar x$, so . Hence $\bar x$ is a fixed point of $f$.\proofend
\end{proof}

\pparagraph{Continuous maps}\label{secconti}

Suppose that $X$, $Y$ are pseudometric spaces and
$f:X\mapsto Y$ is a mapping.  If 
$\bar x\in X$, we say that $f$ is {\bf\em continuous at $\bar x$}
if, whenever $\{x_j\}_{j\in\nn}$ is a sequence in $X$ that converges 
to $\bar x$ it follows that the sequence 
$\{f(x_j)\}_{j\in\nn}$ converges to $f(\bar x)$. 
We say that $f$ is {\bf\em continuous} if 
$f$ is continuous at $\bar x$ for every $\bar x\in X$. 
If $Y$ is metric, then it is clear that 
$f$ is continuous at $\bar x$ if and only if 
$\lim_{x\to\bar x}f(x)=f(\bar x)$. 

It is well known, and easy to prove, that 
\bfact
If $X$ and $Y$ are pseudometric spaces and $f:X\mapsto Y$ is a mapping, then
the following are equivalent:
\begin{itemize}
\item[1.] $f$ is continuous.
\item[2.] For every open subset $U$ of $Y$, the
set $f^{-1}(U)=\{x\in X:f(x)\in U\}$ is open in $X$.
\item[3.] For every closed subset $C$ of $Y$, the
set $f^{-1}(C)=\{x\in X:f(x)\in C\}$ is closed in $X$.\proofend
\end{itemize}
\efact


\pparagraph{Compact sets}\label{seccompact}
If $X$ is a metric space, a subset $K$ of $X$ is said to be 
{\bf\em compact} if every sequence in $K$ has a subsequence that converges
to a member of $K$. The following facts are well known. 
\bfact
If $X$ is a metric space, then a subset 
$K$ of $S$ is compact if and only if whenever $\mathcal{U}$ is
a set of open subsets of $X$ such that $K\subseteq\bigcup\mathcal{U}$ it
follows that there exists a finite subset $\mathcal{U}_0$ of
$\mathcal{U}$ such that $K\subseteq\bigcup\mathcal{U}_0$.\proofend
\efact
\bfact
If $X$ is a finite-dimensional real normed space, then a subset 
$K$ of $S$ is compact if and only if it is closed and bounded.
In particular, a nonempty real interval $I$ is compact if and only 
if $I=[a,b]$ for some $a,b\in\rr$ such that $a\le b$.\proofend
\efact
\bfact
If $X$, $Y$ are metric spaces, $X$ is compact, and $f:X\mapsto Y$ is
continuous, then $f(X)$ is a compact subset of $Y$. \proofend
\efact
\bfact
If $X$ is a compact metric space, and $f:X\mapsto\rr$ is
continuous, then $f$ has a maximum and a minimum on $X$. (That is, there
exist $x_-,x_+\in X$ such that $f(x_-)\le f(x)\le f(x_+)$ for all
$x\in X$.)\proofend
\efact

\pparagraph{Normed spaces}\label{secnormed}
Let $X$ be a real linear space. \ A {\bf\em norm} on $X$ is a 
function \mbox{$X\ni x\mapsto\|x\|\in\rr$} such that 
\begin{itemize}
\item[1.] $\|x\|\ge 0$ for all $x$, 
\item[2.] $\|x\|=0$ if and only if $x=0$, 
\item[3.] \mbox{$\|rx\|=|r|\,\|x\|$} whenever $x\in X$ and $r\in\rr$,
\item[4.] $\|x+x'\|\le\|x\|+\|x'\|$ whenever $x$ and $x'$ belong 
to $X$. 
\end{itemize}
It is clear that, if $X$ is a real linear space 
endowed with a norm $\|\cdot\|:X\mapsto\rr$, 
then the function 
\mbox{$X\times X\ni(x,x')\mapsto d_X(x,x')\equaldef\|x-x'\|\in\rr$} 
is a metric on $X$. 

A {\bf\em real normed space} is a real linear space endowed with a norm.
Clearly, if $X$ is a real normed space then $X$ is a metric  space
if endowed with the metric $d_X$ defined above. 

A {\bf\em real Banach space} is a real normed space such that the metric 
space $(X,d_X)$ is complete.

If $X$, $Y$ are real linear spaces, $L:X\mapsto Y$ is a linear 
map, and $x\in X$, we will use $L\cdot x$ and $Lx$ 
as alternative notations for $L(x)$. 

If $X$, $Y$ are real normed spaces, a linear map $L:X\mapsto Y$
is {\bf\em bounded} if there exists a real number $C$  such that
\begin{equation}\label{issatat}
\|L\cdot x\|\le C\|x\|\quad{\rm  for\,\,all}\quad x\in X\,.
\end{equation}
The infimum of the set of all $C$s for which (\ref{issatat}) 
holds is the {\bf\em norm} (or {\bf\em operator norm}) of $L$.
We use $\|L\|$, or $\|L\|_{op}$, to denote this norm. It is then 
easy to see that
\begin{equation}
\|L\|_{op}\,=\,\sup\Big\{\,\|L\cdot x\|:x\in X\,,\,\,\|x\|\le 1\,\Big\}\,.
\end{equation}
Furthermore, the following are well known and easy to prove.
\bfact\label{fa-iscon}
If $X$ and $Y$ are real normed spaces  then 
\begin{itemize}
\item[1.] A linear map $L:X\mapsto Y$ is bounded if and only if it is 
continuous.
\item[2.] The set ${\rm BLin}(X,Y)$ of all bounded 
linear maps from $X$ to $Y$, endowed with the operator norm,
is a normed space.
\item[3.] ${\rm BLin}(X,Y)$ is a Banach space if $Y$ is a Banach 
space.\proofend
\end{itemize}
\efact
\bfact\label{fa-banach}
Every finite-dimensional real normed space is complete, i.e., 
a Banach space.\proofend
\efact
\bfact\label{fa-bounded}
If $X$ and $Y$ are real normed spaces and $X$ is finite-dimensional 
then  every linear map $L:X\mapsto Y$ is bounded.\proofend
\efact

The spaces $\rr^n$, $\rr_n$ are real normed spaces (and hence Banach
spaces, by Fact \ref{fa-banach}) with the Euclidean norm defined above.  We
write $\bb^n(x,r)$, $\overline{\bb}^n(x,r)$, rather than
$\bmath_{\rr^n}(x,r)$, $\overline{\bmath}_{\rr^n}(x,r)$, to denote the open
and closed balls in these spaces.

The matrix space $\rr^{m\times n}$ is identified with the space 
${\rm Lin}(\rr^n,\rr^m)$ of all linear maps from $\rr^n$ to $\rr^m$ 
via the map
$\rr^{m\times n}\ni A\mapsto\Lambda_A\in{\rm Lin}(\rr^n,\rr^m)$ defined
above. Furthermore, ${\rm Lin}(\rr^n,\rr^m)$ coincides with 
${\rm BLin}(\rr^n,\rr^m)$ by Fact \ref{fa-bounded}. Hence we can
endowed $\rr^{m\times n}$ with the operator norm, by defining
$\|A\|=\|\Lambda_A\|_{op}$ (that is, 
$\|A\|=\max\{\|A\cdot x\|:x\in\rr^n,\|x\|\le 1\}$. Clearly, then,
$\rr^{m\times n}$, endowed  with the operator norm, 
is a Banach space. 

\pparagraph{The Brouwer fixed point theorem}\label{secbrouwer}
The following well-known and very important result, known as the {\bf\em
Brouwer fixed point theorem}, will play a crucial role in our proof of the
maximum principle.

\gnu{We must put a reference for the Brouwer fixed point theorem}

\begin{theorem}
Let $n\in\nn$, and assume that $\bar x\in\rr^n$, $r\in\rr$,
and $r>0$. Let $f$ be a 
continuous map from $\bar\bb^n(\bar x,r)$ to $\bar\bb^n(\bar x,r)$.
Then $f$ has a fixed point, i.e., there exists 
$x\in\bar\bb^n(\bar x,r)$ such that $f(x)=x$.\proofend
\end{theorem}


\pparagraph{Linear functionals, hyperplanes, duals}\label{secdual}
If $X$ is a real linear space, then
\begin{enumerate}
\item[$\bullet$] A linear map from $X$ to $\rr$ is 
a {\bf\em linear functional} on $X$. 
\item[$\bullet$] A {\bf\em hyperplane} in $X$ is a subset $H$ of $X$ 
such that  $H=\{x\in X:\lambda(x)=\alpha\}$ for some real number 
$\alpha$ and some linear map $\lambda:X\mapsto\rr$ (called a
{\bf\em defining linear functional} for $H$) which does not vanish 
identically.
\end{enumerate}
It is clear that if $H$ is a hyperplane in $X$ then any two defining 
linear functionals $\lambda$, $\lambda'$, for $H$ are scalar 
multiples of each other. (That is, there exists $r\in\rr\backslash\{0\}$ 
such that $\lambda'=r\lambda$.)

The set $X^{\dagger,alg}$ of all linear functionals on $X$ is the
{\bf\em algebraic dual} of $X$. Clearly, $X^{\dagger,alg}$ is a real
linear space with the obvious definitions of addition and
multiplication by scalars.

If $X$ is normed, then we know from Fact \ref{fa-iscon} that a linear
functional $\lambda:X\mapsto\rr$ is continuous if and only if it is
bounded.  We use $X^\dagger$ to denote the space of all bounded linear
functionals on $X$, and refer to $X^\dagger$ as the {\bf\em dual} of
$X$. Since $\rr$ is a Banach space, it is clear that $X^\dagger$ is a
Banach space.  If $X$ is finite-dimensional
then $X^{\dagger,alg}=X^{\dagger}$.

It is well known that 
\bfact
If $X$ is a real normed space, $\lambda$ belongs to 
$X^\dagger$, $\lambda\neq 0$,
$\alpha\in\rr$, and \mbox{$H=\{x\in X:\lambda(x)=\alpha\}$}, 
then $H$ is closed if and only if $\lambda$ is bounded.\proofend
\efact

\pparagraph{Spaces of continuous functions}\label{secspaccont}

If $X$, $Y$ are metric spaces, we use  $C^0(X,Y)$ to denote 
the space of all continuous maps from $X$ to $Y$. Given a compact subset
$K$ of $X$, we define the {\bf\em $K$-pseudodistance} $\delta_{X,Y,K}(f,g)$ 
between two members $f,g$ of $C^0(X,Y)$ to be the number
$$
\delta_K(f,g)=\max\{d_X(f(x,),g(x)):x\in K\}\,,
$$ 
which exists because the function
$K\ni x\mapsto d_X(f(x,),g(x))\in\rr$ is \mbox{continuous} and 
the set $K$ is compact. It is then clear that $\delta_{X,Y,K}$ is a
pseudodistance. Furthermore, if $X$ is itself compact then
$\delta_{X,Y,X}$ is a distance, so $C^0(X,Y)$ is a metric space.

It is easy to see that {\em if $X$ is compact and $Y$ is complete
then the metric space $C^0(X,Y)$, with the metric
$\delta_{X,Y,X}$, is complete.}

If $X$ is compact and $Y$ is a normed space then 
the function 
$$
C^0(X,Y)\ni f\mapsto\|f\|_{sup}\equaldef\max\{\|f(x)\|:x\in K\}
$$ 
is a norm, whose associated metric is precisely $\delta_{X,Y,X}$.
Hence $C^0(X,Y)$ is a normed space. Clearly, 
{\em if $Y$ is a Banach space and $X$ is a compact metric space
then  $C^0(X,Y)$ is a Banach space}.

\pparagraph{Vector fields} A {\bf\em vector field}
on an open subset $\Omega$ of $\rr^n$ is a
mapping $X:\Omega\mapsto\rr^n$. Naturally, we can regard a
vector field $X:\Omega\mapsto\rr^n$ as a column
$X=(X^1,\ldots,X^n)^\dagger$ of real-valued functions.
Such a vector field acts as a {\bf\em differential operator}
$\varphi\mapsto X\varphi$ 
on functions: if $\varphi:\Omega\mapsto\rr$ is a 
function, and $x$ is a point of $\Omega$ where 
$\varphi$ is differentiable, then $X\varphi(x)$ is the 
number 


\pparagraph{Hamiltonian vector fields}
\label{sec-hamilfields}
Assume that $n\in\nn$ and $\Om$ is an open subset of $\rr^n$. 
Let $h:\Omega\times\rr_n\mapsto\rr$ be a function 
of class $C^1$. The {\bf\em Hamilton vector field} corresponding 
to $h$ is the vector field $\vec{h}:
\Omega\times\rr_n\mapsto\rr^n\times\rr_n$ defined by 
$$
\vec{h}=\sum_{k=1}^n\Biggl(
\frac{\partial h}{\partial p_k}\frac{\partial\,\,}{\partial x^k}
-\frac{\partial h}{\partial x^k}\frac{\partial\,\,}{\partial p_k}\Biggr)\,,
$$
where $\frac{\partial\,\,}{\partial x^k}$ is the constant 
vector field on $\rr_n\mapsto\rr^n\times\rr_n$ whose 
$\rr^n$ component is $(\delta_k^1,\ldots,\delta_k^n)^\dagger$, 
and whose $\rr_n$ component is $0$, and 
$\frac{\partial\,\,}{\partial p_k}$ is the constant 
vector field on $\rr_n\mapsto\rr^n\times\rr_n$ whose 
whose $\rr^n$ component is $0$ and whose 
$\rr_n$ component is $(\delta^k_1,\ldots,\delta^k_n)^\dagger$, 
and $\delta_i^j=1$ if $i=j$, $\delta_i^j=0$ if $i\neq j$.

\pparagraph{Linear differential equations}\label{seclindifinc-cont}

We want to study the linear differential equation
\begin{equation}\label{lindifinc-cont}
\dot x=A(t)x\,,
\end{equation}
under the assumption that 
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{chapproofmax}.A2.1)] {\em $n\in\nn$, 
$I=[a,b]$ is a compact real interval, and 
$A\in C^{0,pc}(I,\rr^{n\times n})$.}
\end{itemize}\end{itemize}
\begin{definition}
Assume that (\ref{chapproofmax}.A2.1) holds.  A {\bf\em vector
solution} of (\ref{lindifinc-cont}) on $I$ is a continuous map
$\xi:I\mapsto\rr^n$ such that $\dot\xi(t)=A(t)\xi(t)$ 
for all $t\in I$ that are points of \mbox{continuity}  of $A$. 
A {\bf\em matrix solution} of (\ref{lindifinc-cont}) on $I$
is a continuous map \mbox{$\Xi:I\mapsto\rr^{n\times n}$} such that
$\dot\Xi(t)=A(t)\Xi(t)$ for all $t\in I$ that are points of 
\mbox{continuity}  of $A$.  \proofend
\end{definition}

The following existence and uniqueness theorem is well known.
\begin{proposition}\label{exiuniquelin-cont}
Assume that (\ref{chapproofmax}.A2.1) 
holds.  Let $\bar t\in I$, $\bar x\in\rr^n$. 
Then there exists one and only one vector solution  $\xi$ of 
(\ref{lindifinc-cont}) such that $\xi(\bar t)=\bar x$.
\end{proposition}
\begin{proof}
Fix $\bar t$. Endow the space $\rr^{n\times n}$ with the operator norm. 
Let 
$$
k=\sup\{\|A(t)\|:t\in I\}\,.
$$ 
Let $\mathcal{V}$ be the space $C^0(I,\rr^n)$ of all continuous maps
from $I$ to $\rr^n$, endowed with the norm $\|\cdot\|_{_\mathcal{V}}$ 
given by 
\mbox{$\|\xi\|_{_\mathcal{V}}=\max\{e^{-k|t-\bar t|}\|\xi(t)\|:t\in I\}$}.  
Then $\mathcal{V}$ is a Banach space. Define a map 
$\Phi:\mathcal{V}\mapsto\mathcal{V}$ by letting
$$
\Phi(\xi)(t)=\bar x+\int_{\bar t}^tA(s)\xi(s)\,ds\,.
$$
It is then clear that $\xi$
is a vector solution of (\ref{lindifinc-cont}) such that $\xi(\bar t)=\bar
x$ if and only if $\Phi(\xi)=\xi$. 

If $\xi,\xi'\in\mathcal{V}$, and we write $\zeta=\Phi(\xi)$,
$\zeta'=\Phi(\xi')$, we have
\begin{eqnarray*}
e^{-k|t-\bar t|}\|\zeta(t)-\zeta'(t)\|&=&
e^{-k|t-\bar t|}\Big\|\int_{\bar t}^tA(s)(\xi(s)-\xi'(s))\,ds\Big\|\\
&\le&ke^{-k|t-\bar t|}\int_{\min(\bar t,t)}^{\max(\bar t,t)}
\|\xi(s)-\xi'(s)\|\,ds\\
&\le&ke^{-k|t-\bar t|}\int_{\min(\bar t,t)}^{\max(\bar t,t)}
e^{k|s-\bar t|}e^{-k|s-\bar t|}\|\xi(s)-\xi'(s)\|\,ds\\
&\le&ke^{-k|t-\bar t|}\|\xi-\xi'\|_{_\mathcal{V}}
\int_{\min(\bar t,t)}^{\max(\bar t,t)}e^{k|s-\bar t|}\,ds\\
&=&ke^{-k|t-\bar t|}\|\xi-\xi'\|_{_\mathcal{V}}
\Biggl(\frac{1}{k}\Big(e^{k|t-\bar t|}-1\Big)\Biggr)\\
&=&\|\xi-\xi'\|_{_\mathcal{V}}\Big(1-e^{-k|t-\bar t|}\Big)\\
&\le&\|\xi-\xi'\|_{_\mathcal{V}}\Big(1-e^{-k(b-a)}\Big)\\
&=&\kappa\|\xi-\xi'\|_{_\mathcal{V}},
\end{eqnarray*}
where $\kappa=1-e^{-k(b-a)}$.

It follows that 
$\|\zeta-\zeta'\|_{_\mathcal{V}}\le\kappa\|\xi-\xi'\|_{_\mathcal{V}}$.  
Hence $\Phi$ is a map from $\mathcal{V}$ to $\mathcal{V}$ that satisfies
$\|\Phi(\xi)-\Phi(\xi')\|_{_\mathcal{V}}\le\kappa\|\xi-\xi'\|_{_\mathcal{V}}$ 
whenever $\xi,\xi'\in\mathcal{V}$. Since $\kappa<1$, the contraction
mapping principle implies that there exists a unique $\xi\in\mathcal{V}$
such that $\Phi(\xi)=\xi$.  Hence our statement is
proved.
\end{proof}

The following result can be proved by an argument similar to the proof
of Proposition \ref{exiuniquelin-cont}, using instead of $\mathcal{V}$ an
appropriate space of matrix-valued functions. Alternatively, if
$e_1,\ldots,e_n$ is the canonical  basis of $\rr^n$
(that is, $e_i=(\delta_i^1,\delta_i^2,\ldots,\delta_i^n)$, where
$\delta_i^j=1$ if $i=j$ and $\delta_i^j=0$ if $i\neq j$), 
and $\xi_1,\ldots,\xi_n$ are the
unique vector solutions of (\ref{lindifinc-cont}) such that $\xi_j(\bar
t)=\bar Z e_j$ for $j=1,\ldots,n$, we can define $\Xi(t)\in\rr^{n\times n}$ by
letting $\Xi(t)$ be the matrix with columns $\xi_1,\ldots,\xi_n$.
It is then
easy to verify that $\Xi$ is the unique matrix solution of
(\ref{lindifinc-cont}) such that $\Xi(\bar t)=\bar Z$.)

\begin{proposition}
Assume that (\ref{chapproofmax}.A2.1) holds. 
Let $\bar t\in I$, $\bar Z\in\rr^{n\times n}$. 
Then there exists one and only one matrix solution  $\Xi$ of 
(\ref{lindifinc-cont}) such that $\Xi(\bar t)=\bar Z$.\proofend
\end{proposition}

If $s,t\in I$, then there exists a unique matrix solution 
$\Xi$ of  (\ref{lindifinc-cont}) such that $\Xi(s)=\idrn$. (Recall
that $\idrn$ is the identity map of $\rr^n$.) The value of this
solution is a function of $t$, which also depends on $s$ as a parameter,
so we will use $\Xi^A(t,s)$ to denote this value. Then $\Xi^A$ is a function
from $I\times I$ to $\rr^{n\times n}$, uniquely characterized by the integral 
equation
\begin{equation}\label{c-flowid-int}
\Xi^A(t,s)=\idrn+\int_s^tA(r)\Xi^A(r,s)\,dr\,.
\end{equation}
It is clear that $\Xi^A$ satisfies 
\begin{equation}\label{c-flowid-a}
\Xi^A(t,t)=\idrn\qquad{\rm whenever}\quad t\in I\,.
\end{equation}

If we fix two members $t_2,t_3$ of $I$, then both functions 
\mbox{$I\ni t\mapsto\Xi^A(t,t_2)\Xi^A(t_2,t_3)$} and
\mbox{$I\ni t\mapsto\Xi^A(t,t_3)$} are matrix solutions of
(\ref{lindifinc-cont}) which take the value $\Xi^A(t_2,t_3)$ when 
$t=t_2$. Hence these two solutions coincide, from which it follows that
\begin{equation}\label{c-flowid-b}
\Xi^A(t_1,t_2)\Xi^A(t_2,t_3)=\Xi^A(t_1,t_3)\qquad{\rm whenever}\quad 
t_1,t_2,t_3\in I\,.
\end{equation}
Equations (\ref{c-flowid-a}) and (\ref{c-flowid-b}) are the 
{\bf\em flow identities}. They imply, in \mbox{particular}, that 
\begin{equation}\label{c-flowid-baa}
\Xi^A(t,s)\,\,\,{\rm is\,\,\,invertible\,\,\,for\,\,\,every}\,\,\,t,s
\end{equation}
(because of the identity $\Xi^A(t,s)\Xi^A(s,t)=\idrn$) 
and that
\begin{equation}\label{c-flowid-c}
\Xi^A(t,s)=\Xi^A(t,\bar t)\Xi^A(s,\bar t)^{-1}
\qquad{\rm whenever}\quad t,s,\bar t\in I
\end{equation}
(because $\Xi^A(t,\bar t)=\Xi^A(t,s)\Xi^A(s,\bar t)$). In particular,
(\ref{c-flowid-c}) implies that 
\begin{equation}\label{c-flowid-caa}
{\rm the\,\,\,map\,\,\,}I\times I\ni(t,s)\mapsto\Xi^A(t,s)\in\rr^{n\times n}
\,\,\,{\rm is\,\,\,continuous}\,,
\end{equation}
because $\Xi^A(t,\bar t)$ and $\Xi^A(s,\bar t)$ are 
continuous functions of $t,s$.

\begin{definition}
{\rm Assume that conditions (\ref{seclindifinc-cont}.1-2-3-4) hold. Then 
the  \mbox{map} \mbox{$\Xi^A\in C^0(I\times I,\rr^{n\times n})$}
is the {\bf\em fundamental solution} of (\ref{lindifinc-cont}).}\proofend
\end{definition}

Naturally, using the fundamental solution $\Xi^A$ it is easy to write
explicit formulas for all the vector and matrix solutions of
(\ref{lindifinc-cont}): 

\begin{itemize}
\item[(1)] {\em The vector solution $\xi$ of (\ref{lindifinc-cont})
that satisfies  $\xi(\bar t)=\bar x\in\rr^n$ is given by 
\fbox{$\xi(t)=\Xi^A(t,\bar t)\bar x$}.}
\item[(2)] {\em The matrix solution $\Xi$ of (\ref{lindifinc-cont})
that satisfies $\Xi(\bar t)=\bar Z\in\rr^{n\times n}$ is given
by \fbox{$\Xi(t)=\Xi^A(t,\bar t)\bar Z$}.}
\end{itemize}

We now study how the fundamental solution $\Xi^A$ depends on $A$.
First of all, if $A$ is fixed, we let $k_A=\max\{\|A(t)\|:t\in I\}$,
$\kappa_A=e^{k_A(b-a)}$, and observe that the integral
equation (\ref{c-flowid-int}) implies the bound
$$
\|\Xi^A(t,s)\|\le 1+k_A\int_{\min(t,s)}^{\max(t,s)}\|\Xi^A(r,s)\|\,dr\,,
$$
and then Gronwall's inequality yields the estimate
\begin{equation}\label{estigron}
\|\Xi^A(t,s)\|\le e^{k_A|t-s|}\,,
\end{equation}
so in particular $\|\Xi^A(t,s)\|\le\kappa_A$.

Now suppose we are given two piecewise continuous
functions \mbox{$A,A':I\mapsto\rr^{n\times n}$}. 
Define 
\begin{equation}\label{defmetthteta}
\theta_{A,A'}=\int_a^b\|A(t)-A'(t)\|\,dt\,.
\end{equation}
Then, if $t,s\in I$, we have
\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\Xi^A(t,s)-\Xi^{A'}(t,s)\\
&=&\int_s^t\Big(A(r)\Xi^A(r,s)-A'(r)\Xi^{A'}(r,s)\Big)\,dr\\
&=&\int_s^t\Big(A(r)-A'(r)\Big)\Xi^A(r,s)\,dr
+\int_s^tA'(r)\Big(\Xi^A(r,s)-\Xi^{A'}(r,s)\Big)\,dr
\end{eqnarray*}
so 
\begin{eqnarray*}
\|\Xi^A(t,s)-\Xi^{A'}(t,s)\|
&\le&\kappa_A\theta_{A,A'}
+k_{A'}\int_{\min(t,s)}^{\max(t,s)}\Big\|\Xi^A(r,s)-\Xi^{A'}(r,s)\Big\|\,dr\,,
\end{eqnarray*}
and then Gronwall's inequality implies
\begin{equation}\label{c-gronimp}
\|\Xi^A(t,s)-\Xi^{A'}(t,s)\|\le\kappa_A\kappa_{A'}\theta_{A,A'}\,.
\end{equation}

The following result then follows immediately:
\begin{proposition}\label{immed}
Let $n\in\nn$, $a,b\in\rr$, $a\le b$.  Assume that
$S$ is a \mbox{subset} of the space 
$C^{0,pc}([a,b],\rr^{n\times n})$ such that 
there exists a
constant $k\in\rr$ with the \mbox{property} that 
$\|A(t)\|\le k$ whenever $A\in S$, $t\in[a,b]$. Then the fundamental 
\mbox{solution}
map $S\ni A\mapsto\Xi^A\in C^0([a,b]\times[a,b],\rr^{n\times n})$  is 
Lipschitz, if 
$S$ is endowed with the metric $d_S$ given by 
$d_S(A,A')=\theta_{A,A'}$ (cf. (\ref{defmetthteta})). That is, there 
exists a constant $\bar\kappa$
such that $\|\Xi^A-\Xi^{A'}\|_{sup}\le\bar\kappa\theta_{A,A'}$ whenever
$A,A'\in S$. 
\end{proposition}
\begin{proof}
It suffices to take $\bar\kappa=e^{2k(b-a)}$ and use (\ref{c-gronimp}).
\end{proof}

\pparagraph{The Hahn-Banach theorem}\label{sechahn}

THIS SECTION HAS TO BE WRITTEN

\subsection{The classical differential and its properties}
\label{sec-gdq-intro}

\noindent In this subsection we assume that
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{chapproofmax}.A3.1)] 
{\em $X$ is a normed space and $S$ is a  subset of $X$}, 
\item[(\ref{chapproofmax}.A3.2)] {\em $Y$ is a normed space}, 
\item[(\ref{chapproofmax}.A3.3)] $s_*\in S$, 
\item[(\ref{chapproofmax}.A3.4)] {\em $L$ is a bounded linear map 
from $X$ to $Y$},
\item[(\ref{chapproofmax}.A3.5)] {\em $F:S\brokarrow Y$ is a ppd map}.
\end{itemize}\end{itemize}
\begin{definition}
{\rm Assume that (\ref{chapproofmax}.A3.1) holds. The {\bf\em Bouligand 
tangent cone} of $S$ at 
$s_*$ is the set $T_{s_*}^BS$ of all limits
$v=\lim_{j\to\infty}\frac{s_j-s_*}{h_j}$, for all sequences
$\{s_j\}_{j\in\nn}$, $\{h_j\}_{j\in\nn}$, 
such that (i)~$s_j\in S$ and $h_j>0$ for all $j$,
and (ii)~$h_j\to 0$ as $j\to\infty$.}\proofend
\end{definition}
\begin{definition}\label{defoxit}
{\rm Assume that (\ref{chapproofmax}.A3.1-2-3-4-5) hold. 
We say that $L$ is a {\bf\em Fr\'echet differential}---or, simply, 
a {\bf\em differential}---{\bf\em of $F$ at $s_*$} 
if 
\begin{itemize}\item[]\begin{itemize}\item[]\begin{itemize}
\item[(\ref{defoxit}.a)] {\em $F(s)$ is defined  for all points $s$ 
that belong to some relative  neighborhood}
\mbox{$S\cap\bmath_X(s_*,\delta)=\{s\in S:\|s-s_*\|<\delta\}$} of
$s_*$ in $S$,
\end{itemize}\end{itemize}\end{itemize}
and 
\begin{itemize}\item[]\begin{itemize}\item[]\begin{itemize}
\item[(\ref{defoxit}.b)] {\em the identity
\begin{equation}\label{frechet}
\lim_{s\to s_*,s\in S,s\neq s_*}\frac{\|F(s)-F(s_*)-L\cdot(s-s_*)\|}
{\|s-s_*\|}=0
\end{equation}
holds.}
\end{itemize}\end{itemize}\end{itemize}
We say that $F$ is {\bf\em differentiable  at $s_*$} if there exists
a bounded linear map \mbox{$L:X\mapsto Y$} such that 
$L$ is a differential of $F$ at $s_*$.} \proofend
\end{definition}
Clearly, if
$F$ is  differentiable  at $s_*$, then the differential $L$ of $F$
at $s_*$ is uniquely determined on the closed linear subspace
$$
\Sigma(S,s_*)={\rm Clos}_X\Biggl({\rm lin\,span}T_{s_*}^BS\Biggr)\,.
$$
(Indeed, if $L_1$, $L_2$ are two differentials of $F$ at $s_*$, and 
$v\in T_{s_*}^BS$, then we can find sequences 
$\{s_j\}_{j=1}^\infty$ in $S$, $\{h_j\}_{j=1}^\infty$ in $\,]0,+\infty\,[$,
such that $v=\lim_{j\to\infty}\frac{s_j-s_*}{h_j}$. Then, if $i=1,2$, and
$j$ is large enough, we have
$L_i\cdot(s_j-s_*)=F(s_j)-F(s_*)+o(\|s_j-s_*\|)$, and $s_j-s_*=h_jv+o(h_j)$, 
so $L_i\cdot(s_j-s_*)=F(s_j)-F(s_*)+o(h_j)$, and then
$$
L_i\cdot\Biggl(\frac{s_j-s_*}{h_j}\Biggr)=
\frac{F(s_j)-F(s_*)}{h_j}+o(1)\,\,\,{\rm as}\,\,\,j\to\infty\,,
$$
from which it follows that 
$$
L_i\cdot v=\lim_{j\to\infty}\Biggl(\frac{F(s_j)-F(s_*)}{h_j}\Biggr)\,.
$$
Hence $L_1\cdot v=L_2\cdot v$. So $L_1$ and $L_2$ agree on $T_{S_*}^BS$ and 
then they agree on $\Sigma(S,s_*)$, since they are linear and continuous.)

We use $\mathcal{D}F(s_*)$ to denote the set of all differentials of
$F$ at $s_*$. Then $\mathcal{D}F(s_*)$ consists of a single 
linear map if and only if $F$ is differentiable at $s_*$ and
\mbox{$\Sigma(S,s_*)=X$}. In that case, the unique member of 
$\mathcal{D}F(s_*)$ will be referred to as {\bf\em the 
\mbox{differential} of $F$ at $s_*$}, and will be  represented 
with the notation $DF(s_*)$. 

\begin{remark}
{\rm More generally, if $S\subseteq S'\subseteq X$, $s_*\in S$, and
$F:S'\brokarrow Y$, we may want to consider the ``differentials
of $F$ at $s_*$ in the direction of $S$.''  We will not, however,
introduce a separate definition because, if we follow the terminology
introduced earlier, differentiating $F$  at $s_*$ in the direction of $S$
just amounts to differentiating at $s_*$ the restriction 
$F\restr S$ of $F$ to $S$.}\proofend
\end{remark}


\pparagraph{The chain rule}
\begin{proposition}\label{chainrule}
Let $X_1$, $X_2$, $X_3$ be normed real linear spaces, let $S_1$, $S_2$
be subsets of $X_1$, $X_2$, let $F_1$, $F_2$ be ppd maps from $S_1$ to $X_2$
and from $S_2$ to $X_3$, respectively,
such that $F_1(S_1)\subseteq S_2$. Let $s_*\in S_1$
be such that $F_1$ is differentiable at $s_*$, so in particular
$\sigma_*\equaldef F_1(s_*)$ exists and belongs to $S_2$. 
Assume that  $F_2$ is differentiable at $\sigma_*$. Let
$L_1\in\mathcal{D}F_1(s_*)$, $L_2\in\mathcal{D}F_2(\sigma_*)$.
Let $F=F_2\compo F_1$, $L=L_2\compo L_1$. Then $L\in\mathcal{D}F(s_*)$.
\end{proposition}
\begin{proof}
We assume, as we clearly may without loss of generality,
that $s_*=0$, $\sigma_*=0$.  Fix positive numbers 
$r_1^{\#}$, $\bar r_2^{\#}$, such that $F_i$ is everywhere defined on 
$U_i^{\#}\cap S_i$ for $i=1,2$, where
$U_i^{\#}=\bmath_{X_i}(0,r_i^{\#})$. Since 
$\|s\|^{-1}\Big(F_1(s)-L_1(s)\Big)$
goes to zero as $s\to 0$ via values in $U_1^{\#}\cap S_1$, 
we may pick a positive $\tilde r_1$ such that $\tilde r_1\le r_1^{\#}$ 
and $\|F_1(s)-L_1(s)\|\le \|s\|$ whenever $s\in S_1$ and
$\|s\|\le\tilde r_1$. Let $\kappa=1+\|L_1\|$. Then
$\|F_1(s)\|\le\kappa\|s\|$ whenever $s\in S_1$ and
$\|s\|\le\tilde r_1$. Pick a positive $r_1$ such that 
$r_1\le\tilde r_1$ and $\kappa mr_1\le r_2^{\#}$. Then 
$F_1(s)\in U_2^{\#}\cap S_2$ whenever $s\in S_1$ and
$\|s\|\le r_1$. Define $r_2=r_2^{\#}$, and  let
$U_i=\bmath_{X_i}(0,r_i)$ for $i=1,2$. Then, for $i=1,2$, 
$F_i$ is everywhere defined on  $U_i\cap S_i$, and in addition
$F_1(U_1\cap S_1)\subseteq U_2\cap S_2$. 

For $i=1,2$, $0<r\le r_i$, let
$$
\theta_i(r)=\sup\Biggl\{\frac{\|F_i(s)-L_i\cdot s\|}{\|s\|}:
s\in S_i,0<\|s\|\le r\Biggr\}\,,
$$
so that $\theta_i(r)\to 0$ as $r\downarrow 0$. Then, if 
$s\in U_1\cap S_1$, and we write $\sigma=F_1(s)$, we have
\begin{eqnarray*}
\|F(s)-L(s)\|&=&\|F_2(F_1(s))-L_2(L_1(s))\|\\
&\le&\|F_2(F_1(s))-L_2(F_1(s))\|+\|L_2(F_1(s))-L_2(L_1(s))\|\\
&\le&\theta_2(\|\sigma\|)+\|L_2\|\,\|F_1(s)-L_1(s)\|\\
&\le&\theta_2(\|\sigma\|)\|\sigma\|+\|L_2\|\theta_1(\|s\|)\|s\|\,.
\end{eqnarray*}
On the other hand, 
$$
\|\sigma\|=\|F_1(s)\|\le\|F_1(s)-L_1(s)\|+\|L_1(s)\|
\le\theta_1(\|s\|)\,\|s\|+\|L_1\|\,\|s\|\,.
$$
Therefore $\|F(s)-L(s)\|\le\Theta(\|s\|)\,\|s\|$, where
$$
\Theta(r)\equaldef 
\theta_2\Big(r(\theta_1(r)+\|L_1\|)\Big)\Big(\theta_1(r)+\|L_1\|\Big)
+\|L_2\|\theta_1(r)\,.
$$
Clearly, $\lim_{r\downarrow 0}\Theta(r)=0$, so 
$\|s\|^{-1}\|F(s)-L(s)\|$ goes to $0$ as $s\to 0$ via values
in $S_1$, and our proof is complete.
\end{proof}

\pparagraph{Open mapping theorems for classical differentials}
\label{sub-openmap-clarkewarga}
If $X$, $Y$ are metric \mbox{spaces}, $F:X\mapsto Y$ is a map, and  
$x\in X$, we say that $F$ is {\bf\em open at $x$} if $F$ maps 
neighborhoods of $x$ to neighborhoods of $F(x)$ (that is, whenever $U$ 
is a subset of $X$ such that $x\in{\rm Int}_X\,U$ it follows that 
$F(x)\in{\rm Int}_X\,F(U)$).

\begin{definition}\label{def-weak-open-dir}
{\rm Let $X$, $Y$ be normed real linear spaces, let $\bar x$, $v$ be points of 
$X$, $Y$, respectively, let $S$ be a subset of 
$X$ such that $\bar x\in S$, and let $F:S\mapsto Y$ be a map. 
We say that $F$ is {\bf\em directionally open at $\bar x$ along $S$ 
in the direction of $v$} if there exists a convex cone $D$ in $Y$ 
such that
\begin{itemize}\item[]\begin{itemize}\item[]\begin{itemize}
\item[(\ref{def-weak-open-dir}.1)] $v\in{\rm Int}(D)$,
\item[(\ref{def-weak-open-dir}.2)] {\em for every positive number $\alpha$
there exists a positive number $\beta=\beta(\alpha)$ such that}
\begin{itemize}
\item[(\ref{def-weak-open-dir}.2.i)] {\em the image under $F$ of the set 
$\{\,x\in S:\|x-\bar x\|\le\alpha\,\}$ contains the set}
$\{\,F(\bar x)+y:y\in D,\|y\|\le\beta\,\}$. 
\end{itemize}\end{itemize}\end{itemize}\end{itemize}
If the function $\alpha\mapsto\beta(\alpha)$ can be chosen so
that there exist positive numbers $\alpha^*$, $\mu$ such that
$\beta(\alpha)=\mu\alpha$ for $0<\alpha\le\alpha^*$, then
$F$ is said to be {\bf\em directionally open with linear rate
at $\bar x$ along $S$ in the direction of $v$}}.\proofend
\end{definition}

\medbreak
\begin{center}
\fbox{\fbox{
\begin{minipage}{11.3cm}
{\large
\begin{theorem}\label{theoopendirclass} 
{\rm (The $\,$directional $\,$open $\,$mapping $\,$theorem 
$\,$for $\,$the $\,$classical \mbox{differential}).} 
Let $X$, $Y$ be normed real linear spaces such that
$Y$ is finite-dimensional, let $\bar x\in X$, $v\in Y$, let $C$ be
a convex cone in $X$, and let 
$$
S=\bar x+C=\{\,x\in X:x-\bar x\in C\,\}\,.
$$
Let $U$ be a neighborhood of $\bar x$ in $X$, let 
$F:U\cap S\mapsto Y$ be a continuous map, and let
$L:X\mapsto Y$ be a bounded linear map such that
$L\in\mathcal{D}F(\bar s)$. 
Assume that $v\in{\rm Int}(L\cdot C)$.  Then $F$ is directionally
open with linear rate at $\bar x$ along $S$ in the direction of $v$.
\end{theorem}}
\end{minipage}}}
\end{center}

\begin{proof}
We assume, without loss of generality, that $\bar x=0$ and 
$F(\bar x)=0$. We let $m={\rm dim}\,Y$.
If $w\in Y$ and $\rho>0$, we use $\bar\bmath_Y(w,\rho)$ to denote the closed
ball   $\{\,y\in Y:\|y-w\|\le\rho\,\}$.  
We let $(\tilde e_1,\ldots,\tilde e_m)$ be a basis of $Y$, write 
$\tilde e_0=-(\tilde e_1+\ldots+\tilde e_m)$,
and pick an $r$ such that $r>0$ and $v+r\tilde e_j\in L\cdot C$ for
every $j\in\{0,\ldots,m\}$. We then let $e_j=v+r\tilde e_j$, 
and  observe that the vectors $e_0,\ldots,e_m$ are affinely independent, 
i.e., that whenever a linear combination
$a_0e_0+a_1e_1+\ldots+a_me_m$ with $a_0+a_1+\ldots+a_m=0$ is equal to zero, 
it follows that $a_0=a_1=\ldots=a_m=0$. (Indeed, if 
$a_0e_0+a_1e_1+\ldots+a_me_m=0$ and $a_0+a_1+\ldots+a_m=0$, 
we get $(a_0+a_1+\ldots+a_m)v+ra_0\tilde e_0
+ra_1\tilde e_1+\ldots+ra_m\tilde e_m=0$. 
Then $a_0\tilde e_0+a_1\tilde e_1+\ldots+a_m\tilde e_m=0$,
so $(a_1-a_0)\tilde e_1+\ldots+(a_m-a_0)\tilde e_m=0$,
which implies $a_1-a_0=a_2-a_0=\ldots=a_m-a_0=0$,
so $a_0=a_1=a_2=\ldots=a_m$, and then
$0=a_0+a_1+\ldots+a_m=(m+1)a_0$, so 
$a_0=a_1=a_2=\ldots=a_m=0$.) We then 
remark that every  member $y$ of $Y$ can be expressed 
in a unique way as an affine combination (that is, 
a linear combination whose coefficients add up to~$1$) 
$a_0(y)e_0+a_1(y)e_1+\ldots+a_m(y)e_m$ of the $e_j$. 
(Indeed, we can find $b_1,\ldots,b_m$ such that
$r^{-1}(y-v)=b_1\tilde e_1+\ldots+b_m\tilde e_m$, 
since the $\tilde e_j$ form a basis of $Y$. Then we can
let $a_0=(m+1)^{-1}(1-b_1-\ldots-b_m)$ and $a_j=a_0+b_j$
for $j=1,\ldots,m$. It is then easy to verify
that $y=a_0e_0+a_1e_1+\ldots+a_me_m$ and $a_0+a_1+\ldots+a_m=1$.
Uniqueness follows from the fact that $e_0,\ldots,e_m$ are affinely 
independent.) It is clear that the functions
$Y\ni y\mapsto a_j(y)\in\rr$ are continuous.

Since $v=(m+1)^{-1}(e_0+\ldots+e_m)$, we have $a_j(v)=(m+1)^{-1}$
for $j=0,\ldots,m$. Therefore we can fix a $\rho$ such that
$\rho>0$ and $a_j(y)>0$ for all $j\in\{0,\ldots,m\}$ and
all  $y\in\bar\bmath_Y(v,\rho)$. Let $W=\bar\bmath_Y(v,\rho)$. 
Then $W\subseteq L\cdot C$, because 
if $y\in\bar\bmath_Y(v,\rho)$ then 
$y=\sum_{j=0}^ma_j(y)e_j$, the $e_j$ are in $L\cdot C$, 
$\sum_{j=0}^ma_j(y)=1$, and $a_j(y)>0$, so 
$y$ is a convex combination of the $e_j$.

Since $e_j\in L\cdot C$, we can  pick $c_j\in C$ such that
$L\cdot c_j=e_j$. 
Let $M:Y\mapsto X$ be the map that sends $y\in Y$ to 
the affine combination $M\cdot y\equaldef\sum_{j=0}^m a_j(y)c_j$. 
Then $M$ is a continuous (and in fact affine, but not necessarily
linear) map, and $L\circ M={\rm identity}_{Y}$, because
$L(M\cdot y)=\sum_{j=0}^m a_j(y)L\cdot c_j
=\sum_{j=0}^m a_j(y)e_j=y$.   

Let $\kappa=\sup\{\|M\cdot y\|:y\in W\}$. For $s>0$, let 
$$
\theta(s)=\sup\Biggl\{\frac{\|F(x)-L\cdot x\|}{\|x\|}:x\in C,\|x\|
\le s\Biggr\}\,,
$$
so $\theta(s)\to 0$ as $s\downarrow 0$. 

Now fix a $\rho^*$ such that (i)~$0<\rho^*<\rho$, and (ii)~if $v\neq 0$ 
then $\rho^*<\|v\|$. Then define $W^*=\bar\bmath_Y(v,\rho^*)$.  
Also, let $\delta^*$ be such that $\delta^*>0$ and
$\bar\bmath_X(0,\delta^*)\subseteq U$.  Then pick a $\tau^*$ such
that $0<\tau^*$, $\kappa\tau^*\le\delta^*$, and
$\kappa\theta(\tau^*\kappa)\le\rho-\rho^*$. Let
$\alpha^*=\kappa\tau^*$, so $0<\alpha^*\le\delta^*$.  For each
positive number $\alpha$, define
$V_\alpha=\bar\bmath_X(0,\tilde\alpha)$ and
$\tau_\alpha=\frac{\tilde\alpha}{\kappa}$, where
$\tilde\alpha=\min(\alpha,\alpha^*)$.  
Then $V_\alpha\subseteq U$, and 
$$
tM\cdot y\in V_\alpha\cap C\quad
{\rm whenever}\quad\alpha>0\,,y\in W\,,\,0\le t\le\tau^*\,.
$$
We then define 
$$
G_t(y)=\frac{1}{t}\Big(\,F(tM\cdot y)\,\Big)\quad
{\rm for}\quad y\in W\,,\,\,0<t\le\tau^*\,.
$$
Clearly, $G_t$ is a continuous map from $W$ to $Y$, if 
$t\in\,]\,0,\tau^*]$, and 
\begin{eqnarray*}
\|G_t(y)-y\|&=&\frac{1}{t}\Big(\|F(tM\cdot y)-ty\|\Big)\\
&\le&\frac{1}{t}\Big(\|F(tM\cdot y)-\|t(L\circ M)\cdot y)\|\Big)\\
&=&\frac{1}{t}\Big(\|F(tM\cdot y)-\|L(tM\cdot y)\|\Big)\\
&\le&\frac{1}{t}\Big(\theta(t\kappa)t\kappa\Big)\\
&=&\kappa\theta(t\kappa)\,.
\end{eqnarray*}

If $y\in W^*$ and $0<t\le\tau^*$, 
we can consider the map $\mu_{t,y}:W\mapsto Y$
given by 
$$
\mu_{t,y}(z)=z-G_t(z)+y\,.
$$
Then 
$$
\|\mu_{t,y}(z)-v\|=\|z-G_t(z)+y-v\|\le\|z-G_t(z)\|+\|y-v\|
\le\kappa\theta(t\kappa)+\rho^*\le\rho\,,
$$
since $\kappa\theta(\tau^*\kappa)\le\rho-\rho^*$. Therefore
$\mu_{t,y}$ is a continuous map from $W$ to $W$.
By the Brouwer fixed point theorem, there exists, for
each $y\in W^*$ and each $t\in\,]0,t^*]$, a $z=z(t,y)\in W$ 
such that $\mu_{t,y}(z)=z$. Then $z-G_t(z)+y=z$, so $G_t(z)=y$.
But then $F(tM\cdot(z(t,y)))=ty$.  Since this is true for
every $y\in W^*$ and every $t\in\,]0,\tau^*]$, we have shown that
$$
tW^*\subseteq F(tM(W))\,\,\,{\rm whenever}\,\,\,0<t\le\tau^*\,.
$$
Furthermore, the inclusion of the previous line also holds
for $t=0$, since $F(0)=0$. It follows that 
$$
\bigcup_{0\le t\le\tau_\alpha}\,tW^*\subseteq F(V_\alpha\cap C)
\,\,\,{\rm whenever}\,\,\,\alpha>0\,.
$$
Let $D$ be the smallest convex cone containing $W^*$. Then
$v$ is an interior point of $D$. Furthermore, $D$ is closed. 
(Indeed, if $v=0$ then $D=Y$, so $D$ is closed. Assume that
$v\neq 0$. If $\{y_j\}_{j=1}^\infty$ is a sequence
of members of $D$ that converges to a limit $y$, 
then we can write $y_j=t_jw_j$, $t_j\in[0,\infty\,[\,$, 
$w_j\in W^*$. After passing to a subsequence, if necessary,
we may assume that $w=\lim_{j\to\infty}w_j$ exists and belongs to
$W^*$.  Since $v\neq 0$, the inequality $\rho^*<\|v\|$ implies that
$0\notin W^*$, so $w$ and the $w_j$ are all $\neq 0$. Then
$t_j=\frac{\|y_j\|}{\|w_j\|}$, and $\lim_{j\to\infty}t_j=t=
\frac{\|y\|}{\|w\|}$.  It follows that $y=tw$, so $y\in D$.)

Now fix a constant $\nu$ such that 
$\bmath_Y(0,1)\cap D\subseteq\bigcup_{0\le t\le\nu}tW^*$.
(The existence of $\nu$ is trivial. Indeed, if $v=0$ we can 
pick $\nu=\frac{1}{\rho^*}$. If $v\neq 0$, then 
every $w\in W^*$ satisfies $\|w-v\|\ge\rho^*$, so 
$\|v\|\le\rho^*+\|w\|$, and then 
$\|w\|\ge\|v\|-\rho^*$. Since $\rho^*<\|v\|$, we can take 
$\nu$ to be any nmuber such that $\nu\ge(\|v\|-\rho^*)^{-1}$, and conclude
that if $y\in\bmath_Y(0,1)\cap D$, and we
write $y=tw$, with $w\in W^*$, then 
$t=\frac{\|y\|}{\|w\|}\le\frac{1}{\|v\|-\rho^*}
\le\nu$.)  It follows that 
$\bmath_Y(0,s)\cap D\subseteq\bigcup_{0\le t\le\nu s}tW^*$
for every positive $s$.  

Now, if $\alpha$ is an arbitrary positive number, 
and we let $\beta(\alpha)=\nu^{-1}\tau_\alpha$, we have 
$$
\bmath_Y(0,\beta(\alpha))\cap D\subseteq
\bigcup_{0\le t\le\nu\beta(\alpha)}tW^*=\bigcup_{0\le t\le\tau_\alpha}tW^*
\subseteq F(V_\alpha\cap C)\,.
$$
The proof of Theorem  \ref{theoopendirclass} is thus complete. 
\end{proof}


\medbreak
\begin{center}
\fbox{\fbox{
\begin{minipage}{12cm}
{\large
\begin{theorem}\label{theoopenclass} 
{\rm (The $\,$open $\,$mapping $\,$theorem $\,$for $\,$the 
$\,$classical \mbox{differential}).}
Let $X$, $Y$ be normed real linear spaces such that
$Y$ is finite-dimensional, let $\bar x\in X$, 
let $U$ be a neighborhood of $\bar x$ in $X$, and let 
$F:U\mapsto Y$ be a continuous map. Assume that $F$ is 
\mbox{differentiable} at $\bar x$, and let $L=DF(\bar x)$. 
Assume that $L$ is \mbox{surjective}. Then $F$ is open at $\bar x$.
\end{theorem}}
\end{minipage}}}
\end{center}

\begin{proof}
It suffices to apply Theorem  \ref{theoopendirclass} with
$v=0$, $C=X$.
\end{proof}


\pparagraph{Approximating cones}\label{sec-appcones} 
Assume that $Y$ is a normed real linear space, $S$ is a 
subset of $Y$, $s_*\in S$, and $C$ is a convex cone 
in $Y$. We say that $C$ is an {\bf\em approximating cone
to $S$ at $s_*$} if there exist a normed space $X$,
a neighborhood $U$ of $0$ in $X$, a convex cone 
$D$ in $X$, a continuous map $F:U\cap D\mapsto Y$,
and a bounded linear map $L\in{\rm Lin}(X,Y)$, such that
$F(0)=s_*$, $F(U\cap D)\subseteq S$, $LD=C$, 
and $L\in\mathcal{D}F(0)$. 

We say that $C$ is a {\bf\em limiting 
approximating cone to $S$ at $s_*$} if there exists
an increasing sequence $\{C_j\}_{j=1}^\infty$ of convex cones
in $Y$ such that \mbox{$C={\rm Clos}\,\bigcup_{j=1}^\infty C_j$}
and each $C_j$ is an approximating cone
to $S$ at $s_*$.

\pparagraph{Transversality of cones}\label{sec-transcones} 
Suppose $X$ is a real linear space. We say 
that two convex cones $C^1$, $C^2$ in $X$ are {\bf\em transversal}, and write 
$$
C^1\transv C^2\,\,\,,
$$
if $C^1-C^2=X$.   We say that $C^1$ and $C^2$ in $X$ are {\bf\em strongly
transversal}, and write 
$$
C^1\stransv C^2\,\,\,,
$$
if $C^1\transv C^2$ and $C^1\cap C^2\neq\{0\}$.

\pparagraph{The transversal intersection 
theorem for the \mbox{classical} \mbox{differential}}
Let us assume that $Y$ is a normed real linear space, $S^1$ and $S^2$ are
subsets of $Y$, and $s_*\in S^1\cap S^2$.  We say that $S^1$ and $S^2$
are {\bf\em locally separated at $s_*$} if every neighborhood $U$ of
$s_*$ contains a point $s$ of $S^1\cap S^2$ such that $s\neq
s_*$.

We recall that if  $Y$ is a normed real linear space, then $Y^\dagger$ 
denotes the dual space of $Y$. 

\begin{definition}
{\rm Let $Y$ be a normed real linear space. Let $C$ be a convex cone
in $Y$. The {\bf\em polar} of $C$ is the set} 
$$
\quad\quad\qquad\qquad
C^\dagger\,\equaldef\,\{\lambda\in Y^\dagger:\lambda(c)\le 0\,\,\,\,
{\rm for\,\,\,\,all}\,\,\,\,c\in C\}\,.
\quad\qquad\qquad\qquad\quad\,\,\,\quad\square
$$
\end{definition}

\begin{definition}
Let $Y$ be a normed real linear space, and let 
$C^1$, $C^2$  be two convex cones in $Y$. We say that 
the cones $C^1$ and $C^2$  are {\bf\em linearly separated}
if \mbox{$(C^1)^\dagger\cap(-C_2)^\dagger\neq\{0\}$}.\proofend
\end{definition}

In other words, 

\begin{center}
\fbox{\fbox{\begin{minipage}{12cm}
{\bf\em Two convex cones  $C^1$ and $C^2$  are linearly separated
if and only if there exists a nontrivial continuous linear functional 
\mbox{$\lambda:Y\mapsto\rr$}
such that  $\langle\lambda,v\rangle\le 0$ for all $v\in C^1$ and
$\langle\lambda,v\rangle\ge 0$ for all $v\in C^2$.}
\end{minipage}}}\end{center}

\begin{proposition}\label{mystery}
Let $C^1$, $C^2$ be convex cones in a finite-dimensional 
real linear space $Y$. Then the following three conditions are equivalent:
\ben\item[]\ben\item[]\ben
\item[(\ref{mystery}.1)] $C^1\transv C^2$;
\item[(\ref{mystery}.2)] $C^1$ and $C^2$ are not linearly separated;
\item[(\ref{mystery}.3)] either
\ben
\item[(\ref{mystery}.3.a)] $C^1$ are $C^2$ are linear subspaces of $Y$ 
such that $Y=C^1\oplus C^2$, 
\een
or
\ben
\item[(\ref{mystery}.3.b)] $C^1\stransv C^2$.
\een\een\een\een
\end{proposition}
\begin{proof}
The equivalence of (\ref{mystery}.1)
and ((\ref{mystery}.2) is very easy to prove: if 
(\ref{mystery}.2) does not hold, then there  exists a  
$\lambda\in Y^\dagger\backslash\{0\}$ such 
that $\langle\lambda,c\rangle\le 0$ for all $c\in C^1$ and
$\langle\lambda,c\rangle\ge 0$ for all $c\in C^2$, so 
that $\langle\lambda,c\rangle\le 0$ for all $c\in C^1-C^2$
and then $C^1-C^2$ cannot be the \mbox{whole}
space because $\lambda$ is nontrivial, so (\ref{mystery}.1)
does not hold; conversely, if (\ref{mystery}.1) does not hold, then 
the convex cone $C=C^1-C^2$ is not the whole space, and  then 
the \mbox{closure} $\bar C\equaldef{\rm Clos}_Y(C)$ 
is not the \mbox{whole} space either, because
$Y$ is finite-dimensional; therefore the Hahn-Banach 
\mbox{theorem} 
implies the existence of a nontrivial linear 
functional $\lambda\in Y^\dagger$ which is nonpositive on $C^1-C^2$; 
then $\lambda$ is nonpositive
on $C^1$ and nonnegative on $C^2$, so 
\mbox{$\lambda\in(C^1)^\dagger\cap(-C^2)^\dagger$} and $\lambda\neq 0$, 
showing that  (\ref{mystery}.2) does  not hold.

The implication (\ref{mystery}.3)$\Rightarrow$(\ref{mystery}.2) is 
clearly trivial. 
To prove that (\ref{mystery}.2)$\Rightarrow$(\ref{mystery}.3), it suffices 
to show that if 
$C^1$ and $C^2$ are transversal but not strongly transversal 
then (a) holds. So assume that $C^1\transv C^2$ but 
$C^1$ and $C^2$ are not strongly transversal.
It then follows that $C^1\cap C^2=\{0\}$.  
We will show that
$C^1$ and $C^2$ are linear subspaces. This will then imply---since
$C^1-C^2=Y$ and $C^1\cap C^2=\{0\}$---that $C^1+C^2=Y$ 
and $C^1\cap C^2=\{0\}$, so $C^1\oplus C^2=Y$, establishing 
(\ref{mystery}.3.a). 

To prove that $C^1$ and $C^2$ are linear subspaces, we will  just do it for
$C^1$, the other case being similar. Since $C^1$ is a convex cone, it suffices
to show (cf. Fact \ref{fa-convcone-subspace}) that 
$$
c\in C^1\Longrightarrow -c\in C^1\qquad{\rm whenever}\quad c\in C\,.
$$

\noindent Pick $c\in C^1$. Using the fact that $C^1\transv C^2$, i.e., that
$C^1-C^2=X$, express $-c$ as a difference $-c=c^1-c^2$, $c^i\in C^i$.  Then
$c^1+c=c^2$. But $c^1+c\in C^1$ and $c^2\in C^2$. Since $C^1\cap
C^2=\{0\}$, we conclude that $c^1+c=0$. So $-c=c^1$, showing that 
$-c\in C^1$, and completing our proof. 
\end{proof}

\bigbreak
\begin{center}
\fbox{\fbox{
\begin{minipage}{12cm}
{\large
\begin{theorem}\label{theosepbasic-weak}
{\rm (The transversal intersection theorem for classical 
\mbox{differentials}).} Assume that $Y$  is a finite-dimensional
real linear space, $S^1$ and $S^2$ are subsets of $Y$, $s_*\in S^1\cap
S^2$, and $C^1$, $C^2$ are limiting \mbox{approximating} cones to $S^1$,
$S^2$ at $s_*$.  Assume that the cones $C^1$ and $C^2$ are strongly
transversal.  Then the sets $S^1$ and $S^2$ are not locally separated at $s_*$.
\end{theorem}}
\end{minipage}}}
\end{center}

\bigbreak
\begin{proof}
Let us write $\,\,C^1={\rm Clos}\,\bigcup_{j=1}^\infty C^1_j\,$,
$\,\,C^2={\rm Clos}\,\bigcup_{j=1}^\infty C^2_j\,$,
where each $C_j^i$ is an \mbox{approximating} cone
to $S^i$ at $s_*$ and $C_j^i\subseteq C_{j+1}^i$ for all $i,j$. 
Fix a basis $(e_1,\ldots,e_m)$ of $Y$, and 
then let $e_0=-(e_1+\ldots+e_m)$. Using the fact that $C_1\transv C_2$,
find $e_k^i$, for $i=1,2$, $k=0,\ldots,m$, such that 
$e_k^i\in C^i$ and $e_k^1-e_k^2=e_k$. Then find sequences
$\{e_{k,j}^i\}_{j=1}^\infty$, for each $i$ and each $k$,
such that $e_{k,j}^i\in C_j^i$ for every $i,j,k$, and 
$\lim_{j\to\infty}e_{k,j}^i=e_k^i$. Let 
$e_{k,j}=e_{k,j}^1-e_{k,j}^2$. Then 
$\lim_{j\to\infty}e_{k,j}=e_k$ for $k=0,\ldots,m$.  Since
$0$ is an interior point of the convex hull of 
$e_0,\ldots,e_m$, it is clear that there exists a $j^*$ such 
$0$ is an interior point of the convex hull $\Sigma_j$ of 
$e_{0,j},\ldots,e_{m,j}$ if $j\ge j^*$. Since $\Sigma_j
\subseteq C_j^1-C_j^2$, we see that $C_j^1\transv C_j^2$
if $j\ge j^*$. 

We now claim that 
\begin{itemize}
\item[(\&)] {\em There exists a $j^{\#}$ such that
$j^{\#}\ge j^*$ and $C_j^1\stransv C_j^2$
whenever $j\ge j^{\#}$.}  
\end{itemize}
If (\&) was not true, there would exist an infinite subset $J$ of
\mbox{$\{j\in\nn:j\ge j^*\}$} such that $C_j^1$ is not strongly transversal
to $C_j^2$ for $j\in J$. In view of \mbox{Proposition} \ref{mystery},
\mbox{since} $C_j^1\transv C_j^2$ whenever $j\in J$, 
we can conclude that for every $j\in J$ the cones $C_j^1$ and $C_j^2$
are in fact linear subspaces and $C_j^1\oplus C_j^2=Y$. The fact that the
sequences $\{C_j^1\}_{j=1}^\infty$, $\{C_j^2\}_{j=1}^\infty$, are
increasing implies that $\bigcup_{j=1}^\infty C^1_j=\bigcup_{j\in
J} C^1_j$ and \mbox{$\bigcup_{j=1}^\infty C^2_j=\bigcup_{j\in
J} C^2_j$}.  Furthermore, since for $i=1,2$ the families $\{C_j^i\}_{j\in
J}$ are increasing \mbox{sequences} of linear \mbox{subspaces},
and $Y$ is finite-dimensional, there exists a $j^\dagger$ such that
$\{C_j^i\}=C_{j^\dagger}^i$ \mbox{whenever} $j\in J$, $j\ge j^\dagger$,
$i=1,2$. But then 
$$
\bigcup_{j=1}^\infty C^1_j=\bigcup_{j\in J}C^1_j=C_{j^\dagger}^1
\qquad{\rm and}\qquad
\bigcup_{j=1}^\infty C^2_j=\bigcup_{j\in J}C^2_j=C_{j^\dagger}^2\,.
$$
It follows that $\bigcup_{j=1}^\infty C^1_j$ and $\bigcup_{j=1}^\infty
C^2_j$ are linear subspaces of $Y$. In particular, these two sets are
closed, and then $C^1=\bigcup_{j\in J}C^1_j=C_{j^\dagger}^1$ and
$C^2=\bigcup_{j\in J}C^1_j=C_{j^\dagger}^2$.  But then
$C_{j^\dagger}^1\stransv C_{j^\dagger}^2$, since $C^1\stransv
C^2$. This contradicts the fact that $C_{j^\dagger}^1\oplus
C_{j^\dagger}^2=Y$. This contradiction proves (\&).

If we pick any $j$ such that $j\ge j^\#$, and substitute
$C_j^1$, $C_j^2$ for $C^1$, $C^2$, then 
$C^1$, $C^2$ are approximating cones to $S^1$, $S^2$ 
at $s_*$,  and in addition $C^1\stransv C^2$. 
In other words, we have shown
that we may assume, without loss of generality, that 
\begin{itemize}
\item[(\&\&)] {\em 
$C^1$ and $C^2$ are approximating cones to $S^1$ and $S^2$ at $s_*$.}
\end{itemize}
From now on, we assume that (\&\&) holds, and also that 
$s_*=0$.   

Fix a neighborhood $V$ of $0$ in $Y$. 
For each $i\in\{1,2\}$, choose a normed \mbox{space} $X^i$, a 
\mbox{convex} cone
$D^i$ in $X^i$, a neighborhood $U^i$ of 
$0$ in $X^i$, a continuous map \mbox{$F^i:U^i\cap D^i\mapsto Y$},
and a continuous linear map $L^i:X^i\mapsto Y$, such that
$C^i=L\cdot D^i$, $F(0)=0$, $L\in\mathcal{D}F^i(0)$, 
and $F^i(U^i\cap D^i)\subseteq S^i$.
After shrinking the $U^i$, if necessary, we may assume
that $F^i(U^i\cap D^i)\subseteq V$ for $i=1,2$.

Using the fact that 
$C^1\stransv C^2$, 
choose a linear functional $\mu:Y\mapsto\rr$ 
and a vector $\bar y\in C^1\cap C^2$ such that $\mu(\bar y)>0$. Let
$\alpha=\mu(\bar y)$. 
Write $\bar y=L^1\bar v^1=L^2\bar v^2$, with $\bar v^1\in D^1$,
$\bar v^2\in D^2$. Let $U=U^1\times U^2$, 
$D=D^1\times D^2$. Define a map $F:U\cap D\mapsto Y\times\rr$ by letting 
$$
F(x^1,x^2)=(F^1(x^1)-F^2(x^2),\mu(F^1(x^1)))\,\,\,.{\rm if}
\,\,\,x^1\in U^1\cap D^1\,,\,\,x^2\in U^2\cap D^2\,.
$$
Then $F$ is clearly continuous.  A simple calculation
shows that  $F$ is differentiable at $(0,0)$, and a differential 
of $F$ at $(0,0)$ is the linear map $L:X^1\times X^2\mapsto Y\times\rr$
given by 
$$
L\cdot(v^1,v^2)=(L^1\cdot v^1-L^2\cdot v^2,\mu(L^1\cdot v^1))
$$
for $(v^1,v^2)\in X^1\times X^2$. Given any $k\in\{0,\ldots,m\}$, 
the fact that $C^1\transv C^2$ implies that 
we can find $(\tilde v_k^1,\tilde v_k^2)\in D$ such that 
$L^1\cdot\tilde  v_k^1-L^2\cdot\tilde  v_k^2=e_k$. Then  
$L\cdot(\tilde v_k^1,\tilde v_k^2)=(e_k,r_k)$ for some 
$r_k\in\rr$.  If $r\ge r$, then we can find a pair
$(v_k^1(r),v_k^2(r))\in D$ such that 
$L\cdot(v_k^1(r),v_k^2(r))=(e_k,r)$ by letting 
$$
v_k^1(r)=\tilde v_k^1+\beta\bar v_1\,,
\qquad v_k^2=\tilde v_k^2+\beta\bar v_2\,,\qquad
\beta=\frac{r-r_k}{\alpha}\,.
$$
Let $\bar =\max(r_0,\ldots,r_m)$. 
Then we have shown that the vector $(e_k,r)$ belongs to 
$L\cdot D$ if $k\in\{0,\ldots.m\}$ and $r\ge\bar r$. Let 
$\Sigma$ be the convex hull of the set $\{e_0,\ldots,e_m\}$.
Then $\Sigma\times[\bar r,\infty\,[$ is a subset of $L\cdot D$. In 
particular,
this implies that the vector $(0,\bar r+1)$  belongs to
the interior ${\rm Int}(L\cdot D)$ of $L\cdot D$. Therefore 
$(0,1)\in{\rm Int}(L\cdot D)$ as well, and then 
Theorem  \ref{theoopendirclass} implies that 
there exists a positive number  $\rho$ such  that 
\mbox{$(0,r)\in F(U\cap D)$} whenever $0\le r\le\rho$. Pick any $r$ such 
that $0<r\le\rho$. Then \mbox{$(0,r)\in F(U\cap D)$}, so 
there exist points $x^1\in U^1\cap D^1$, $x^2\in U^2\cap D^2$, such that 
\mbox{$F^1(x^1)-F^2(x^2)=0$} and $\mu(F^1(x^1))=r$.  If we let 
\mbox{$y=F^1(x^1)=F^2(x^2)$}, then \mbox{$y\in S^1\cap S^2$}.
Since $\mu(y)=r\neq 0$, $y$ cannot be the origin of $Y$.
Since $y$ belongs to\mbox{$F^1(U^1\cap D^1)\subseteq V$}, $y$ satisfies
$y\in V$.  Since $V$ is arbitrary, we have shown that 
$S^1\cap S^2$ contains nonzero points arbitrarily close  to the origin,
which is the desired conclusion. 
\end{proof}


\vbox{\pparagraph{The linear separation
theorem for the \mbox{classical} differential}

\bigbreak
\begin{center}
\fbox{\fbox{
\begin{minipage}{12cm}
{\large
\begin{theorem}\label{theosepbasic-dual}
{\rm (The linear separation theorem for the
classical  \mbox{differential}).} Let $Y$ be a finite-dimensional real 
normed space.
Let $S^1$, $S^2$ be subsets of $Y$, let $s_*\in S^1\cap S^2$,
and let $C^1$, $C^2$ be limiting 
\mbox{approximating} cones 
to $S^1$, $S^2$ at $s_*$. Assume that the sets $S^1,S^2$ are 
locally separated at $s_*$. Then either
\ben
\item[(a)] $C^1$ and $C^2$ are linear subspaces and $Y=C^1\oplus C^2$
\een
or 
\ben	
\item[(b)] $C^1$ and $C^2$ are linearly separated.
\een
\end{theorem}} 
\end{minipage}}}
\end{center}

}

\begin{proof}
Assume that both (a) and (b) are false. Then Proposition 
\ref{mystery} implies that either (a) holds or $C^1\stransv C^2$.
But we are assuming that (a) is false. So $C^1\stransv C^2$.
Then Theorem \ref{theosepbasic-dual}
tells us that $S^1$ and $S^2$ are not
locally separated at $s_*$, and we have reached a contradiction.
\end{proof}

\pparagraph{Compatible sets of variational vectors}
If $V$ is a subset of a real linear space $Y$, we use 
$convcone_Y(V)$ to denote the smallest convex cone in $Y$ that contains
all the members of $V$.  

\begin{definition}
{\rm Let $Y$ be a normed space, let $S$ be a subset of 
$Y$, let $s_*\in S$, and let $v\in Y$.
We say that $v$ is a {\bf\em variational vector}
for $S$ at $s_*$ if there exist a positive number $\eps$ and a
continuous map $\varphi:[0,\eps]\mapsto S$ such that
$\lim_{t\downarrow 0}\Big(\frac{\varphi(t)-s_*-tv}{t}\Big)=0$}.\proofend
\end{definition}

We use $VV_{s_*}S$ to denote the set of all variational vectors
for $S$ at $s_*$. It is easy to see that {\em if $s_*\in S$ then the set 
$VV_{s_*}S$ is always a cone in $Y$}. 

\begin{definition}
{\rm Let $Y$ be a normed space, let $S$ be a subset of 
$Y$, let $s_*\in S$, and let $\mathcal{V}\subseteq Y$.
We say that $\mathcal{V}$ is a {\bf\em compatible set
of variational vectors for $S$ at $s_*$} if 
for every finite subset $\mathcal{V}_0$ of 
$\mathcal{V}$ the convex cone $convcone_Y(\mathcal{V}_0)$ 
is an approximating cone for $S$ at $s_*$.}\proofend
\end{definition}

It is easy to see that 
\bfact
If $\mathcal{V}$ is a compatible set of variational vectors for $S$ at 
$s_*$ then every $v\in\mathcal{V}$ is a variational vector for $S$ at
$s_*$.\proofend
\efact

The converse, however, is clearly not true in general. Indeed, if
$\mathcal{V}$ is a set of variational vectors for $S$ at $s_*$ then it does
not follow that every convex combination of members of $\mathcal{V}$ is in
$VV_{s_*}(S)$. (For example, if $S$ is the union of the $x$ axis 
and the $y$ axis in $\rr^2$, and $\mathcal{V}$ consists 
of the vectors $v=(1,0)$ and $w=(0,1)$, then 
$\mathcal{V}\subseteq VV_{s_*}(S)$, but the vector 
$\frac{1}{2}(v+w)$ is not in $VV_{s_*}(S)$.) On the other hand,
for $\mathcal{V}$ to be compatible it is necessary 
that every convex combination of members of $\mathcal{V}$ be
in $VV_{s_*}(S)$. 

\begin{proposition}
Let $Y$ be a finite-dimensional real normed space, 
let $S$ be a subset of  $Y$, and let $s_*\in S$.
Let $\mathcal{V}$ be a compatible set
of variational vectors for $S$ at $s_*$. Then the 
cone ${\rm Clos}\Big(convcone_Y(\mathcal{V})\Big)$ is
a limiting approximating cone for $S$ at $s_*$.
\end{proposition}

\begin{proof}
Let  $C={\rm Clos}\Big(convcone_Y(\mathcal{V})\Big)$.  Let 
$\smath_Y=\{y\in Y:\|y\|=1\}$, so $\smath_Y$ is a compact subset
of $Y$. Let $K=C\cap\smath_Y$, so $K$ is compact as well.

For each $j\in\nn$, let $F_j$ be a finite subset 
of $K$ such that the balls $\bmath_Y(x,2^{-j-1})$, 
$x\in F_j$, cover $K$. Then pick, for each $x\in F_j$, a vector
$v_{j,x}\in convcone_Y(\mathcal{V})$ such that 
$\|v_{j,x}-x\|\le 2^{-j-1}$. Find, for each $j$ and each $x$, 
a finite subset $E(j,x)$ of $\mathcal{V}$ such that 
$v_{j,x}$ is a linear combination of the members of 
$E(j,x)$ with positive coefficients. Let 
$E_j=\bigcup_{k=1}^j\bigcup_{x\in F_k}E(k,x)$. Let 
$C_j=convcone_Y(E_j)$. Then  the $C_j$ form an increasing
sequence of convex cones. Clearly, $C_j\subseteq C$ for all $j$. 
Furthermore, if $x\in C$ and $j\in\nn$, we can find 
$x'\in F_j$ such that $\|x-x'\|\le 2^{-j-1}$,
and then $\|x-v_{j,x'}\|\le 2^{-j}$ 
and $v_{j,x'}\in C_j$. Hence 
$C={\rm Clos}\,\bigcup_{j=1}^\infty C_j$. 
Finally, each $C_j$ is an approximating cone to 
$S$ at $s_*$, because $C_j$ is the smallest convex cone in $Y$
containing the finite subset $E_j$ of 
$\mathcal{V}$.  Hence ${\rm Clos}\Big(convcone_Y(\mathcal{V})\Big)$ is
a limiting approximating cone for $S$ at $s_*$.\proofend
\end{proof}

\subsection{Classical control systems} 

\pparagraph{Piecewise continuous controls} 
We recall that the concept of a ``$U$-control'' was \mbox{introduced} in 
Definition \ref{ucontrol} in page \pageref{ucontrol}.

If $U$ is a metric space and $I=[a,b]$ is a nonempty
compact interval, then a map
$\eta:I\mapsto M$ is {\bf\em piecewise continuous} if 
there exists a finite subset $F$ of the interior of $I$
such that $\eta$ is continuous at every point $t$ of $I\backslash F$,
and has left and right limits $\eta(t-)$, $\eta(t+)$, at every point 
$t\in F$.

We use $C^{0,pc}(I,U)$  to denote the set of all piecewise
continuous $U$-controls defined on $I$.

The following trivial observation will be important later.

\begin{center}
\fbox{\begin{minipage}{11cm} 
{\em The closure of the graph ${\rm Gr}(\eta)=\{(\eta(t),t):
t\in I\}$ of a piecewise continuous
$U$-control $\eta:I\mapsto U$ is a compact subset of $U\times\rr$.}
\end{minipage}}.
\end{center}

\pparagraph{Classical control systems} 

\begin{definition}\label{shoddy}
{\rm Let $n$ be a positive integer. A {\bf\em classical $n$-dimensional 
\mbox{control} system} is a $5$-tuple $\Sigma=(\Omega,U,I,f,\mathcal{U})$
such that 
\begin{itemize}\item[]\begin{itemize}\item[]\begin{itemize}
\item[(\ref{shoddy}-1)] {\em $\Omega$ is an open subset of $\rr^n$,}  
\item[(\ref{shoddy}-2)] {\em $U$ is a set,} 
\item[(\ref{shoddy}-3)] {\em $I$ is a nonempty subinterval of
$\rr$,} 
\item[(\ref{shoddy}-4)] {\em $f$ is a map
from $\Omega\times U\times I$ to $\rr^n$,}
\item[(\ref{shoddy}-3)] {\em $\mathcal{U}$ is a set of $U$-controls
such that $\dom(\eta)\subseteq I$ for every $\eta\in\mathcal{U}$.}\proofend
\end{itemize}\end{itemize}\end{itemize}}
\end{definition}

\begin{definition}\label{isck}
{\rm If $n\in\nn$ and $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a classical 
$n$-dimensional control system, we say that $\Sigma$ is 
\begin{itemize}
\item[$\bullet$] 
{\bf\em of class $C^k$}---for a $k\in\zz_+$---if
the map $\Omega\ni x\mapsto f(x,u,t)\in X$ is 
of class $C^k$ for every $(u,t)\in U\times I$;
\item[$\bullet$] 
{\bf\em of class $C^{0,0}$}  if $U$ is a metric space and
$f$ is a continuous map
from $\Omega\times U\times I$ to $\rr^n$;
\item[$\bullet$] 
{\bf\em of class $C^{0,0,pc}$}  if it is of class $C^{0,0}$ and 
in addition every $\eta\in\mathcal{U}$ is piecewise continuous;
\item[$\bullet$] {\bf\em of class $C^{0,0,0}$}  if it is of class 
$C^{0,0}$ and in addition every $\eta\in\mathcal{U}$ is continuous;
\item[$\bullet$] {\bf\em of class $C^{1,0}$}  if it is of class
$C^1$ and of class $C^{0,0}$, and the map
$$
\Omega\times U\times I\ni(x,u,t)\mapsto Df(x,u,t)\in\rr^{n\times n}
$$
is continuous, where
\begin{itemize}
\item[(i)] $\rr^{n\times n}$ is the space of $n\times n$ real 
matrices, identified in the standard way with the space
of linear maps from $\rr^n$ to $\rr^n$, 
\item[(ii)] if $x_*\in\Omega$, $u_*\in U$, and $t_*\in I$, then
$Df(x_*,u_*,t_*)$ is the differential of $f$ at 
$(x_*,u_*,t_*)$  with respect
to $x$ (that is, the matrix $L\in\rr^{n\times n}$ having
the property that
\mbox{$f(x_*+h,u_*,t_*)-f(x_*,u_*,t_*)-L\cdot h=o(\|h\|)$ as $h\to 0$});
\end{itemize}.
\item[$\bullet$] {\bf\em of class $C^{1,0,pc}$}  if it is of class
$C^{1,0}$ and $C^{0,0,pc}$;
\item[$\bullet$] {\bf\em of class $C^{1,0,0}$}  if it is of class
$C^{1,0}$ and $C^{0,0,0}$. \proofend
\end{itemize}}
\end{definition}

\pparagraph{Trajectories} We introduced the concept of
`` $\Sigma$-trajectory'' in Definition \ref{remonfull} 
(page \pageref{remonfull})
for general control systems  $\Sigma$. For classical $n$-dimensional control 
\mbox{systems}  $\Sigma$ of class $C^{0,0,pc}$, the definition of 
`` $\Sigma$-trajectory'' can be reformulated without using the notions of 
local absolute continuity and equality almost everywhere, thanks to the 
following trivial fact:
\bfact\label{facttrivo}
If $\xi$, $\zeta$ are $\rr^n$-valued functions on a compact interval 
$[a,b]$ such that $\xi$ is continuous and $\zeta$ is piecewise continuous,
then the following two \mbox{conditions} are equivalent:
\begin{itemize}\item[]\begin{itemize}\item[]\begin{itemize}
\item[(\ref{facttrivo}.1)] the derivative $\dot\xi(t)$ of $\xi$ at 
$t$ exists and is equal to $\zeta(t)$ for all but finitely many 
$t\in[a,b]$;
\item[(\ref{facttrivo}.2)] $\xi$ is absolutely continunous 
and $\,\,\,\dot\xi(t)=\zeta(t)\,\,\,$ for almost every 
\mbox{$t\in[a,b]$}.\proofend
\end{itemize}\end{itemize}\end{itemize}
\efact


\begin{definition}\label{shoddy-b}
{\rm Assume that $n$ is a nonegative integer,
$\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a 
classical \mbox{$n$-dimensional} control system of class $C^{0,0,pc}$, 
and $\eta\in\mathcal{U}$.  
\begin{enumerate} 
\item[$\bullet$] A {\bf\em trajectory}
for $\eta$ is a continuous map $\xi:I_\xi\mapsto\Omega$,
defined on a nonempty subinterval $I_\xi$ of
$\dom(\eta)$,  such that the
derivative $\dot\xi(t)$ exists and is equal to $f(\xi(t),\eta(t),t)$
for all $t\in I_\xi$ that are points of continuity of $\eta$.
\item[$\bullet$] A {\bf\em full trajectory} for $\eta$ is a trajectory
for $\eta$ whose domain is the entire interval $\dom(\eta)$.
\item[$\bullet$] A {\bf\em trajectory-control pair} for $\Sigma$ is a pair 
$\gamma=(\xi,\eta)$ such that $\eta\in\mathcal{U}$ and $\xi$ is a full 
trajectory for $\eta$.
\item[$\bullet$] We use $TCP(\Sigma)$ to denote the set of all 
trajectory-control pairs $(\xi,\eta)$ for the system $\Sigma$.\proofend
\end{enumerate}}
\end{definition}

\begin{remark}
{\rm Assume that $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a classical
$n$-dimensional control system  of class $C^{0,0,pc}$, 
and $\eta\in\mathcal{U}$.  If $\xi:I_\xi\mapsto\Omega$ is a 
continuous map, then the map $I_\xi\ni t\mapsto f(\xi(t),\eta(t),t)$
is piecewise continuous. If $\xi$ satisfies the integral equation
\begin{equation}\label{inteeq}
\xi(t)-\xi(s)=\int_s^tf(\xi(r),\eta(r),r)dr\quad
{\rm whenever}\quad s,t\in I_\xi\,,
\end{equation}
(where the right-hand side is understood as a Riemann integral)
then $\dot\xi(t)$ exists and equals $f(\xi(t),\eta(t),t)$ at every
$t\in I_\xi$ which is a point of continuity of $\eta$.
Hence $\xi$ is a trajectory for $\eta$. Conversely, if $\xi$ is a 
trajectory for 
$\eta$, then $\xi$ satisfies (\ref{inteeq}). (Indeed, if
we fix $s$, and define $\zeta(t)=\xi(s)+\int_s^tf(\xi(r),\eta(r),r)dr$
for $t\in I_\xi$, then the function $\xi-\zeta$ is continuous
on $I_\xi$, and its derivative exists and  is equal 
to $\dot\xi(t)-f(\xi(t),\eta(t),t)$---i.e., to $0$---at every 
$t\in I_\xi$ which is a point of
continuity of $\eta$. Hence $\xi-\zeta$ is constant 
on every subinterval of $I_\xi$ where $\eta$ is continuous.
Since $\xi-\zeta$ is continuous, and $\eta$ is piecewise
continuous, $\xi-\zeta$ is in fact constant on $I_\xi$,
so $\xi\equiv\zeta$, because $\xi(s)=\zeta(s)$.)

It follows that {\em a continuous map 
$\xi:I_\xi\mapsto\Omega$ defined on a nonempty subinterval 
$I_\xi$ of $\,\,\dom(\eta)$ is a trajectory for the
control $\eta\in\mathcal{U}$ if and only if 
$\xi$ satisfies equation (\ref{inteeq}).}}\proofend
\end{remark}

\begin{remark}\label{existconstants}
{\rm If $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a classical $n$-dimensional
control system of class $C^{0,0}$, then for every compact subset
$K$ of $\Omega\times U\times I$ there exists a constant
$c^0(K)$ such that $\|f(x,u,t)\|\le c^0(K)$ whenever 
$(x,u,t)\in K$. If in addition $f$ is of class $C^{1,0}$, then
there exists a constant
$c^1(K)$ such that $\|Df(x,u,t)\|\le c^1(K)$ whenever 
$(x,u,t)$ belongs to $K$.} \proofend
\end{remark}

\begin{proposition} \label{localexi} (Local existence of 
trajectories.) Let 
\mbox{$\Sigma=(\Omega,U,I,f,\mathcal{U})$} be a classical $n$-dimensional
control system which is of class $C^{1,0,pc}$. Then for every 
$\eta\in\mathcal{U}$ and  every 
point $(\bar x,\bar t)\in\Omega\times\dom(\eta)$ there exist
a positive number $\eps$ and a trajectory $\xi$ for $\eta$ such that
$\xi(\bar t)=\bar x$ and $\xi$ is defined on  the interval
$J_{\eta,\bar t,\eps}$, where
$$
J_{\eta,\bar t,\eps}\,\equaldef\,\dom(\eta)\cap[\bar t-\eps,\bar t+\eps]\,,
$$
\end{proposition}

\begin{proof}
We fix $\eta$ and $\bar t$, and write 
$J_\eps$ for $J_{\eta,\bar t,\bar\eps}$ is compact. 
Pick positive numbers $\bar\delta$, $\bar\eps$ such that the ball
$B\equaldef\bar\bmath^n(\bar x,\bar\delta)$ is contained in $\Omega$
and the set $J_{\bar\eps}$ is compact.  (This is possible,
since $\bar t\in\dom(\eta)$.) Let $D$ be the closure of the set
$\{\eta(t):t\in J_{\bar\eps}\}$. Then $D$ is a compact
subset of $U$.  Let $K=B\times D\times J_{\bar\eps}$.  Let
$c^0(K)$, $c^1(K)$ be the constants defined in Remark
\ref{existconstants}.

For each $\eps$ such that $0<\eps\le\bar\eps$, let 
$M_{\eps}$ be the space of all continuous functions 
$\xi:J_{\eps}\mapsto B$ 
such that $\xi(\bar t)=\bar x$. Define a map $\Phi_{\eps}$ on 
$M_{\eps}$ as follows: if $\xi\in M_{\eps}$,
then $\Phi_{\eps}(\xi)$ is the $\rr^n$-valued function 
on $J_\eps$ given by 
$$
\Phi_{\eps}(\xi)(t)=\bar x+\int_{\bar t}^tf(\xi(s),\eta(s),s)\,ds
\qquad{\rm for}\qquad t\in J_\eps\,.
$$
Clearly, if $\xi\in M_\eps$ then
$\Phi_{\eps}(\xi)$ is a continuous function
such that \mbox{$\Phi_{\eps}(\xi)(\bar t)=\bar x$}.
Furthermore, the bound
\mbox{$\|\Phi_{\eps}(\xi)(t)-\bar x\|\le\eps c^0(K)$} clearly holds, 
implying that $\Phi_{\eps}$ takes \mbox{values} in $M_{\eps}$ 
as long as $\eps c^0(K)\le\bar\delta$. 

Define the distance $d(\xi,\zeta)$ between two members
$\xi,\zeta$ of $M_{\eps}$ by letting 
$$
d(\xi,\zeta)=\max\{\|\xi(t)-\zeta(t)\|:t\in J_\eps\}\,.
$$ 
It is then clear that $M_{\eps}$ is a complete
metric space. 

If $\xi,\zeta$ belong to $M_{\eps}$, and $t\in J_\eps$, 
then
$$
\Phi_{\eps}(\xi)(t)-\Phi_{\eps}(\zeta)(t)=
\int_{\bar t}^t\Big(f(\xi(s),\eta(s),s)-f(\zeta(s),\eta(s),s)\Big)\,ds
$$
so 
$$
\|\Phi_{\eps}(\xi)(t)-\Phi_{\eps}(\zeta)(t)\|\le\eps c^1(K)d(\xi,\zeta)\,,
$$
since 
\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
f(\xi(s),\eta(s),s)-f(\zeta(s),\eta(s),s)=\\
&\,&\Big(\int_0^1Df(\zeta(s)+h(\xi(s)-\zeta(s)),\eta(s),s)\,dh\Big)\cdot
\Big(\xi(s)-\zeta(s)\Big)\,.
\end{eqnarray*}
Hence $d(\Phi_{\eps}\xi,\Phi_{\eps}\zeta)\le
\eps c^1(K)d(\xi,\zeta)$ for all $\xi,\zeta\in M_{\eps}$.

It follows that, if we choose $\eps$ such that 
$\eps c^0(K)\le\bar\delta$ and $\eps c^1(K)<1$, then $\Phi_\eps$ 
is a contraction map from the complete metric space $M_\eps$ to
itself.  Hence $\Phi_\eps$ has a fixed point by Theorem 
\ref{the-contraction}. It is then clear that
this fixed point is a trajectory of $\eta_{\bar t,\eps}$.\proofend
\end{proof}

\begin{remark}
{\rm The above result is also valid if $\Sigma$ is just of class 
$C^{0,0,pc}$.  In
that case, the proof using the contraction mapping principle does not
work, and must be replaced by a more sophisticated
argument, using a different fixed point theorem. Instead of
$M_\eps$, we work in the space $N_\eps$
of all Lipschitz-continuous maps
$\xi:J_\eps\mapsto B$ such that $\xi(\bar t)=\bar x$ and
$\|\xi(t)-\xi(s)\|\le c^0(K)|t-s|$ for all $s,t\in J_\eps$.
Then $N_\eps$ is a compact convex subset of the Banach space
$C^0(J_\eps,\rr^)$ of all continuous functions from $J_\eps$ to
$\rr^n$. The map $\Phi_\eps$ constructed above maps
$N_\eps$ to $N_\eps$ if $\eps c^0(K)\le\bar\delta$.
Furthermore, $\Phi_\eps$ is continuous. Hence 
$\Phi_\eps$ has a fixed point by Schauder's
fixed point theorem.} \proofend
\end{remark}

In the following statements, we write
$$
s\wedge t=\min(s,t)\,,\,\,\,\qquad s\vee t=\max(s,t)\,.
$$

\begin{proposition} \label{gronweq} (Gronwall's inequality.)
Assume that $\alpha$ and $\beta$ are 
\mbox{nonnegative} \mbox{continuous} functions
on an interval $[a,b]$, $c$ is a nonnegative constant,
and $s\in[a,b]$ is such that 
$$
\alpha(t)\le c+\int_{s\wedge t}^{s\vee t}\beta(r)\alpha(r)\,dr
\,\,\,{\rm for\,\,all}\,\,t\in[a,b]\,.
$$
Then
$$
\alpha(t)\le c e^{\int_{s\wedge t}^{s\vee t}\beta(r)dr}
\,\,\,{\rm for\,\,all}\,\,t\in[a,b]\,.
$$
\end{proposition}
\begin{proof}
Define 
$$
\psi(t)=\Big(c+\int_{s\wedge t}^{s\vee t}
\beta(r)\alpha(r)\,dr\Big)e^{-\int_{s\wedge t}^{s\vee t}\beta(r)dr}\,.
$$
Then, if $t\ge s$, we have 
$$
\psi(t)=\Big(c+\int_{s}^{t}
\beta(r)\alpha(r)\,dr\Big)e^{-\int_{s}^{t}
\beta(r)dr}\,, 
$$
so
\begin{eqnarray*}
\dot\psi(t)&=&\beta(t)\alpha(t)e^{-\int_s^t\beta(r)dr}
-\beta(t)\Big(c+\int_s^t\beta(r)\alpha(r)\,dr\Big)e^{-\int_s^t\beta(r)dr}\\
&=&\beta(t)e^{-\int_s^t\beta(r)dr}\Big(\alpha(t)-c-\int_s^t\beta(r)\alpha(r)\,dr\Big)\\
&\le& 0\,.
\end{eqnarray*}
If $t\le s$, then
$$
\psi(t)=\Big(c+\int_{t}^{s}
\beta(r)\alpha(r)\,dr\Big)e^{-\int_{t}^{s}\beta(r)dr}\,, 
$$
so
\begin{eqnarray*}
\dot\psi(t)&=&-\beta(t)\alpha(t)e^{-\int_t^s\beta(r)dr}
+\beta(t)\Big(c+\int_t^s\beta(r)\alpha(r)\,dr\Big)e^{-\int_t^s\beta(r)dr}\\
&=&\beta(t)e^{-\int_s^t\beta(r)dr}
\Big(-\alpha(t)+c+\int_t^s\beta(r)\alpha(r)\,dr\Big)\\
&\ge& 0\,.
\end{eqnarray*}
It follows that $\psi(t)\le\psi(s)$ for all $t$. Since 
$\psi(s)=c$, we have 
$$
\alpha(t)\le c+\int_{s\wedge t}^{s\vee t}\beta(r)\alpha(r)\,dr
\le ce^{\int_{s\wedge t}^{s\vee t}\beta(r)dr}\quad{\rm for\,\,all}
\,\,t\in[a,b]\,,
$$
completing our proof.
\end{proof}

\pparagraph{Continuity and convergence of controls}
We would like to define what it means for a parametrized family of
controls $\{\eta_p\}_{p\in P}$---where $P$ is a metric space---to
``depend continuously on $p$.'' As usual, it suffices to define
what it means for a sequence $\{\eta_j\}_{j\in\nn}$
of \mbox{controls} to converge to a control $\eta$. 

\newcommand{\cowarningdist}{\begin{minipage}{6cm}

$\qquad$Recall that if $U$ is a metric 
space, $a,b\in\rr$, and $a\le b$,
then $C^{0,pc}([a,b],U)$ is the set of all piecewise continuous
maps $\eta:[a,b]\mapsto U$.  

$\qquad$If $U$ is a metric space and $X$ is a normed space, we use
$C^0(U,X)$ to denote $\,$the $\,$set $\,$of $\,$all $\,$continuous 


\end{minipage}}

\newcommand{\warningdist}{\fbox{\begin{minipage}{6cm}{\bf WARNING:} The 
``$\varphi$-distance'' $d_\varphi$ really is a {\bf\em pseudodistance}
on $C^{0,pc}([a,b],U)$, rather than a distance, because 
it is not necessarily true that $d_\varphi(\eta_1,\eta_2)=0$ implies 
$\eta_1=\eta_2$.
\end{minipage}}}


%If $\eta_1,\eta_2$ are functions defined on an interval
%$I$, and $M\ge 0$, we will write $\eta_1\sim_M\eta_2$ 
%$\eta_1(t)=\eta_2(t)$ for all $t\in I\backslash E$, 
%for some subset $E$ of $I$ which is a finite union $E_1\cup\cdots\cup E_m$
%of intervals whose lengths $|E_k|$ satisfy 
%$\sum_{k=1}^m|E_k|\le M$. 

\noindent\cowarningdist$\quad$\warningdist

\noindent maps \mbox{$\varphi:U\mapsto X$}, and refer to the members of
$C^0(U,X)$ as {\bf\em $X$-valued test functions on $U$}. 

If we are given two controls $\eta_1,\eta_2$ belonging to 
$C^{0,pc}([a,b],U)$, and a test function $\varphi\in  C^0(U\times[a,b],X)$,
we can define the ``$\varphi$-distance'' $d_\varphi(\eta_1,\eta_2)$
to be the number
$$
d_\varphi(\eta_1,\eta_2)\equaldef\sup\Biggl\{\Biggl\|\int_a^t\Big(
\varphi(\eta_1(s),s)-\varphi(\eta_2(s),s)\Big)\,ds
\Biggr\|:a\le t\le b\Biggr\}\,.
$$


\begin{definition} \label{convedef} 
{\rm Assume that  $U$ is a metric space, $a,b\in\rr$, $a\le b$,
$\{\eta_j\}_{j\in J}$ is a sequence in $C^{0,pc}([a,b],U)$,
and $\eta\in C^{0,pc}([a,b],U)$. We say that the sequence
$\{\eta_j\}_{j\in J}$ {\bf\em c-converges} to $\eta$
(and write $\eta_j\to_c\eta$, or $\eta=c-\lim_{j\to\infty,j\in J}\eta_j$)
if 
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{convedef}.1)] there exists a compact subset 
$U_0$ of $U$ such that $\eta_j([a,b])\subseteq U_0$ for all 
$j\in J$,
\end{itemize}\end{itemize}
and
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{convedef}.2)] 
$\,\lim_{j\to\infty,j\in J}d_\varphi(\eta_j,\eta)=0\,\,$ for every continuous 
real-valued \mbox{function}
\mbox{$\varphi\in  C^0(U\times[a,b],\rr)$}.\proofend
\end{itemize}\end{itemize}}
\end{definition}

\begin{definition} \label{convedefpara} 
{\rm Assume that  $U$ and $P$ are metric spaces, $a,b\in\rr$, $a\le b$,
and $\{\eta_p\}_{p\in P}$ is a family of members of $C^{0,pc}([a,b],U)$.
We say that the family $\{\eta_p\}_{p\in P}$ {\bf\em depends 
c-continuously on the parameter $p$} if whenever 
$\{p_j\}_{j\in\nn}$ is a sequence of points of $P$ that converges to a 
$p\in P$ it follows that $\eta_{p_j}\to_c\eta_p$.}\proofend
\end{definition}

\begin{remark} \label{remonconvede} 
{\rm If a sequence $\{\eta_j\}_{j\in J}$ in $C^{0,pc}([a,b],U)$
c-converges to a \mbox{limit} $\eta\in C^{0,pc}([a,b],U)$, then it is 
clear that $d_\varphi(\eta_j,\eta)\to 0$ as $j\to_J\infty$ 
not only for all real-valued test 
functions \mbox{$\varphi\in  C^0(U\times[a,b],\rr)$}, but also for all
vector-valued test functions \mbox{$\varphi\in  C^0(U\times[a,b],\rr^m)$}, 
for every $m$, since in that case $\varphi$ has
real-valued components $\varphi^1,\ldots,\varphi^m$, so
$d_\varphi(\eta_j,\eta)\to 0$ if and only if
$d_{\varphi^k}(\eta_j,\eta)\to 0$ for $k=1,\ldots,m$.}\proofend 
\end{remark}

If $n\in\nn$ and $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is an
$n$-dimensional classical control system of class $C^{0,0}$, $a,b\in I$,
$a\le b$, and $\xi:[a,b]\mapsto\Omega$ is continuous, then 
we can define an $\rr^n$-valued test function 
$\varphi_{f,\xi}:U\times[a,b]\mapsto\rr^n$ by letting
$$
\varphi_{f,\xi}(u,t)=f(\xi(t),u,t)\,\,\,{\rm for}\,\,\,
u\in U\,,\,\,t\in[a,b]\,.
$$

\begin{proposition} \label{estimatix} 
Assume that  $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a classical
$n$-dimensional  control system of class $C^{1,0,pc}$. 
Let $a,b\in I$ be such that $a\le b$.
Let $Q$ be a compact convex subset of $\Omega$, and let 
$U_0$ be a compact subset of $U$. Let $K=Q\times U_0\times[a,b]$, so
$K$ is a compact subset of $\Omega\times U\times I$. Let 
$c^0(K)$, $c^1(K)$ be the
constants defined in Remark \ref{existconstants}. 
Assume that $\eta_1,\eta_2$ are members of $\,\mathcal{U}$
that are defined on $[a,b]$, and $\xi_1,\xi_2$ are
$Q$-valued full trajectories for $\eta_1,\eta_2$. Then the estimate
\begin{eqnarray}\label{concise}
&\,&\!\!\!\!\!\!\! \|\xi_1(t)-\xi_2(t)\|\le\\
&\,&\,\,\,e^{|t-s|c^1(K)}\Biggl(\|\xi_1(s)-\xi_2(s)\|+
2\min\Big(d_{\varphi_{f,\xi_1}}(\eta_1,\eta_2),
d_{\varphi_{f,\xi_2}}(\eta_1,\eta_2)\Big)\Biggr)\nonumber
\end{eqnarray}
holds for all $s,t\in[a,b]$.
\end{proposition}

\begin{proof}
We write
\begin{eqnarray*}
\xi_1(t)-\xi_2(t)&=&\xi_1(s)-\xi_2(s)+
\int_{s}^t\Big(f(\xi_1(r),\eta_1(r),r)-f(\xi_2(r),\eta_2(r),r)\Big)\,dr\\
&=&\xi_1(s)-\xi_2(s)+
\int_{s}^t\Big(f(\xi_1(r),\eta_1(r),r)-f(\xi_1(r),\eta_2(r),r)\Big)\,dr\\
&\,&\phantom{\xi_1(s)-\xi_2(s)}
+\int_{s}^t\Big(f(\xi_1(r),\eta_2(r),r)-f(\xi_2(r),\eta_2(r),r)\Big)\,dr\,.
\end{eqnarray*}
Clearly, 
$$
\Big\|\int_{s}^t\Big(f(\xi_1(r),\eta_1(r),r)-f(\xi_1(r),\eta_2(r),r)\Big)
\,dr\Big\|\le 2d_{\varphi_{f,\xi_1}}(\eta_1,\eta_2),
$$
and 
$$
\Big\|f(\xi_1(r),\eta_2(r),r)-f(\xi_2(r),\eta_2(r),r)\Big\|
\le c^1(K)\|\xi_1(r)-\xi_2(r)\|\,,
$$
because 
\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
f(\xi_1(r),\eta_2(r),r)-f(\xi_2(r),\eta_2(r),r)=\\
&\,&\Big(\int_0^1Df(\xi_2(r)+h(\xi_1(r)-\xi_2(r)),\eta_2(r),r)\,dh\Big)\cdot
\Big(\xi_1(r)-\xi_2(r)\Big)\,.
\end{eqnarray*}
Hence
\begin{eqnarray}\label{concise-a}
\Big\|\xi_1(t)-\xi_2(t)\Big\|&\le&\|\xi_1(s)\!-\!\xi_2(s)\|
+2d_{\varphi_{f,\xi_1}}(\eta_1,\eta_2)\\
&\,&\,\,\,\,\,+\!c^1(K)\int_{s\wedge t}^{s\vee t}\|\xi_1(r)-\xi_2(r)\|\,dr
\,.\nonumber
\end{eqnarray}
It then follows from Proposition \ref{gronweq} and (\ref{concise-a})
that 
\begin{equation}\label{concisefox}
\|\xi_1(t)-\xi_2(t)\|\le e^{|t-s|c^1(K)}\Big(\|\xi_1(s)-\xi_2(s)\|+
2d_{\varphi_{f,\xi_1}}(\eta_1,\eta_2)\Big)\,.
\end{equation}
Interchanging the pairs $(\xi_1,\eta_1)$ and $(\xi_2,\eta_2)$ yields a 
similar inequality with $d_{\varphi_{f,\xi_1}}$ replaced by
$d_{\varphi_{f,\xi_2}}$. Inequality (\ref{concise}) then follows.\proofend
\end{proof}

\begin{proposition} \label{uniqueness} (Uniqueness of solutions of the 
Cauchy problem.)
Assume that  $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a classical
$n$-dimensional control system of class $C^{1,0,pc}$. 
Let $a,b\in I$ be such  that $a\le b$, and let 
$\eta$ be such that $\eta\in\mathcal{U}$ and
$\dom(\eta)=[a,b]$. Let  $\xi_1,\xi_2$ be full trajectories for $\eta$ such 
that $\xi_1(a)=\xi_2(a)$. Then $\xi_1\equiv\xi_2$. 
\end{proposition}

\begin{proof}
Let $G$ be the set of all $t\in[a,b]$ such that 
$\xi_1(s)=\xi_2(s)$ for all $s$ such that $a\le s\le t$. 
Then it is clear that $G$ is a closed interval of the form
$[a,c]$, with $a\le c\le b$. We prove that $c=b$. Suppose
that $c<b$. Use Proposition \ref{estimatix}, taking $s=c$,
letting $Q$ be some closed ball centered at $\xi_1(c)$ such that 
$Q\subseteq\Omega$ and both $\xi_1(t)$ and $\xi_2(t)$ 
belong to $Q$ for all $t$ in some interval $[c,c+\eps]$, 
and taking $\eta_1$, $\eta_2$ to be equal to each other 
and equal to the restriction of $\eta$ to $[c,c+\eps]$.
Then Proposition \ref{estimatix} implies that
$\|\xi_1(t)-\xi_2(t)\|=0$ for $t\in[c,c+\eps]$. Hence
$[c,c+\eps]\subseteq G$, contradicting the definition of $c$.\proofend
\end{proof}

\begin{proposition} \label{continuousdep} (Continuous dependence
of trajectories on the control and initial condition.)
Assume that  $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a classical
$n$-dimensional control system
of class $C^{1,0,pc}$. Let $a,b\in I$ be such that $a\le b$.
Let $U_0$ be a compact subset of $U$. Let $\eta_*\in\mathcal{U}$ be such 
that $\dom(\eta_*)=[a,b]$ and $\eta_*([a,b])\subseteq U_0$, and let
$\xi_*:[a,b]\mapsto\Omega$ be a full trajectory for $\eta_*$. 
Then for every positive number $\eps$ 
there exist positive numbers $\delta$, $\mu$, such that
\begin{itemize}
\item[(1)] $\bar\bmath^n(\xi_*(a),\delta)\subseteq\Omega$, 
\item[(2)] if  $\bar x\in\bar\bmath^n(\xi_*(a),\delta)$, and
$\eta\in\mathcal{U}$ is such that $\dom(\eta)=[a,b]$, 
$\eta([a,b])\subseteq U_0$, and   
\mbox{$d_{\varphi_{f,\xi_*}}(\eta,\eta_*)\le\mu$},  then 
\begin{itemize}
\item[(2.a)] there exists a full trajectory $\xi$ for $\eta$
such that $\xi(a)=\bar x$,
\item[(2.b)] the trajectory $\xi=\xi_{\bar x,\eta}$ described in
(2.a) (which is clearly  unique 
\mbox{because} of Proposition \ref{uniqueness})
satisfies $\|\xi_{\bar x,\eta}(t)-\xi_*(t)\|\le\eps$ 
for all $t\in[a,b]$.  
\end{itemize}\end{itemize}
\end{proposition}

\begin{proof}
If we let $Q_0=\{\xi_*(t):a\le t\le b\}$, then $Q_0$ is a  
compact subset of $\Omega$.  Choose a positive number 
$\bar\delta$ such that the compact set
$$
Q=\{x\in\rr^n:\|x-\xi_*(t)\|\le\bar\delta\,\,{\rm for\,\,some}\,\,t\in[a,b]\}
$$
is contained in  $\Omega$. Let $K=Q\times U_0\times[a,b]$. Let
$c^0(K)$, $c^1(K)$ be the constants defined in Remark
\ref{existconstants}. 

Let $\tilde\eps=\min(\eps,\bar\delta)$. 
Let $\delta,\mu$ be such that $0<\delta\le\bar\delta$ and $\mu>0$.
Let $\bar x\in\Omega$ be such that $\|\bar x-\xi_*(a)\|\le\delta$, 
and let $\eta$ be a $U_0$-valued  piecewise continuous control 
defined on $[a,b]$ such that $d_{f,\xi_*}(\eta,\eta_*)\le\mu$.
For each $\beta\in[a,b]$, let $\eta^\beta$ be the restriction of $\eta$ to
the interval $[a,\beta]$.  Let 
$S(\bar x,\eta)$ be the set of all $\beta\in[a,b]$ such that
there exists a trajectory $\xi:[a,\beta]\mapsto\Omega$
for $\eta^\beta$ such that $\xi(a)=\bar x$ and
$\|\xi(t)-\xi_*(t)\|\le\tilde\eps$ for all $t\in[a,\beta]$. 
Let $\bar\beta=\bar\beta(\bar x,\eta)$ be the supremum of all the members
of $S(\bar x,\eta)$. Then $\bar\beta\in S(\bar x,\eta)$. 
(Indeed, suppose that $\bar\beta\notin S(\bar x,\eta)$. 
Then there exists an increasing sequence $\{\beta_j\}_{j\in\nn}$ 
of members of $S(\bar x,\eta)$ such that 
$\beta_j\uparrow\bar\beta$ as $j\to\infty$.
For each $j$, we can pick a trajectory $\xi_j:[a,\beta_j]\mapsto\Omega$
such that $\xi_j(\bar t)=\bar x$ and 
$\|\xi_j(t)-\xi_*(t)\|\le\tilde\eps$ for all $t\in[a,\beta_j]$. 
The uniqueness property (Proposition \ref{uniqueness})
implies that $\xi_{j+1}\equiv\xi_j$ on $[a,\beta_j]$. 
Hence there exists a continuous curve 
$\xi:[a,\bar\beta[\,\mapsto\Omega$
such that $\xi\equiv\xi_j$ on $[a,\beta_j]$. Then 
$\dot\xi(t)=f(\xi(t),\eta(t),t)$ for all except finitely many $t$'s.
Therefore $\|\dot\xi(t)\|\le c^0(K)$ for all except finitely many $t$'s,
and then the inequality $\|\xi(t)-\xi(s)\|\le c^0(K)|t-s|$ holds whenever
$t,s\in[a,\bar\beta[\,$. It follows that the limit
$\hat x=\lim_{t\uparrow\bar\beta}
\xi(t)$ exists. We can then extend $\xi$ to the closed
interval $[a,\bar\beta]$ by letting
$\xi(\bar\beta)=\hat x$, and it is clear that
the extended $\xi$ is a trajectory for $\eta^{\bar\beta}$
such that $\xi(\bar t)=\bar x$ and 
$\|\xi(t)-\xi_*(t)\|\le\tilde\eps$ for all $t\in[a,\bar\beta]$.
Hence $\bar\beta\in S(\bar x,\eta)$.)

We now show that $\bar\beta(\bar x,\eta)=b$ if $\delta$, $\mu$ are 
sufficiently small.   To see this, we let $\xi_{\bar x,\eta}$ be 
the trajectory $\xi$ for $\eta^{\bar\beta(\bar x,\eta)}$ 
such that $\xi(\bar t)=\bar x$. (This trajectory exists 
and satisfies $\|\xi(t)-\xi_*(t)\|\le\tilde\eps$ for all 
$t\in[a,\bar\beta(\bar x,\eta)]$ because 
$\bar\beta(\bar x,\eta)\in S(\bar x,\eta)$, and
is unique by Proposition \ref{uniqueness}.)  Then Proposition 
\ref{estimatix} (applied with $s=a$) yields the bound
$$
\|\xi_{\bar x,\eta}(t)-\xi_*(t)\|\le e^{(b-a)c^1(K)}\Big(\delta+
2d_{f,\xi_*}(\eta,\eta_*)\Big)\quad{\rm for\,\,all}\quad
t\in[a,\bar\beta(\bar x,\eta)]\,.
$$
Now choose 
\begin{equation}\label{usull}
\delta=\mu=\frac{1}{4}e^{-(b-a)c^1(K)}\tilde\eps\,.
\end{equation}
Then 
\begin{equation}\label{usull-aa}
\|\xi_{\bar x,\eta}(t)-\xi_*(t)\|\le\frac{3\tilde\eps}{4}
\quad{\rm for\,\,all}\quad t\in[a,\bar\beta(\bar x,\eta)]
\end{equation}
provided that
\begin{equation}\label{usull-a}
\eta\in C^{0,pc}([a,b],U_0)\,,
\qquad\|\bar x-\xi_*(a)\|\le\delta\,,\qquad
{\rm and}\,qquad d_{f,\xi_*}(\eta,\eta_*)\le\mu\,.
\end{equation}
It follows that 
%\begin{equation}\label{usull-b}
%\bar\beta(\bar x,\eta)=b\qquad{\rm if\qquad(\ref{usull-a})\quad holds}\,.  
%\end{equation}
(Indeed, suppose that (\ref{usull-a}) holds and $\bar\beta(\bar x,\eta)<b$.
Let $\bar\beta=\bar\beta(\bar x,\eta)$. Then (\ref{usull-aa}) implies that 
$\|\xi_{\bar x,\eta}(\bar\beta)-\xi_*(\bar\beta)\|\le\frac{3\tilde\eps}{4}$.
But then $\|\xi_{\bar x,\eta}(\bar\beta)-\xi_*(\bar\beta)\|<\tilde\eps$.
\mbox{Proposition} \ref{localexi} implies that there exists 
a positive $\rho$ such that $\bar\beta+\rho\le b$, 
\mbox{having} the \mbox{property}
that on the interval $[\bar\beta,\bar\beta+\rho]$ there is a 
trajectory $\xi$ for the control $\eta\restr[\bar\beta,\bar\beta+\rho]$ 
such that $\xi(\bar\beta)=\xi_{\bar x,\eta}(\bar\beta)$. 
Since $\|\xi_{\bar x,\eta}(\bar\beta)-\xi_*(\bar\beta)\|<\tilde\eps$
we may assume, after making $\rho$ smaller, if necessary, that
$\|\xi_{\bar x,\eta}(t)-\xi_*(t)\|<\tilde\eps$
for all $t\in[\bar\beta,\bar\beta+\rho]$. If we then
define a map $\zeta:[a,\bar\beta+\rho]\mapsto\Omega$
by letting $\zeta(t)=\xi_{\bar x,\eta}(t)$ for $t\in[a,\bar\beta]$
and $\zeta(t)=\xi(t)$ for $t\in[\bar\beta,\bar\beta+\rho]$,
we see that $\zeta$ is a trajectory for the control
$\eta^{\bar\beta+\rho}$, $\zeta(\bar t)=\bar x$, and
$\|\zeta(t)-\xi_*(t)\|\le\tilde\eps$ for all 
$t\in[\bar\beta,\bar\beta+\rho]$. Therefore 
$\bar\beta+\rho\in S(\bar x,\eta)$, contradicting the 
definition of $\bar\beta$.)

It is then clear that, if we choose $\delta$ and $\mu$ such that
(\ref{usull}) holds, then all our desired conclusions 
are satisfied. \proofend
\end{proof}

\begin{corollary} \label{continuousdepcoro} 
Assume that  $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a classical
$n$-dimensional control system of class $C^{1,0,pc}$. 
Let $a,b\in I$ be such that $a\le b$. Let $\{\eta_j\}_{j\in J}$  
be a sequence of members
of $\mathcal{U}$ that are defined on $[a,b]$ and c-converge 
to an $\eta_*\in\mathcal{U}$ as $j\to_J\infty$.   
Let $\xi_*:[a,b]\mapsto\Omega$
be a full trajectory of $\eta_*$. Let $\{x_j\}_{j\in J}$  be a 
sequence of members of $\Omega$ such that 
$\lim_{j\to\infty,j\in J}x_j=\xi_*(a)$.	
Then there exists a $j^\#\in\nn$ such that 
\begin{itemize}
\item[(1)] if $j\in J$ and $j\ge j^\#$ then there exists a full 
trajectory $\xi_j:[a,b]\mapsto\Omega$ for $\eta_j$ such that 
$\xi_j(a)=x_j$,
\item[(2)] the $\xi_j$ described in (1), which are 
clearly unique because of Proposition \ref{uniqueness},
converge uniformly to $\xi_*$ as $j\to_J\infty$.
\end{itemize}
\end{corollary}

\begin{proof}
Since the $\eta_j$ c-converge to $\eta_*$, we can pick a compact subset 
$U_0$ of $U$ such that $\eta_*$ and all the $\eta_j$ take values
in $U_0$. Proposition \ref{continuousdep} then implies the existence,
for each $k\in\nn$, of positive numbers $\delta_k$, $\mu_k$ such 
that $\bar\bmath^n(\xi_*(a),\delta_k)\subseteq\Omega$, having the property
that whenever $\bar x\in\bar\bmath^n(\xi_*(a),\delta_k)$, 
$\eta\in\mathcal{U}$, $\dom(\eta)=[a,b]$, 
and $d_{\varphi_{f,\xi_*}}(\eta,\eta_*)\le\mu_k$, it follows that
there exists a full trajectory $\xi=\xi_{\bar x,\eta}$ for 
$\eta$ such that $\xi(a)=\bar x$ and $\|\xi(t)-\xi_*(t)\|\le 2^{-k}$
for all $t\in[a,b]$.  Clearly, we may assume that the sequence
$\{\delta_k\}_{k\in\nn}$ is decreasing. Let $j^{\#}$ be such that
$\|x_j-\xi_*(a)\|\le\delta_1$ and  
$d_{\varphi_{f,\xi_*}}(\eta_j,\eta_*)\|\le\mu_1$ whenever 
$j\in J$ and $j\ge j^{\#}$. Let $\xi_j=\xi_{x_j,\eta_j}$ for $j\in J$, 
$j\ge j^{\#}$.
Then $\xi_j$ is a full trajectory for $\eta_j$ and $\xi_j(a)=x_j$. 

Given $k$, we can pick a $j(k)\!\in\!J$ having the property that 
\mbox{$\|x_j\!-\!\xi_*(a)\|\le\delta_k$} and  
$d_{\varphi_{f,\xi_*}}(\eta_j,\eta_*)\!\le\!\mu_k$ \mbox{whenever}
$j\!\in\!J$ and $j\!\ge\!j(k)$. Then 
\mbox{$\|\xi_j(t)\!-\!\xi_*(t)\|\!\le\!2^{-k}$} \mbox{whenever}
$j\in J$, $j\ge j(k)$ and $t\in[a,b]$. Therefore $\xi_j\to\xi_*$ uniformly
as $j\to_J\infty$.\proofend
\end{proof}

\begin{corollary} \label{continuousdepcoro-b} 
Assume that  $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a classical
$n$-dimensional control system
of class $C^{1,0,pc}$. Let $a,b\in I$ be such that $a\le b$.
Let $P$ be a metric space, and let $\{\eta_p\}_{p\in P}$  be a 
family of members of $\mathcal{U}$ that are defined on $[a,b]$ and 
depend \mbox{c-continuously} on $p$. Let 
$$
\mathcal{O}=\Big\{(\bar x,p):(\exists\xi\in C^0([a,b],\Omega))((\xi,\eta_p)\in 
TCP(\Sigma)\wedge\xi(a)=\bar x)\Big\}\,.
$$
Then 
\begin{itemize}
\item[(1)] $\mathcal{O}$ is an open subset of
$\Omega\times P$,
\item[(2)] if we let $\xi_{\bar x,p}$ be, for 
$(\bar x,p)\in\mathcal{O}$, the member  $\xi$ of $C^0([a,b],\Omega)$ 
such that $(\xi,\eta_p)\in TCP(\Sigma)$ and $\xi(a)=\bar x$ (which is 
unique  because of Proposition \ref{uniqueness}), then the map
$$
\mathcal{O}\ni(\bar x,p)\mapsto\xi_{\bar x,p}\in C^0([a,b],\Omega)
$$ 
is continuous. (That is, whenever $\{(\bar x_j,p_j)\}_{j\in J}$
is a sequence in $\mathcal{O}$ that \mbox{converges} to a limit
$(\bar x,p)\in\mathcal{O}$, it follows that 
$\xi_{\bar x_j,p_j}\to_J\xi_{\bar x,p}$ uniformly.)
\end{itemize}
\end{corollary}
\begin{proof}
Let $\tilde{\mathcal{O}}=(\Omega\times P)\backslash\mathcal{O}$. We prove that
$\tilde{\mathcal{O}}$ is a closed subset of $\Omega\times P$. Suppose 
that $\{(\bar x_j,p_j)\}_{j\in J}$
is a sequence in $\tilde{\mathcal{O}}$ that converges to a limit
\mbox{$(\bar x,p)\in\Omega\times P$}.  We show that 
$(\bar x,p)\in\tilde{\mathcal{O}}$ by contradiction.
Suppose that $(\bar x,p)\in\mathcal{O}$.  Let 
$\xi_*$ be the full trajectory for $\eta_p$ such that
$\xi_*(a)=\bar x$. Then Corollary \ref{continuousdepcoro} 
implies that if $j\in J$ is large enough then there exists a full trajectory
$\xi_j$ for $\eta_{p_j}$ such that $\xi_j(a)=\bar x_j$.
But then $(\bar x_j,p_g)\in\mathcal{O}$ for large $j$,
and we have reached a contradiction. This proves that
$\mathcal{O}$ is open. The second statement then follows
immediately from Corollary \ref{continuousdepcoro}.\proofend
\end{proof}


\pparagraph{The variational equation}\label{secvarieq}
In this subsection we will assume that
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{secvarieq}.1)] {\em $\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a 
classical $n$-dimensional control system of class $C^{1,0,pc}$;}
\item[(\ref{secvarieq}.2)] {\em $a,b$ are such that $a,b\in I$ and $a\le b$;}
\item[(\ref{secvarieq}.3)] {\em $\gamma_*=(\xi_*,\eta_*)$ is a
trajectory-control pair of $\Sigma$  such that $\dom(\eta)=[a,b]$}.
\end{itemize}\end{itemize}

\begin{definition}
{\rm Assume that conditions (\ref{secvarieq}.1-2-3) hold. 
The  \mbox{ordinary} \mbox{differential} equation
\begin{equation}\label{variequ}
\dot y=Df(\xi_*(t),\eta_*(t),t)\cdot y
\end{equation}
is called the {\bf\em variational equation for $\Sigma$
along the trajectory-control pair $\gamma_*$}.} \proofend
\end{definition}

It is clear that the variational equation is of the form 
(\ref{lindifinc-cont})---with 
\begin{equation}\label{varigen}
A(t)=Df(\xi_*(t),\eta_*(t),t)
\end{equation}
and that the matrix-valued function $A$ satisfies 
$A\in C^{0,pc}([a,b],\linx)$, because $Df$ is continuous
on $\Omega\times U\times[a,b]$,
$\xi_*$ is continuous, and the control $\eta_*$ is piecewise continuous. 


\begin{definition}
{\rm Assume that conditions (\ref{secvarieq}.1-2-3) hold. Then
\begin{itemize}
\item[$\bullet$] {\em the \mbox{matrix-valued} \mbox{function} 
$A:[a,b]\mapsto\rr^{n\times n}$ defined by (\ref{varigen}) is 
known as the {\bf\em \mbox{variational}
generator for $\Sigma$ along $\gamma_*\,$}};
\item[$\bullet$] {\em the fundamental solution of (\ref{variequ}) is 
called the {\bf\em \mbox{variational} propagator for $\Sigma$
along $\gamma_*\,$.}}  \proofend
\end{itemize}}
\end{definition}
We will use $\mathcal{A}^{\Sigma,\gamma_*}$, $\Pi^{\Sigma,\gamma_*}$ to 
denote, respectively, the variational generator and the variational 
propagator for $\Sigma$ along $\gamma_*\,$, so 
\fbox{$\mathcal{A}^{\Sigma,\gamma_*}(t)\equaldef
Df(\xi_*(t),\eta_*(t),t)$} and 
\fbox{$\Pi^{\Sigma,\gamma_*}\equaldef\Xi^{\mathcal{A}^{\Sigma,\gamma_*}}$}.


With the above notations, the variational equation (\ref{variequ})
becomes
\begin{equation}\label{variequalt}
\dot y=
\mathcal{A}^{\Sigma,\gamma_*}(t)\cdot y\,.
\end{equation}
The vector solution of  (\ref{variequalt}) with an initial condition
$y(\bar t)=\bar y$ is then given by 
$$
y(t)=\Pi^{\Sigma,\gamma_*}(t,\bar t)\bar y\,.
$$

\pparagraph{The infinitesimal reachable cone
and the infinitesimal endpoint value cone}
\label{sec-pontrycone} We recall that in Definition \ref{reacha}
(page \pageref{reacha})
we introduced the {\bf\em endpoint value set} $\evset(\Sigma)$
for a control system $\Sigma$, i.e., the set  
of all \mbox{endpoint} values $\partial\xi$, for all pairs
$(\xi,\eta)\in TCP(\Sigma)$. We will now define a terminal
point  analogue,  the ``reachable set'' $\rset(\Sigma,a,x_*)$,
and then introduce, under the assumption that conditions 
(\ref{secvarieq}.1-2-3) $\,$hold,  two sets of vectors---the 
``infinitesimal reachable cone'' $IRC(\Sigma,\gamma_*)$,
and the ``infinitesimal endpoint value 
cone''  $IEVC(\Sigma,\gamma_*)$---that are meant to be first-order 
approximations  to $\rset(\Sigma,a,x_*)$ and
$\evset(\Sigma)$ at the points $\xi(b)$, $\partial\xi$,
of $\rr^n$, $\rr^n\times\rr^n$, respectively.   The main technical 
point of the proof of the Maximum Principle will then be, precisely,
to show that these ``infinitesimal'' cones are indeed limiting 
approximating cones to the corresponding sets. 

\begin{definition}
{\rm For a control system $\Sigma$, a time $a$, and a point $x_*$
belonging to the state space of $\Sigma$, 
we use $\rset(\Sigma,a,x_*)$ to denote the 
set of all points $\xi(b)$, for all pairs $(b,\xi)$ 
such that $b\ge a$, $(\xi,\eta)\in TCP(\Sigma)$ 
for some $\eta\in\mathcal{U}$ having the property that 
$\dom(\eta)=[a,b]$,
and $\xi(a)=x_*$. 
The set $\rset(\Sigma,a,x_*)$ is the {\bf\em 
$\Sigma$-reachable set starting at $x_*$ at time $a$}.}\proofend
\end{definition}

First, we define vectors
\mbox{$v^{\Sigma,u,t,\gamma_*}\,,\,w^{\Sigma,u,t,\gamma_*}\in\rr^n$},
$W^{\Sigma,u,t,\gamma_*}\,,\,
W^{\Sigma,\gamma_*}(h)\in\rr^n\times\rr^n$, for $a\le t<b$, $u\in U$,
$h\in\rr^n$, by letting ,
\begin{eqnarray}\label{varivec}
v^{\Sigma,u,t,\gamma_*}\,&\equaldef&\, f(\xi_*(t),u,t)-f(\xi_*(t),\eta_*(t+),t)
\,,\label{varivec}\\
w^{\Sigma,u,t,\gamma_*}\,&\equaldef&\,\Pi^{\Sigma,\gamma_*}(b,t)\cdot 
v^{\Sigma,u,t,\gamma_*}\,,\label{varivecend}\\
W^{\Sigma,u,t,\gamma_*}\,&\equaldef&\,(0,w^{\Sigma,u,t,\gamma_*})\,,
\label{varivecend-a}\\
W^{\Sigma,\gamma_*}(h)\,&\equaldef&\,\Big(h,
\Pi^{\Sigma,\gamma_*}(b,a)\cdot h\Big)\,,\label{varivectwoend}
\end{eqnarray}
and observe that $W^{\Sigma,\gamma_*}$ is a linear map 
from $\rr^n$ to $\rr^n\times\rr^n$.


\begin{definition}
{\rm Assume that conditions (\ref{secvarieq}.1-2-3) hold. 
\begin{itemize}
\item[1.] The {\bf\em infinitesimal reachable cone}
for \ $(\Sigma,\gamma_*)$ \ is
the smallest \mbox{closed convex} cone in $\rr^n$ that contains all the vectors
$w^{\Sigma,u,t,\gamma_*}$, for all $(u,t)\in U\times[a,b[\,$.
\item[2.] The {\bf\em $\,\,$infinitesimal $\,\,$two-endpoint $\,\,$value 
$\,\,$cone}  $\,\,$for $\,$$\,\,\,$$(\Sigma,\gamma_*)$$\,\,\,$$\,$ is 
$\,\,$the  $\,\,$smallest  \mbox{closed convex} cone in 
$\rr^n\times\rr^n$ that contains all the vectors
$W^{\Sigma,u,t,\gamma_*}$, for all 
\mbox{$(u,t)\in U\times[a,b[\,$}, and all the 
$W^{\Sigma,t,\gamma_*}(h)$, for all \mbox{$h\in\rr^n$}.
\end{itemize} 
We use $IRC(\Sigma,\gamma_*)$ and $ITEVC(\Sigma,\gamma_*)$, 
to denote, respectively, the infinitesimal
\mbox{reachable} cone and the infinitesimal 
two-endpoint value cone  for $(\Sigma,\gamma_*)$.}\proofend
\end{definition}

\begin{remark}
{\rm The name commonly used in the literature for what we have called
``infinitesimal reachable cone'' is \mbox{``Pontryagin} cone.''}\proofend
\end{remark}

\pparagraph{Packets of needle variations}
\label{sec-constru-ox}
Throughout this subsection, it will be assumed that 
\ben\item[]\ben
\item[(\ref{sec-constru-ox}.1)] {\em $n\in\nn$,  and 
$\,\,\Sigma=(\Omega,U,I,f,\mathcal{ U})\,\,$ is a 
classical $n$-dimensional control system of class $C^{1,0,pc}$};

\item[(\ref{sec-constru-ox}.2)] {\em $a\in I$, $b\in I$, and $a\le b$};

\item[(\ref{sec-constru-ox}.3)] {\em $\gamma_*=(\xi_*,\eta_*)$ (the 
``reference trajectory-control pair'') is a member of $TCP(\Sigma)$};

\item[(\ref{sec-constru-ox}.4)] {\em $\dom(\eta_*)=[a,b]$}.
\een\een

\noindent We recall that if $\mu\in\nn$ then  $\rr_{\mu}$ denotes
the set of all real $\mu$-dimensional
{\em row} vectors $\veceps=(\eps_1,\ldots,\eps_\mu)$. We use $\rr_{+,\mu}$ to 
denote the nonnegative orthant of  $\rr_\mu$, i.e., 
the set of all $\mu$-tuples
$\veceps=(\eps_1,\ldots,\eps_\mu)\in\rr_\mu$ such that 
$\eps_j\ge 0$ for $j=1,\ldots,\mu$. For 
$\veceps=(\eps_1,\ldots,\eps_\mu)\in\rr_{+,\mu}$, we write
$$
|\veceps|\,\,\equaldef\,\,\eps_1+\cdots+\eps_\mu\,.
$$
If $r>0$, we use $\mathcal{ S}_\mu(r)$ to denote the 
$\mu$-dimensional simplex 
$$
\mathcal{ S}_\mu(r)\,\equaldef\,\{\,\veceps\in\rr_{+,\mu}:
|\veceps|\le r\,\}\,.
$$

\begin{definition}
{\rm Assume that conditions \ (\ref{sec-constru-ox}.1-2-3) \ hold.
\begin{itemize}
\item[1.] An {\bf\em \mbox{infinitesimal} needle variation packet for
$\Sigma$ at $\gamma_*$} is a finite sequence
\begin{equation}\label{infipack}
{\bf V}=((u_1,t_1),(u_2,t_2),\ldots,(u_m,t_m))
\end{equation}
of members of $U\times\rr$, such that
$a\le t_1\le t_2\le\cdots\le t_m<b$.
\item[2.] The {\bf\em infinitesimal two-endpoint value map associated
to $\,\,\Sigma\,$, $\,\,\gamma_*\,$, and the 
\mbox{infinitesimal} \ \ needle \ \ variation \mbox{\ \ packet} 
\ \ (\ref{infipack})} \ \ is \ \ the \ \ linear \ \ map 
\mbox{$ITEVM_{\Sigma,,\gamma_*,{\bf V}}:\rr^n\times
\rr_{m}\mapsto\rr^n\times\rr^n$} given by
\end{itemize}}
$$
\quad\!\!ITEVM_{\Sigma,\gamma_*,{\bf V}}(h,\veceps)\!=\!W^{\Sigma,\gamma^*}(h)
\!+\!\sum_{j=1}^m\eps_jW^{\Sigma,u_j,t_j,\gamma_*}
\,\,\,\,{\rm if}\,\,\,\,h\in\rr^n\,,\,\veceps\!=\!(\eps_1,\ldots,\eps_m)\!\,.
\,\,\,\,\,\square
$$
\end{definition}
%It follows that 
%\begin{eqnarray*}
%\mathcal{IE}_{\Sigma,,\gamma_*,{\bf V}}(\veceps)
%&=&\sum_{j=1}^m\eps_j\Pi^{f,\gamma_*}
%(b,t_j)\cdot v^{u_j,t_j,\gamma_*}\\
%&=&\sum_{j=1}^m\eps_j\Pi^{f,\gamma_*}(b,t_j)\cdot
%\Big(f(\xi_*(t_j),u_j,t_j)-f(\xi_*(t_j),\eta_*(t_j),t_j)\Big)\,.
%\end{eqnarray*}

\begin{definition}
{\rm Assume that conditions (\ref{sec-constru-ox}.1-2-3) hold, $\mu\in\nn$,
and $\gamma_*=(\xi_*,\eta_*)\in TCP(\Sigma)$.
A {\bf\em \mbox{$\mu$-parameter} equal-time variation} of $\gamma_*$
is a family 
$$
\mathcal{V}=\{\gamma^{\veceps}\}_{\veceps\in\mathcal{S}_\mu(r)}
$$
of members of \ $TCP(\Sigma)$, 
defined for some positive real number $r$, such that,
if $\gamma^{\veceps}=(\xi^{\veceps},\eta^{\veceps})$, then
\begin{itemize}
\item[1.] each $\eta^{\veceps}$ belongs to 
$\mathcal{U}\cap\,C^{0,pc}([a,b],U)\,$,
\item[2.] the controls $\eta^{\veceps}$  depend
\mbox{c-continuously} on the parameter $\veceps$,
\item[3.] the trajectories $\xi^{\veceps}$  depend
\mbox{continuously} on $\veceps$, in the sense that if 
$\veceps^j\to\veceps$ then $\xi^{\veceps^j}\to\xi^{\veceps}$ uniformly
on $[a,b]$, 
\end{itemize}
$\quad\,\,$4. $\gamma^0=\gamma_*$.}\proofend
\end{definition} 

\begin{lemma}\label{guido}
Assume that conditions (\ref{sec-constru-ox}.1-2-3) hold, and $\mu\in\nn$.
Let $\mathcal{F}=\{\eta^{\veceps}\}_{\veceps\in\mathcal{S}_\mu(r)}$
be a family of controls belonging to 
$\mathcal{U}\cap\,C^{0,pc}([a,b],U)\,$ and such that
$\eta^0=\eta_*$ and the $\eta^{\veceps}$  depend
\mbox{c-continuously} on $\veceps$.
Let ${\bf h}=\{h^{\veceps}\}_{\veceps\in\mathcal{S}_\mu(r)}$
be a family of vectors in $\rr^n$ such that $h^0=0$
and the $h^{\veceps}$  depend \mbox{continuously} on $\veceps$.
Then there exists an $\bar r$ such that $0<\bar r\le r$ having 
the property that 
\begin{itemize}
\item[(\#)] for every $\veceps\in\mathcal{S}_\mu(\bar r)$
there exists a unique full trajectory $\xi^{\veceps}:[a,b]\mapsto
\Omega$ for $\eta^{\veceps}$ such that 
$\xi^{\veceps}(a)=\xi_*(a)+h^{\veceps}$. 
\end{itemize}
\end{lemma}
\begin{proof}
Corollary \ref{continuousdepcoro-b} implies that
$\xi^{\veceps}$ exists if $|\veceps|$ is sufficiently small.
Proposition \ref{uniqueness} implies that  
is unique. Corollary \ref{continuousdepcoro-b} then implies that
the $\xi^{\veceps}$  depend
\mbox{continuously} on $\veceps$.
\end{proof}
\begin{corollary}
Assume that conditions (\ref{sec-constru-ox}.1-2-3) hold, and $\mu\in\nn$.
Let $\mathcal{F}=\{\eta^{\veceps}\}_{\veceps\in\mathcal{S}_\mu(r)}$,
${\bf h}=\{h^{\veceps}\}_{\veceps\in\mathcal{S}_\mu(r)}$
be as in the statement of Lemma \ref{guido}. 
Then there exist an $\bar r$ such that $0<\bar r\le r$ and a 
\mbox{$\mu$-parameter} equal-time variation
\mbox{$\mathcal{V}=\{\gamma^{\veceps}\}_{\veceps\in\mathcal{S}_\mu(r)}$}
of $\gamma_*$ such that each $\gamma^{\veceps}$ is of the form
$(\xi^{\veceps},\eta^{\veceps})$ for some $\xi^{\veceps}$,
and  \mbox{$\xi^{\veceps}(a)=\xi_*(a)+h^{\veceps}$}.\proofend
\end{corollary}

\begin{definition}
{\rm Assume that conditions (\ref{sec-constru-ox}.1-2-3) hold, $\mu\in\nn$,
and $\mathcal{V}=\{\gamma^{\veceps}\}_{\veceps\in\mathcal{S}_m(h)}$
is a \mbox{$\mu$-parameter} equal-time variation of $\gamma_*$.
For each $\veceps\in\rr_{+,m}$ sufficiently close to $0$, let
$\gamma^{\veceps}=(\xi^{\veceps},\eta^{\veceps})$. Let
$$
{\bf EVM}_{\Sigma,,\gamma_*,\mathcal{V}}(\veceps)
\equaldef\partial\xi^{\veceps}\,.
$$
Then ${\bf EVM}_{\Sigma,,\gamma_*,\mathcal{V}}$---which is defined on some 
relative neighborhood of $0$ in $\rr_{+,\mu}$, and takes
values in $\rr^n\times\rr^n$---is the  {\bf\em endpoint
value map associated to $\mathcal{V}$ at $\gamma_*$}.} \proofend
\end{definition} 

Suppose that
$\mathcal{V}=\{\gamma^{\veceps}\}_{\veceps\in\mathcal{S}_\mu(r)}$ is a
\mbox{$\mu$-parameter} equal-time variation of the trajectory-control pair
$\gamma_*=(\xi_*,\eta_*)$.  Let
$\gamma^{\veceps}=(\xi^{\veceps},\eta^{\veceps})$. Then each 
$\eta^{\veceps}$ can be integrated against a
test function $\varphi\in C^0(U\times[a,b],\rr)$. This gives rise to a
function $\theta_{\varphi,\mathcal{V}}:\mathcal{S}_\mu(r)\mapsto\rr$,
given by
$$
\theta_{\varphi,\mathcal{V}}(\veceps)=
\int_a^b\varphi(\eta^{\veceps}(t),t)dt\,.
$$
\begin{definition}\label{assocpacket}
{\rm Assume that conditions (\ref{sec-constru-ox}.1-2-3) hold, and suppose
that ${\bf V}$ is an \mbox{infinitesimal} needle variation packet for
$\Sigma$ at $\gamma_*$ given by (\ref{infipack}). 
A {\bf\em needle \mbox{variation} packet 
associated to ${\bf V}$} is an $m+1$-parameter equal-time variation 
\mbox{$\mathcal{V}=\{\gamma^{\veceps}\}_{\veceps\in\mathcal{ S}_m(h)}$}
of $\gamma_*$ such that, if $\gamma^{\veceps}=(\xi^{\veceps},\eta^{\veceps})$,
then
\begin{itemize}
%\item[(1)] for every test function $\varphi\in C^0(U\times[a,b],\rr)$,
\item[(1)] for every test function $\varphi$ belonging to
$C^0(U\times[a,b],\rr)$,
the function $\theta_{\varphi,\mathcal{V}}$ is differentiable
at $\veceps=0$ along $\mathcal{S}_{m+1}(r)$, and its differential 
$D\theta_{\varphi,\mathcal{V}}(0)$ is the linear map from
$\rr_{m+1}$ to $\rr$ given by 
\begin{equation}\label{difisgiv}
D\theta_{\varphi,\mathcal{V}}(0)(\eps_0,\eps_1,\ldots,\eps_m)=
\sum_{j=1}^m\eps_j\Big(\varphi(u_j,t_j)
-\varphi(\eta_*(t_j+),t_j)\Big)\,;
\end{equation}
\item[(2)]  the map $\mathcal{S}_{m+1}(r)\ni\veceps\mapsto\xi^{\veceps}(a)$ 
is differentiable at $\veceps=0$ along $\mathcal{S}_{m+1}(r)$, 
and its differential is the linear map 
$\rr_{m+1}\ni(\eps_0,\eps_1,\ldots,\eps_m)\mapsto \eps_0h\in\rr^n$;
\end{itemize}}
\begin{itemize}
\item[{\rm (3)}] {\rm There exists a constant $\kappa$ such that
for every $\veceps\in\mathcal{S}_{m+1}(r)$ there is a subset 
$I(\veceps)$ of $[a,b]$ which is a finite
union of intervals the sum of whose measures is $\le\kappa|\veceps|$
and is such that  $\eta^{\veceps}(t)=\eta_*(t)$ whenever
$t\in[a,b]\backslash I(\veceps)$.} \proofend
\end{itemize}
\end{definition}


\begin{proposition}
Assume that conditions (\ref{sec-constru-ox}.1-2-3) hold, and suppose that
${\bf V}$ is an \mbox{infinitesimal} needle variation packet for
$\Sigma$ at $\gamma_*$, given by (\ref{infipack}). Let 
\mbox{$\mathcal{V}=\{\eta^{\veceps}\}_{\veceps\in\mathcal{ S}_m(h)}$} 
be a needle \mbox{variation} packet in $\mathcal{U}$
associated to ${\bf V}$.  Then the 
\mbox{endpoint map $\mathcal{E}_{\Sigma,,\gamma_*,\mathcal{V}}$} is 
differentiable at $0$ along $\mathcal{ S}_m(h)$, and
$$
D\mathcal{E}_{\Sigma,,\gamma_*,\mathcal{V}}(0)=
\mathcal{IE}_{\Sigma,,\gamma_*,{\bf V}}\,.
$$
\end{proposition}

\begin{proof}
We pick $\bar h$ such that (\#) holds.  The fact that
the family $\mathcal{V}$ depends continuously on $\veceps$
implies that there exists a compact subset $U_0$ of $U$ such that
$\eta^{\veceps}([a,b])\subseteq U_0$ for all 
$\veceps\in\mathcal{S}_{m}(\bar h)$. (Indeed, take $U_0$ 
to be the closure of $W$, where $W$ is the union of the sets
$\eta^{\veceps}([a,b])$, for all $\veceps\in\mathcal{S}_{m}(\bar h)$. 
We show that $U_0$ is compact. Suppose 
${\bf u}=\{u_j\}_{j=1}^\infty$ is a sequence of members
of $U_0$. Then for each  $u_j$ there exists a 
$\tilde u_j\in W$ such that $d_U(u_j,\tilde u_j)\le 2^{-j}$.
Since $\tilde u_j\in W$, we can write 
$\tilde u_j=\eta^{\veceps_j}(t_j)$ for some 
$\veceps_j\in\mathcal{S}_{m}(\bar h)$, $t_j\in[a,b]$. 
Since both $\mathcal{S}_{m}(\bar h)$ and $[a,b]$ are compact, 
we can find an infinite subset $J$ of $\nn$
such that the limits $\veceps^\#=\lim_{j\to\infty,j\in J}\veceps_j$ 
and $t^\#=\lim_{j\to\infty,j\in J}t_j$ exist and belong to 
$\mathcal{S}_{m}(\bar h)$, $[a,b]$, respectively. 
The sequence of controls $\{\eta^{\veceps_j}\}_{j\in J}$ 
c-converges to $\eta^{\veceps^\#}$. Therefore there exists a compact
subset $Z$ of $U$ such that $\eta^{\veceps_j}([a,b])\subseteq K$
for all $j\in J$. Since $\eta^{\veceps_j}(t_j)\in K$, we can find
an infinite subset $J'$ of $J$ such that 
the limit 
$u^\#=\lim_{j\to\infty,j\in J'}\eta^{\veceps_j}(t_j)$ exists.
Then $u^\#=\lim_{j\to\infty,j\in J'}\tilde u_j$. Furthermore,
$u^\#\in U_0$, because the $\tilde u_j$ belong to $W$. 
Finally, $u^\#=\lim_{j\to\infty,j\in J'}u_j$ as well.
So we have extracted a subsequence of ${\bf u}$ that converges
to a point of $U_0$. This proves the desired compactness.)

For $\veceps\in\mathcal{S}_{m}(\bar h)$, we write
$\zeta^{\veceps}(t)=\xi^{\veceps}(t)-\xi_*(t)$. 
Proposition \ref{continuousdep} implies that the curves 
$\xi^{\veceps}$ converge uniformly to $\xi_*$ as $\veceps\to 0$, 
so $\zeta^{\veceps}\to 0$ uniformly as $\veceps\to 0$. Hence we may assume,
after making $\bar h$ smaller, if necessary, that
there exists a positive number $\bar\delta$ such that
the ball $\{x\in\rr^n:\|x-\xi_*(t)\|\le\delta\}$ is contained 
in $\Omega$ for every $t\in[a,b]$, and 
$\|\zeta^{\veceps}(t)\|\le\bar\delta$ whenever 
$\veceps\in\mathcal{S}_{m}(\bar h)$ and $t\in[a,b]$. 
We let 
$\Omega_0=\{x\in\Omega:(\exists t\in[a,b])(\|x-\xi_*(t)\|\le\bar\delta\}$. 
Then $\Omega_0$ is compact. We then define
$$
K=\Omega_0\times U_0\times[a,b]\,,
$$
and let $c^0(K)$, $c^1(K)$ be the constants introduced in Remark
\ref{existconstants}. 

Clearly, 
\begin{eqnarray*}
\dot\zeta^{\veceps}(t)&=&
f(\xi^{\veceps}(t),\eta^{\veceps}(t),t)-f(\xi_*(t),\eta_*(t),t)\\
&=&f(\xi^{\veceps}(t),\eta^{\veceps}(t),t)
-f(\xi^{\veceps}(t),\eta_*(t),t)+f(\xi^{\veceps}(t),\eta_*(t),t)
-f(\xi_*(t),\eta_*(t),t)\\
&=&f(\xi^{\veceps}(t),\eta^{\veceps}(t),t)
-f(\xi^{\veceps}(t),\eta_*(t),t)+\mathcal{A^{\veceps}}(t)\cdot
\zeta^{\veceps}(t)\\
&=&E_1^{\veceps}(t)+E_2^{\veceps}(t)+
+\mathcal{A^{\veceps}}(t)\cdot\zeta^{\veceps}(t)\,,
\end{eqnarray*}
where
\begin{eqnarray*}
\mathcal{A^{\veceps}}(t)&=&\int_0^1Df(\xi_*(t)+\alpha\zeta^{\veceps}(t),
\eta_*(t),t)\,d\alpha\,,\\
E_1^{\veceps}(t)&=&f(\xi_*(t),\eta^{\veceps}(t),t)-f(\xi_*(t),\eta_*(t),t)\\
E_2^{\veceps}(t)&=&\Big(f(\xi^{\veceps}(t),\eta^{\veceps}(t),t)-
f(\xi^{\veceps}(t),\eta_*(t),t)\Big)\\
&\,&\quad-\Big(f(\xi_*(t),\eta^{\veceps}(t),t)-f(\xi_*(t),\eta_*(t),t)\Big)\,.
\end{eqnarray*}
Let $\Pi^{\veceps}$ be the fundamental matrix solution of
the equation $\dot x=\mathcal{A^{\veceps}}(t)\cdot x$. Then
\begin{equation}\label{righto}
\zeta^{\veceps}(t)=
\int_a^t\Pi^{\veceps}(t,s)\cdot\Big(E_1^{\veceps}(s)+E_2^{\veceps}(s)\Big)\,ds\,.
\end{equation}
(Indeed, if we let $\rho^{\veceps}(t)$ denote the right-hand side
of (\ref{righto}), then 
\begin{eqnarray*}
\dot\rho^{\veceps}(t)&=&E_1^{\veceps}(t)+E_2^{\veceps}(t)
+\int_a^t\frac{\partial\Pi^{\veceps}}{\partial t}(t,s)
\Big(E_1^{\veceps}(s)+E_2^{\veceps}(s)\Big)\,ds\\
&=&E_1^{\veceps}(t)+E_2^{\veceps}(t)
+\mathcal{A}^{\veceps}(t)\int_a^t\Pi^{\veceps}(t,s)
\Big(E_1^{\veceps}(s)+E_2^{\veceps}(s)\Big)\,ds\\
&=&E_1^{\veceps}(t)+E_2^{\veceps}(t)
+\mathcal{A}^{\veceps}(t)\rho^{\veceps}(t)\,.
\end{eqnarray*}
So $\rho^{\veceps}$ satisfies
$\dot\rho^{\veceps}=E_1^{\veceps}+E_2^{\veceps}+
\mathcal{A}^{\veceps}\rho^{\veceps}$, while we have shown that 
$\dot\zeta^{\veceps}=E_1^{\veceps}+E_2^{\veceps}+
\mathcal{A}^{\veceps}\zeta^{\veceps}$. Since 
$\rho^{\veceps}(a)=0$ and $\zeta^{\veceps}(a)=0$, we have
$\rho^{\veceps}\equiv\zeta^{\veceps}$, as stated.)

It is clear that $\mathcal{A}^0$ is the variational 
generator $\mathcal{A}^{f,\gamma_*}$. 
Therefore $\mathcal{A}^{\veceps}$ converges uniformly to
$\mathcal{A}^{f,\gamma_*}$, and then Proposition \ref{immed}
implies that the fundamental \mbox{solutions} 
$\Pi^{\veceps}$ converge uniformly to $\Pi^{f,\gamma_*}$. Therefore,
if we define
$$
\sigma_1(\veceps)=\sup\Big\{\,\|\Pi^{\veceps}(t,s)-\Pi^{f,\gamma_*}(t,s)\|:
t,s\in[a,b]\,\Big\}\,,
$$
we have $\lim_{\veceps\to 0}\sigma_1(\veceps)=0$.

Let $I(\veceps)$ be a subset of $[a,b]$ which is a finite
union of intervals the sum of whose measures is $\le\kappa|\veceps|$
and is such that  $\eta^{\veceps}(t)=\eta_*(t)$ whenever
$t\in[a,b]\backslash I(\veceps)$. 
The vector $E_1^{\veceps}(t)$ and $E_2^{\veceps}(t)$ clearly
vanish whenever $t\neq I(\veceps)$, and are 
bounded in norm by $2c^0(K)$ and $4c^0(K)$, resepctively,
for every $t$. Hence 
\begin{equation}\label{righto-ux}
\int_a^b\Big(\|E_1^{\veceps}(t)\|+\|E_2^{\veceps}(t)\|\Big)\,dt
\le 6c^0(K)\kappa|\veceps|\,.
\end{equation}
Therefore, if we write
\begin{equation}\label{righto-b}
\zeta^{\veceps}(b)=
\int_a^b\Pi^{f,\gamma_*}(b,s)\cdot\Big(E_1^{\veceps}(s)+E_2^{\veceps}(s)\Big)
\,ds+R_1(\veceps)\,,
\end{equation}
so that 
$$
R_1(\veceps)=\int_a^b\Big(\Pi^{\veceps}(b,s)-\Pi^{f,\gamma_*}(b,s)
\Big)\cdot\Big(E_1^{\veceps}(s)+E_2^{\veceps}(s)\Big)\,ds\,,
$$
we have
\begin{equation}\label{righto-ee}
\|R_1(\veceps)\|\le 6c^0(K)\kappa\sigma_1(\veceps)|\veceps|\,.
\end{equation}

If we let 
$$
\tilde\kappa=\sup\Big\{\,\|\Pi^{f,\gamma_*}(t,s)\|:
t,s\in[a,b]\,\Big\}\,,
$$
then (\ref{righto}) and (\ref{righto-ux}) imply the bound
$$
\|\zeta^{\veceps}(t)\|\le 6c^0(K)\kappa\tilde\kappa|\veceps|\quad
{\rm for}\quad t\in[a,b]\,.
$$
Therefore
\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\|E_2^{\veceps}(t)\|\\
&=&\Big\|\Big(f(\xi^{\veceps}(t),\eta^{\veceps}(t),t)-
f(\xi_*(t),\eta^{\veceps}(t),t)\Big)-
\Big(f(\xi^{\veceps}(t),\eta_*(t),t)-f(\xi_*(t),\eta_*(t),t)\Big)\Big\|\\
&\le&\Big\|f(\xi^{\veceps}(t),\eta^{\veceps}(t),t)-
f(\xi_*(t),\eta^{\veceps}(t),t)\Big\|
+\Big\|f(\xi^{\veceps}(t),\eta_*(t),t)-f(\xi_*(t),\eta_*(t),t)
\Big\|\\
&\le& 2c^1(K)\|\xi^{\veceps}(t)-\xi_*(t)\|\\
&\le& 12c^0(K)c^1(K)\kappa\tilde\kappa|\veceps|\,.
\end{eqnarray*}
Hence 
$$
\int_a^b\|E_2^{\veceps}(t)\|\,dt\le
12c^0(K)c^1(K)\kappa^2\tilde\kappa|\veceps|^2\,.
$$
Therefore, if we write
\begin{equation}\label{righto-bb}
\zeta^{\veceps}(b)=
\int_a^b\Pi^{f,\gamma_*}(b,s)\cdot E_1^{\veceps}(s)\,ds+R_1(\veceps)
+R_2(\veceps)\,,
\end{equation}
so that 
$$
R_2(\veceps)=\int_a^b\Pi^{f,\gamma_*}(b,s)\cdot E_2^{\veceps}(s)\,ds\,,
$$
we have
\begin{equation}\label{righto-oo}
\|R_2(\veceps)\|\le 12c^0(K)c^1(K)\kappa^2\tilde\kappa^2|\veceps|^2\,.
\end{equation}
If we combine (\ref{righto-bb}), (\ref{righto-ee}), and
(\ref{righto-oo}), and observe that $\zeta^{\veceps}(b)=
\mathcal{E}_{\Sigma,\gamma_*,\mathcal{V}}(\veceps)-
\mathcal{E}_{\Sigma,\gamma_*,\mathcal{V}}(0)$,  we find that
\begin{equation}\label{righto-bb}
\mathcal{E}_{\Sigma,\gamma_*,\mathcal{V}}(\veceps)=
\mathcal{E}_{\Sigma,\gamma_*,\mathcal{V}}(0)+
\int_a^b\Pi^{f,\gamma_*}(b,s)\cdot E_1^{\veceps}(s)\,ds+o(|\veceps|)\quad
{\rm as}\quad \veceps\to 0\,.
\end{equation}
Hence, to prove that $\mathcal{E}_{\Sigma,\gamma_*,\mathcal{V}}(\veceps)$ 
is differentiable at $\veceps=0$ with a certain differential
$\Lambda$, if suffices to show that the map
$\veceps\mapsto M(\veceps)\equaldef
\int_a^b\Pi^{f,\gamma_*}(b,s)\cdot E_1^{\veceps}(s)\,ds$
is differentiable at $\veceps=0$ with differential
$\Lambda$.

Now, let $\varphi(u,t)=\Pi^{f,\gamma_*}(b,t)f(\xi_*(t),u,t)$. 
Then $\varphi$ is a continous vector-valued function on
$U\times[a,b]$. Furthermore, it is clear that
$$
M(\veceps)=\theta_{\varphi,\mathcal{V}}(\veceps)-
\theta_{\varphi,\mathcal{V}}(0)\,.
$$
It follows that $M$ is differentiable at $\veceps=0$, and
$DM(0)$ is the map 
$$
(\eps_1,\ldots,\eps_m)\mapsto
\sum_{j=1}^m\eps_j\Big(\varphi(u_j,t_j)-\varphi(\eta_*(t_j+),t_j)\Big)\,,
$$
i.e., the map
$$
(\eps_1,\ldots,\eps_m)\mapsto
\sum_{j=1}^m\eps_j\Pi^{f,\gamma_*}(b,t_j)\Big(f(\xi_*(t_j),u_j,t)
-f(\xi_*(t_j),\eta_*(t_j+),t)\Big)\,.
$$
In other words, $DM(0)$ is the map 
$(\eps_1,\ldots,\eps_m)\mapsto\sum_{j=1}^m\eps_j w^{u_j,t_j,\gamma_*}$, 
i.e., the map $\mathcal{IE}_{\Sigma,\gamma_*,{\bf V}}$. Hence
$\mathcal{E}_{\Sigma,\gamma_*,\mathcal{V}}$ 
is differentiable at $\veceps=0$ and
$D\mathcal{E}_{\Sigma,\gamma_*,\mathcal{V}}(0)=
\mathcal{IE}_{\Sigma,\gamma_*,{\bf V}}$, completing our proof.
\end{proof}


\pparagraph{Existence of packets of needle variations}
\label{sec-constru-exi}
In this \mbox{subsection}, we assume that 

\ben\item[]\ben
\item[(\ref{sec-constru-exi}.1)] {\em $\Sigma=(\Omega,U,I,f,\mathcal{ U})$ 
is a control system in $\rr^n$ of class $C^{0,0,pc}$};

\item[(\ref{sec-constru-exi}.2)] {\em $a\in I$, $b\in I$, and $a\le b$};

\item[(\ref{sec-constru-exi}.3)] {\em $\gamma_*=(\xi_*,\eta_*)$ (the 
``reference trajectory-control pair'') is a member of $TCP(\Sigma)$};

\item[(\ref{sec-constru-exi}.4)] {\em $\dom(\eta_*)=[a,b]$}.
\een\een

\begin{definition}
Let $U$ be a set, and let $\mathcal{U}$ be a set of $U$-valued controls.
We say that $\mathcal{U}$ is {\bf\em closed under interval intertwining}
if, whenever $J$ is an interval, $J'$ is a subinterval of $J$,
and $\eta_1,\eta_2$ are two members of $\mathcal{U}$ defined
on $J$, it follows that $\eta_3\in\mathcal{U}$, where $\eta_3:J\mapsto U$
is the $U$-control defined by
$$
\eta_3(t)=\left\{\begin{array}{lll}
\eta_1(t)&{\rm if}&t\in J'\\
\eta_2(t)&{\rm if}&t\in J\backslash J'\,\,\,.
\end{array}\right.
$$
\end{definition}


\begin{proposition}
Assume that conditions (\ref{sec-constru-exi}.1-2-3) hold
and suppose in addition that $\mathcal{U}$ contains all
constant $U$-controls defined on $[a,b]$ and is closed
under interval intertwining. Let 
${\bf V}$ be an \mbox{infinitesimal} needle variation packet for
$\Sigma$ at $\gamma_*$. Then there exists a needle \mbox{variation} 
packet in $\mathcal{U}$ associated to ${\bf V}$.
\end{proposition}

\begin{proof}
Let 
\begin{equation}\label{northeast}
{\bf V}\,=\,(\,(u_1,t_1)\,,\,(u_2,t_2)\,,\,\ldots\,,\,(u_m,t_m)\,)\,,\qquad
a\le t_1\le t_2\le\ldots\le t_m<b\,.
\end{equation}

We define functions $\tau_j^-:\rr_{+,m}\mapsto\rr$,  
$\tau_j^+:\rr_{+,m}\mapsto\rr$, for $j=1,\ldots,m$, 
by letting
$\tau_1^-(\veceps)=t_1$, $\tau_1^+(\veceps)=t_1+\eps_1$,
if $\veceps=(\eps_1,\ldots,\eps_m)$, and then
$$
\tau_{j}^-(\veceps)=\max(\tau_{j-1}^+(\veceps),t_j)\,,\qquad
\tau_{j}^+(\veceps)=\tau_{j}^-(\veceps)+\eps_j\qquad{\rm for}\quad
j=2,\ldots,m\,.
$$
We let $I_j(\veceps)=[\tau_j^-(\veceps),\tau_j^+(\veceps)\,[\,$, and 
write $\mathcal{I}(\veceps)=\bigcup_{j=1}^mI_j(\veceps)$.
It is then clear that 
\begin{itemize}
\item[(a)] the $\tau_j^-$ and $\tau_j^+$ are continuous
functions;
\item[(b)] $t_j\le\tau_j^-(\veceps)$ for all $\veceps,j$,
\item[(c)] $\tau_j^-(0)=\tau_j^+(0)=t_j$ for all $j$;
\item[(d)] the intervals $I_j(\veceps)$, for $j=1,\ldots,m$, 
are pairwise disjoint, and ordered from left to right
(that is, $t<t'$ whenever $t\in I_{j-1}(\veceps), t'\in I_{j}(\veceps)$,
$j=2,\ldots,m$);
\item[(e)] $I_j(\veceps)$ has length $\eps_j$ for each
$\veceps,j$.
\end{itemize}
Furthermore, for each $j$ the $I_j(\veceps)$ converge to $\{t_j\}$ as
$\veceps\to 0$, in the sense that the endpoints 
$\tau_j^-(\veceps)$ and $\tau_j^+(\veceps)$ go to $t_j$.

We pick $h$ such that $\tau_m^+(\veceps)\le b$ 
whenever $|\veceps|\le h$. (This is possible because
$\tau_m^+(\veceps)=t_m<b$.)  Then $\mathcal{I}(\veceps)\subseteq[a,b]$
whenever $|\veceps|\le h$. 

We then construct a control variation
$\mathcal{V}=\{\eta^{\veceps}\}_{\veceps\in \mathcal{S}_m(h)}$, 
by defining
\begin{equation}
\eta^{\veceps}(t)=\left\{\begin{array}{lclcl}
\eta_*(t)&\quad&{\rm
  if}&\quad&t\in[a,b]\backslash\mathcal{I}(\veceps)\,,\\
u_j&\quad&{\rm
  if}&\quad&t\in I_j(\veceps)\,.\end{array}\right.
\end{equation}

It clearly follows from our assumptions on $\mathcal{U}$ that the
$\eta^{\veceps}$ belong to $\mathcal{U}$. Also, it is easy  to 
see that the $\eta^{\veceps}$ depend c-continuously 
on the parameter $\veceps$ and $\eta^0=\eta^*$. So 
$\mathcal{V}$ is an $m$-parameter equal-time variation
of $\eta_*$ in $\mathcal{U}$. We now prove that $\mathcal{V}$ is in fact
a needle variation packet in $\mathcal{U}$ associated to ${\bf V}$. 
For this purpose, we verify the two conditions of Definition
\ref{assocpacket}. Condition (2) is trivial (with $\kappa=1$),
since we can take $I(\veceps)$ to be $\mathcal{I}(\veceps)$. 
To verify Condition (1), we pick a test function 
$\varphi\in C^0(U\times[a,b],\rr)$ and compute
\begin{eqnarray*}
\theta_{\varphi,\mathcal{V}}(\veceps)-
\theta_{\varphi,\mathcal{V}}(0)
&=&\int_a^b\Big(\varphi(\eta^\eps(t),t)-\varphi(\eta^0(t),t)\Big)\,dt\\
&=&\sum_{j=1}^m\int_{\tau_j^-(\veceps)}^{\tau_j^+(\veceps)}
\Big(\varphi(u_j,t)-\varphi(\eta_*(t),t)\Big)\,dt\\
&=&\sum_{j=1}^m\eps_j\Big(\varphi(u_j,t_j)-\varphi(\eta_*(t_j+),t_j))\Big)
+E(\veceps)\,,
\end{eqnarray*}
where the error $E(\veceps)$ is given by
$$
E(\veceps)=
\sum_{j=1}^m\int_{\tau_j^-(\veceps)}^{\tau_j^+(\veceps)}
\Biggl(\Big(\varphi(u_j,t)-\varphi(u_j,t_j)\Big)-
\Big(\varphi(\eta_*(t),t)-\varphi(\eta_*(t_j+),t_j)\Big)\Biggr)\,dt\,.
$$
Since $\lim_{t\to t_j}\Big((\varphi(u_j,t)-\varphi(u_j,t_j)\Big)
=\lim_{t\to t_j+}\Big((\varphi(\eta_*(t),t)-\varphi(\eta_*(t_j+),t_j)\Big)=0$,
the error term satisfies $|E(\veceps)|=o(|\veceps|)$ as
$\veceps\to 0$. Hence $\theta_{\varphi,\mathcal{V}}$ is differentiable
at $0$ along $\mathcal{S}_m(h)$, and the differential 
$D\theta_{\varphi,\mathcal{V}}(0)$ is given by (\ref{difisgiv}). 
This completes the proof that Condition
(1) holds. 
\end{proof}

\begin{definition}
If $U$ is a metric space, $I=[a,b]$ is a compact intevral,
and $\eta_*\in C^0([a,b],U)$, an {\bf\em $L^1$ neighborhood
of $\eta_*$ in $C^0([a,b],U)$} is a set $\mathcal{N}$ such
that 
$$
\mathcal{N}=\Big\{\eta\in C^0([a,b],U):\int_a^bd_U(\eta(t),
\eta_*(t)\,dt\le\delta\Big\}
$$
for some positive number $\delta$.\proofend
\end{definition}

\begin{proposition}
Assume that conditions (\ref{sec-constru-exi}.1-2-3) hold
and suppose in addition that $U$ is path-connected, 
$\eta_*$ is continuous, and $\mathcal{U}$ contains an $L^1$ neighborhood
of $\eta_*$ in $C^0([a,b],U)$. Let 
${\bf V}$ be an \mbox{infinitesimal} needle variation packet for
$\Sigma$ at $\gamma_*$. Then there exists a needle \mbox{variation} 
packet in $\mathcal{U}$ associated to ${\bf V}$.
\end{proposition}

\begin{proof}
Let ${\bf V}$ be given by (\ref{northeast}). For each 
$j\in\{1,\ldots,m\}$, choose a continuous path $\gamma_j:[0,1]\mapsto
U$ such that $\gamma_j(0)=\eta_*(t_j)$ and 
$\gamma_j(1)=u_j$. We let 
$$
U_0=\eta_*([a,b])\cup\{u_1,\ldots,u_m\}\cup\bigcup_{j=1}^m\gamma_j([0,1])\,,
$$
so $U_0$ is a compact subset of $U$. We let 
$$
\hat\kappa=\max\{d_U(u,v):u\in U_0,v\in U_0\}\,.
$$

For $\veceps=(\eps_1,\ldots,\eps_m)$, we define 
$\tilde\eps_j=\eps_j+4\eps_j^2$. 
We then define a pair of functions \mbox{$\tau_j^-:\rr_{+,m}\mapsto\rr$},
$\tau_j^+:\rr_{+,m}\mapsto\rr$, for each $j\in\{1,\ldots,m\}$, by
letting \mbox{$\tau_1^-(\veceps)=t_1$, $\tau_1^+(\veceps)=t_1+\tilde\eps_1$},
and then writing
$$
\tau_{j}^-(\veceps)=\max(\tau_{j-1}^+(\veceps),t_j)\,,\qquad
\tau_{j}^+(\veceps)=\tau_{j}^-(\veceps)+\tilde\eps_j\qquad{\rm for}\quad
j=2,\ldots,m\,.
$$
We let 
\begin{eqnarray*}
I_j^1(\veceps)&=&[\tau_j^-(\veceps),\tau_j^-(\veceps)+\eps_j^2\,[\,,\\
I_j^2(\veceps)&=&[\tau_j^-(\veceps)+\eps_j^2,\tau_j^-(\veceps)+
2\eps_j^2\,[\,,\\
I_j(\veceps)^3&=&[\tau_j^-(\veceps)+2\eps_j^2,\tau_j^-(\veceps)
+2\eps_j^2+\eps_j\,[\,,\\ 
I_j^4(\veceps)&=&[\tau_j^-(\veceps)+2\eps_j^2+\eps_j,
\tau_j^-(\veceps)+3\eps_j+\eps_j,[\,,\\
I_j^5(\veceps)&=&[\tau_j^-(\veceps)+3\eps_j+\eps_j,
\tau_j^-(\veceps)+4\eps_j+\eps_j\,[\,=
[\tau_j^-(\veceps)+3\eps_j+\eps_j,\tau_j^+(\veceps)\,[\,,\\
I_j(\veceps)&=&\bigcup_{i=1}^5I_j^i(\veceps)=
[\tau_j^-(\veceps),\tau_j^+(\veceps)\,[\,,\\
\mathcal{I}(\veceps)&=&\bigcup_{j=1}^mI_j(\veceps)\,.
\end{eqnarray*}
It is then clear that 
\begin{itemize}
\item[(a)] the $\tau_j^-$ and $\tau_j^+$ are continuous
functions;
\item[(b)] $t_j\le\tau_j^-(\veceps)$ for all $\veceps,j$,
\item[(c)] $\tau_j^-(0)=\tau_j^+(0)=t_j$ for all $j$;
\item[(d)] the intervals $I_j(\veceps)$, for $j=1,\ldots,m$, 
are pairwise disjoint, and ordered from left to right
(that is, $t<t'$ whenever $t\in I_{j-1}(\veceps), t'\in I_{j}(\veceps)$,
$j=2,\ldots,m$);
\item[(e)] $I_j(\veceps)$ has length $\tilde\eps_j$ for each
$\veceps,j$.
\end{itemize}
Furthermore, for each $j$ the $I_j(\veceps)$ converge to $\{t_j\}$ as
$\veceps\to 0$, in the sense that the endpoints 
$\tau_j^-(\veceps)$ and $\tau_j^+(\veceps)$ go to $t_j$.

We pick $h_0$ such that $\tau_m^+(\veceps)\le b$ 
whenever $|\veceps|\le h_0$. (This is possible because
$\tau_m^+(\veceps)=t_m<b$.)  Then $\mathcal{I}(\veceps)\subseteq[a,b]$
whenever $|\veceps|\le h_0$. 

We then construct a family 
$\mathcal{V}_0=\{\eta^{\veceps}\}_{\veceps\in \mathcal{S}_m(h_0)}$
of $U$-controls by defining
\begin{equation}
\eta^{\veceps}(t)=\left\{\begin{array}{lclcl}
\eta_*(t)&\quad&{\rm
  if}&\quad&t\in[a,b]\backslash\mathcal{I}(\veceps)\,,\\
\eta_*\Big(\tau_j^-(\veceps)-\frac{(t-\tau_j^-(\veceps))
(\tau_j^-(\veceps)-t_j)}{\eps_j^2}
\Big)&\quad&{\rm  if}&\quad&t\in I_j^1(\veceps)\,,\\
\gamma_j\Big(\frac{t-\tau_j^-(\veceps)-\eps_j^2}{\eps_j^2}\Big)&\quad&{\rm
  if}&\quad&t\in I_j^2(\veceps)\,,\\
u_j&\quad&{\rm   if}&\quad&t\in I_j^3(\veceps)\,,\\
\gamma_j\Big(\frac{\tau_j^-(\veceps)+3\eps_j^2+\eps_j-t}{\eps_j^2}\Big)
&\quad&{\rm if}&\quad&t\in I_j^4(\veceps)\,,\\
\eta_*\Big(t_j+\frac{(t-\tau_j^-(\veceps)-3\eps_j^2-\eps_j)
(\tau_j^+(\veceps)-t_j)}{\eps_j^2}
\Big)&\quad&{\rm  if}&\quad&t\in I_j^5(\veceps)\,.\end{array}\right.
\end{equation}
It is then easy to verify that all the controls $\eta^\eps$ are continuous
$U$-valued functions.  Furthermore, if $\veceps\in\mathcal{S}_m(h_0)$,
then 
\begin{eqnarray*}
\int_a^bd_U(\eta^{\veceps}(t),\eta_*(t))dt&=&
\sum_{j=1}^m\int_{I_j(\veceps)}d_U(\eta^{\veceps}(t),\eta_*(t))dt\\
&\le&\hat\kappa\sum_{j=1}^m(\eps_j+4\eps_j^2)\\
&\le&\hat\kappa(|\veceps|+4|\veceps|^2)\,.
\end{eqnarray*}
Pick a positive number $\delta$ such that 
$$
\Big\{\eta\in C^0([a,b],U):\int_a^bd_U(\eta(t),\eta_*(t)\,dt
\le\delta\Big\}\subseteq\mathcal{U}\,,
$$
and then choose $h$ such that $0<h\le h_0$ and 
$\hat\kappa(h+4h^2)\le\delta$, and define 
$$
\mathcal{V}=\{\eta^{\veceps}\}_{\veceps\in \mathcal{S}_m(h)}\,.
$$
It is then clear that the 
$\eta^{\veceps}$ belong to $\mathcal{U}$ if $\veceps\in \mathcal{S}_m(h)$.
Also, it is easy  to see that the $\eta^{\veceps}$ depend c-continuously 
on the parameter $\veceps$ and $\eta^0=\eta^*$. So 
$\mathcal{V}$ is an $m$-parameter equal-time variation
of $\eta_*$ in $\mathcal{U}$. We now prove that $\mathcal{V}$ is in fact
a needle variation packet in $\mathcal{U}$ associated to ${\bf V}$. 
For this purpose, we verify the two conditions of Definition
\ref{assocpacket}. Condition (2) is trivial (with $\kappa=1+4h$),
since we can take $I(\veceps)$ to be $\mathcal{I}(\veceps)$. 

To verify Condition (1), we pick a test function 
$\varphi\in C^0(U\times[a,b],\rr)$, define
$$
\kappa_\varphi=\max\Big\{|\varphi(u,t)|:u\in U_0,t\in[a,b]\Big\}\,,
$$
and compute
\begin{eqnarray*}
\theta_{\varphi,\mathcal{V}}(\veceps)-
\theta_{\varphi,\mathcal{V}}(0)
&=&\int_a^b\Big(\varphi(\eta^\eps(t),t)-\varphi(\eta^0(t),t)\Big)\,dt\\
&=&\sum_{j=1}^m\sum_{k=1}^5
\int_{I_j(\veceps)}\Big(\varphi(\eta^\eps(t),t)-\varphi(\eta^0(t),t)\Big)\,dt\\
&=&\sum_{j=1}^m\eps_j\Big(\varphi(u_j,t_j)-\varphi(\eta_*(t_j),t_j))\Big)
+E(\veceps)\,,
\end{eqnarray*}
where the error $E(\veceps)$ is given by
\begin{eqnarray*}
E(\veceps)&=&E_1(\veceps)+E_2(\veceps)\,,\\
E_1(\veceps)&=&
\sum_{j=1}^m\int_{I_j^3(\veceps)}\Biggl(\Big(\varphi(u_j,t)-
\varphi(u_j,t_j)\Big)-
\Big(\varphi(\eta_*(t),t)-\varphi(\eta_*(t_j),t_j)\Big)\Biggr)\,dt\,,\\
E_2(\veceps)&=&
\sum_{j=1}^m\sum_{k\in\{1,2,4,5\}}\int_{I_j^k(\veceps)}\Big(
\varphi(\eta^\eps(t),t)-\varphi(\eta^0(t),t)\Big)\,dt\,.
\end{eqnarray*}
Since $\lim_{t\to t_j}\Big((\varphi(u_j,t)-\varphi(u_j,t_j)\Big)
=\lim_{t\to t_j}\Big((\varphi(\eta_*(t),t)-\varphi(\eta_*(t_j),t_j)\Big)=0$,
the first error term satisfies $|E_1(\veceps)|=o(|\veceps|)$ as
$\veceps\to 0$. Since the length of $I_j^k(\veceps)$
is $\eps_j^2$ whenever $j=1,\ldots,m$ and $k\in\{1,2,4,5\}$, 
the second error term satisfies 
$|E_2(\veceps)|\le 8\kappa_\varphi|\veceps|^2$, so 
$|E_2(\veceps)|=o(|\veceps|)$ as well. Hence
$|E(\veceps)|=o(|\veceps|)$.

Hence $\theta_{\varphi,\mathcal{V}}$ is differentiable
at $0$ along $\mathcal{S}_m(h)$, and the differential 
$D\theta_{\varphi,\mathcal{V}}(0)$ is given by (\ref{difisgiv}). 
This completes the proof that Condition (1) holds. \proofend
\end{proof}


\ssection{The main theorem}
\label{sec-maintheo}

\begin{theorem}\label{mainot}
{\bf\em (The fixed time separation theorem.)}  
Assume that 
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{mainot}.1)] $n\in\nn$ and 
$\Sigma=(\Omega,U,I,f,\mathcal{U})$ is a
classical $n$-dimensional control system of class $C^{1,0,pc}$;
\item[(\ref{mainot}.2)] $a,b$ are real numbers such that $a<b$; 
\item[(\ref{mainot}.3)] $\gamma_*=(\xi_*,\eta_*)\in TCP(\Sigma)$ 
and $\dom(\eta_*)=[a,b]$; 
\item[(\ref{mainot}.4)] $\mathcal{S}$ is a subset of $\Omega\times\Omega$ 
such that  $\partial\xi_*\in\mathcal{S}$; 
\item[(\ref{mainot}.5)] $\mathcal{C}$ is a limiting approximating cone
to $\mathcal{S}$ at $\partial\xi_*$;
\item[(\ref{mainot}.6)] the sets $\mathcal{R}^\Sigma$ and 
$\mathcal{S}$ are locally separated at $\partial\xi_*$;
\item[(\ref{mainot}.7)] either 
\begin{itemize}
\item[(\ref{mainot}.7.a)] $\mathcal{U}$ contains all constant 
$U$-controls
defined on $[a,b]$ and is closed under interval intertwining,
\end{itemize}
or
\begin{itemize}
\item[(\ref{mainot}.7.b)]  $U$ is path-connected, $\eta_*$ is continuous, 
and $\mathcal{U}$ contains an $L^1$ neighborhood
of $\eta_*$ in $C^0([a,b],U)$.
\end{itemize}\end{itemize}\end{itemize}
Define the {\bf\em Hamiltonian} 
$$
\Omega\times\rr_n\times U\times[a,b]\ni(x,p,u,t)\mapsto 
H(x,p,u,t)\in\rr
$$ by letting 
$$
H(x,p,u,t)\equaldef p\cdot f(x,u,t)\,.
$$
Then there exists a $\pi$ such that 
\begin{itemize}
\item[(\ref{mainot}.a)] $\pi:[a,b]\mapsto\rr_n$ is a continuous map;
\item[(\ref{mainot}.b)] (the "adjoint equation") 
\begin{equation}\label{adjeq-sepft}
\dot\pi(t)=-
\pi(t)\cdot\frac{\partial f}{\partial x}(\xi_*(t),\eta_*(t),t)
\end{equation}
at all points of continuity $t$ of $\eta_*$;
\item[(\ref{mainot}.c)] (the "Hamiltonian maximization condition")
\begin{equation}\label{hamilmin-sepft}
H(\xi_*(t),\pi(t),\eta_*(t),t)=\max\Big\{H(\xi_*(t),\pi(t),u,t):u\in U\}
\end{equation}
at all points of continuity $t$ of $\eta_*$; 
\item[(\ref{mainot}.d)] (the "transversality condition") 
\begin{equation}\label{transve-sepft}
(\pi(a),-\pi(b))\in \mathcal{C}^\dagger\,;
\end{equation}
\item[(\ref{mainot}.d)]  (the "nontriviality condition") 
$\pi(t)\neq 0$ for some $t$ (which implies
that $\pi(t)\neq 0$ for all $t$).
\end{itemize}
\end{theorem}

\medbreak
\noindent{\bf REMARK:}  The adjoint equation (\ref{adjeq-sepft})
can also be written as
\begin{equation}\label{ae-ham-noab}
\dot\pi(t)=-\frac{\partial H}{\partial x}(\xi_*(t),\pi(t),\eta_*(t),t)\,.
\end{equation}
The fact that the dynamical equation $\dot x=f(x,u,t)$ holds when
we substitute $\xi_*(t)$ for $x$ and $\eta_*(t)$ for $u$ is equivalent to
the condition that 
\begin{equation}\label{dyneq-ham-noab}
\dot\xi_*(t)=\frac{\partial H}{\partial p}(\xi_*(t),\pi(t),\eta_*(t),
t)\,.
\end{equation}
The pair of equations (\ref{dyneq-ham-noab}), (\ref{ae-ham-noab}) is equivalent
to the single equation
\begin{equation}\label{full-ham-noab}
\dot\Theta(t)=\vec{H}_{\eta_*(t),t}(\Theta(t))\,,
\end{equation}
evolving in $\Omega\times\rr_n$  (i.e., $(x,p)$ space), where 
$$
\Theta(t)=(\xi_*(t),\pi(t))\,,\qquad H_{u,t}(x,p)\equaldef H(x,p,u,t)\,.
$$ 

\bigbreak

\begin{center}
\fbox{\fbox{\begin{minipage}{11cm}
$\qquad$
\begin{center}
{\bf THE INTRINSIC FORM OF THE ADJOINT EQUATION:}
\end{center}
The adjoint equation (\ref{adjeq-sepft}) 
is equivalent to the following statement:
\begin{center}\begin{minipage}{10.5cm}
{\large\em For every vector field $X$ of class $C^1$ on 
$\Omega$, the equality
$$
\frac{d\,\,}{dt}\Big(\pi(t)\cdot X(\xi_*(t))\Big)
=\pi(t)\cdot[f_{\eta_*(t),t},X](\xi_*(t))
$$
holds, where
$$
f_{u,t}(x)\equaldef f(x,u,t)\,.
$$}
\end{minipage}\end{center}
\end{minipage}}}\end{center}

\noindent{\bf ASSUMPTIONS FOR THEOREM OC-FT:} 
\begin{itemize}
\item[(A1)] $\Omega$ is an open subset of $\rr^n$.
\item[(A2)] $U$ is a metric space.
\item[(A3)] $a,b\in\rr$ and $a<b$.
\item[(A4)] $\Omega\times U\times[a,b]\ni(x,u,t)\mapsto f(x,u,t)\in\rr^n$ and 
$\Omega\times U\times[a,b]\ni(x,u,t)\mapsto L(x,u,t)\in\rr$ are continuous maps
such that 
\begin{itemize}
\item[(A4.a)] the maps $\Omega\ni x\mapsto f(x,u,t)\in\rr^n$ and 
$\Omega\ni x\mapsto L(x,u,t)\in\rr$ are of class $C^1$ for each fixed
$(u,t)$,
\item[(A4.b)] the maps $\Omega\times U\times[a,b]\ni(x,u,t)\mapsto 
\frac{\partial f}{\partial x}(x,u,t)\in\rr^{n\times n}$ and 
$\Omega\times U\times[a,b]\ni(x,u,t)\mapsto\frac{\partial L}{\partial x}(x,u,t)
\in\in\rr_n$ are continuous.
\end{itemize}
\item[(A5)] $\mathcal{U}=C^{0,pc}([a,b],U)$.
\item[(A6)] $S$ is a subset of $\Omega$.	
\item[(A7)] $(\xi_*,\eta_*)\in TCP(\Omega,U,\mathcal{U},f)$
and $\xi_*(b)\in S$. 
\item[(A8)] $C$ is a tangent cone of $S$ at $\xi_*(b)$. 
\item[(A9)] $J(\xi_*,\eta_*)\le J(\xi,\eta)$ for all 
$(\xi,\eta)\in TCP(\Omega,U,\mathcal{U},f)$ such that $\xi(b)\in S$,
where
$$
J(\xi,\eta)\equaldef\int_a^bL(\xi(t),\eta(t),t)\,dt\,.
$$
\end{itemize}

\medbreak
\noindent{\bf THEOREM OC-FT:}  {\bf\em (Optimal control theorem, fixed
time.)} {\em Assume that (A1,2,3,4,5,6,7,8,9) hold. 
Define the {\bf\em Hamiltonian} 
$$
\Omega\times\rr_n\times U\times\rr\times[a,b]\ni(x,p,u,p_0,t)\mapsto 
H(x,p,u,p_0,t)\in\rr
$$
by letting 
$$
H(x,p,u,p_0,t)\equaldef p\cdot f(x,u,t)-p_0(x,u,t)\,.
$$
Then there exists  a pair $(\pi,\pi_0)$ 
such that 
\begin{itemize}
\item[(\#)] $\pi:[a,b]\mapsto\rr_n$ is a continuous map 
and $\pi_0\in\rr$;
\item[(AE)] (the "adjoint equation") 
\begin{equation}\label{adjeq-ocft}
\dot
\pi(t)=-\pi(t)\cdot\frac{\partial f}{\partial x}(\xi_*(t),\eta_*(t),t)
+\pi_0\frac{\partial L}{\partial x}(\xi_*(t),\eta_*(t),t)
\end{equation}
at all points of continuity $t$ of $\eta_*$;
\item[(HM)] (the "Hamiltonian maximization condition")
$$
H(\xi_*(t),\pi(t),\eta_*(t),\pi_0,t)=
\max\Big\{H(\xi_*(t),\pi(t),u,\pi_0,t):u\in U\}
$$ 
at all points of continuity $t$ of $\eta_*$; 
\item[(TC)] (the "transversality condition") 
$$
\pi(b)\in -C^\dagger\,;
$$ 
\item[(NT)] (the "nontriviality condition") 
$(\pi(t),\pi_0)\neq(0,0)$ for some $t$ (which implies
that $(\pi(t),\pi_0)\neq(0,0)$ for all $t$);
\item[(NN)] (the "nonnegativtiy condition") $\pi_0\ge 0$.
\end{itemize}}

\medbreak
\noindent{\bf REMARK:}  The adjoint equation (\ref{adjeq-ocft})
can also be written as
\begin{equation}\label{ae-ham}
\dot\pi(t)=-\frac{\partial H}{\partial x}(\xi_*(t),\pi(t),\eta_*(t),\pi_0,t)\,.
\end{equation}
The fact that dynamical equation $\dot x=f(x,u,t)$ holds when
we substitute $\xi_*(t)$ for $x$ and $\eta_*(t)$ for $u$ is equivalent to
the condition that 
\begin{equation}\label{dyneq-ham}
\dot\xi_*(t)=\frac{\partial H}{\partial p}(\xi_*(t),\pi(t),\eta_*(t),
\pi_0,t)\,.
\end{equation}
The pair of equations (\ref{dyneq-ham}), (\ref{ae-ham}) is equivalent
to the single equation
\begin{equation}\label{full-ham}
\dot\Theta(t)=\vec{H}_{\eta_*(t),\pi_0,t}(\Theta(t))\,,
\end{equation}
evolving in $\Omega\times\rr_n$, i.e., $(x,p)$ space
(the cotangent bundle of $\Omega$), where 
$$
\Theta(t)=(\xi_*(t),\pi(t))\,,
$$
and 
$$
H_{u,p_0,t}(x,p)\equaldef H(x,p,u,p_0,t)\,.
$$


\bigbreak
\begin{center}
\fbox{\fbox{\begin{minipage}{11cm}
$\qquad$
\begin{center}
{\bf THE INTRINSIC FORM OF THE ADJOINT EQUATION:}
\end{center}
{\large\em The adjoint equation (\ref{adjeq-ocft}}) 
is equivalent to the following statement:
\begin{center}\begin{minipage}{10.5cm}
{\large\em For every vector field $X$ of class $C^1$ on 
$\Omega$, the equality
$$
\frac{d\,\,}{dt}\Big(\pi(t)\cdot X(\xi_*(t))\Big)
=\pi(t)\cdot[f_{\eta_*(t),t},X](\xi_*(t))+\pi_0 X
L_{\eta_*(t),t}(\xi_*(t))
$$
holds, where
$$
f_{u,t}(x)\equaldef f(x,u,t)\quad{\rm and}\quad
L_{u,t}(x)\equaldef L(x,u,t)\,.
$$}
\end{minipage}\end{center}
\end{minipage}}}\end{center}
 \def\tA{Vaca.}
 \def\tB{Oxen.}
 \def\tS{Ox.}

\vspace{1cm}
\noindent{\bf ASSUMPTIONS FOR THEOREM OC-VT:} 
\begin{itemize}
\item[(\tA 1)] $\Omega$ is an open subset of $\rr^n$.
\item[(\tA 2)] $U$ is a metric space.
\item[(\tA 3)] $T_*\in\rr$ and $T_*>0$.
\item[(\tA 4)] $\Omega\times U\ni(x,u)\mapsto f(x,u)\in\rr^n$ and 
$\Omega\times U\ni(x,u)\mapsto L(x,u)\in\rr$ are continuous maps
such that 
\begin{itemize}
\item[(\tA 4.a)] the maps $\Omega\ni x\mapsto f(x,u)\in\rr^n$ and 
$\Omega\ni x\mapsto L(x,u)\in\rr$ are of class $C^1$ for each fixed
$u$,
\item[(\tA 4.b)] the maps $\Omega\times U\ni(x,u)\mapsto 
\frac{\partial f}{\partial x}(x,u)\in\rr^{n\times n}$ and 
$\Omega\times U\ni(x,u,t)\mapsto\frac{\partial L}{\partial x}(x,u)
\in\in\rr_n$ are continuous.
\end{itemize}
\item[(\tA 5)] $\alpha>0$ and 
$\mathcal{U}=\bigcup_{T_*-\alpha\le T\le T_*+\alpha}
C^{0,pc}([0,T],U)$.
\item[(\tA 6)] $S$ is a subset of $\Omega$.	
\item[(\tA 7)] $(\xi_*,\eta_*)\in TCP(\Omega,U,\mathcal{U},f)$,
$\eta_*\in C^{0,pc}([0,T_*],U)$, and $\xi_*(T_*)\in S$. 
\item[(\tA 8)] $C$ is a tangent cone to $S$ at $\xi_*(T_*)$. 
\item[(\tA 9)] $J(\xi_*,\eta_*)\le J(\xi,\eta)$ for all 
$(\xi,\eta)\in TCP(\Omega,U,\mathcal{U},f)$ such that $\xi(b)\in S$,
where
$$
J(\xi,\eta)\equaldef\int_0^TL(\xi(t),\eta(t),t)\,dt\,\,
{\rm if}\,\,\eta\in C^{0,pc}([0,T],U)\,..
$$
\end{itemize}

\medbreak
\noindent{\bf THEOREM OC-VT:}  {\bf\em (Optimal control theorem, variable
time.)} {\em Assume that (\tA 1,2,3,4,5,6,7,8,9) hold. 
Define the {\bf\em Hamiltonian} 
$\Omega\times\rr_n\times U\times\rr\ni(x,p,u,p_0,t)\mapsto 
H(x,p,u,p_0)\in\rr$ by letting 
$$
H(x,p,u,p_0)\equaldef p\cdot f(x,u)-p_0(x,u)\,.
$$
Then there exists  a pair $(\pi,\pi_0)$ 
such that 
\begin{itemize}
\item[(\#)] $\pi:[a,b]\mapsto\rr_n$ is a continuous map 
and $\pi_0\in\rr$;
\item[(AE)] (the "adjoint equation") 
\begin{equation}\label{adjeq-ocvt}
\dot\pi(t)=-
\pi(t)\cdot\frac{\partial f}{\partial x}(\xi_*(t),\eta_*(t))
+\pi_0\frac{\partial L}{\partial x}(\xi_*(t),\eta_*(t))
\end{equation}
at all points of continuity $t$ of $\eta_*$;
\item[(HM)] (the "Hamiltonian maximization condition")
$$
H(\xi_*(t),\pi(t),\eta_*(t),\pi_0)=
\max\Big\{H(\xi_*(t),\pi(t),u,\pi_0):u\in U\}
$$ 
at all points of continuity $t$ of $\eta_*$; 
\item[(ZH)] (the "zero Hamiltonian condition")
$$
H(\xi_*(t),\pi(t),\eta_*(t),\pi_0)=0
$$ 
at all points of continuity $t$ of $\eta_*$; 
\item[(TC)] (the "transversality condition") 
$$
\pi(b)\in -C^\dagger\,;
$$ 
\item[(NT)] (the "nontriviality condition") 
$(\pi(t),\pi_0)\neq(0,0)$ for some $t$ (which implies
that $(\pi(t),\pi_0)\neq(0,0)$ for all $t$);
\item[(NN)] (the "nonnegativtiy condition") $\pi_0\ge 0$.
\end{itemize}}


\vspace{1cm}
\noindent{\bf ASSUMPTIONS FOR THEOREM SEP-VT:} 
\begin{itemize}
\item[(\tS 1)] $\Omega$ is an open subset of $\rr^n$.
\item[(\tS 2)] $U$ is a metric space.
\item[(\tS 3)] $T_*\in\rr$ and $T_*>0$.
\item[(\tS 4)] $\Omega\times U\ni(x,u)\mapsto f(x,u)\in\rr^n$ 
is a continuous map such that 
\begin{itemize}
\item[(\tS 4.a)] the map $\Omega\ni x\mapsto f(x,u)\in\rr^n$ 
is of class $C^1$ for each fixed $u$,
\item[(\tS 4.b)] the map $\Omega\times U\ni(x,u)\mapsto 
\frac{\partial f}{\partial x}(x,u)\in\rr^{n\times n}$ is continuous.
\end{itemize}
\item[(\tS 5)] $\alpha>0$ and
$\mathcal{U}=\bigcup_{T_*-\alpha\le T\le T_*+\alpha}
C^{0,pc}([0,T],U)$.
\item[(\tS 6)] $S$ is a subset of $\Omega$.	
\item[(\tS 7)] $\eta_*\in C^{0,pc}([0,T_*],U)$, 
$(\xi_*,\eta_*)\in TCP(\Omega,U,\mathcal{U},f)$,
and $\xi_*(T_*)\in S$. 
\item[(S8)] $C$ is a tangent cone of $S$ at $\xi_*(b)$. 
\item[(S9)] $\mathcal{R}_{\Omega,U,\mathcal{U},f}(\xi_*(0))
\cap S=\{\xi_*(T)$, where
$$
\mathcal{R}_{\Omega,U,\mathcal{U},f}(\xi_*(0))\equaldef
\Big\{\xi(T):\eta\in C^{0,pc}([0,T],U), 
(\xi,\eta)\in TCP(\Omega,U,\mathcal{U},f)\Big\}\,.
$$
\end{itemize}

\medbreak
\noindent{\bf THEOREM SEP-VT:}  {\bf\em (Separation theorem, variable time.)}
{\em Assume that (\tS 1,2,3,4,5,6,7,8,9) hold. 
Define the {\bf\em Hamiltonian} 
$$
\Omega\times\rr_n\times U\times\rr\times[a,b]\ni(x,p,u,p_0,t)\mapsto 
H(x,p,u)\in\rr
$$ 
by letting 
$$
H(x,p,u)\equaldef p\cdot f(x,u)\,.
$$
Then there exists a $\pi$ such that 
\begin{itemize}
\item[(\#)] $\pi:[a,b]\mapsto\rr_n$ is a continuous map;
\item[(AE)] (the "adjoint equation") 
\begin{equation}\label{adjeq-sepvt}
\dot\pi(t)=-
\pi(t)\cdot\frac{\partial f}{\partial x}(\xi_*(t),\eta_*(t))
\end{equation}
at all points of continuity $t$ of $\eta_*$;
\item[(HM)] (the "Hamiltonian maximization condition")
$$
H(\xi_*(t),\pi(t),\eta_*(t))=
\max\Big\{H(\xi_*(t),\pi(t),u):u\in U\}
$$ 
at all points of continuity $t$ of $\eta_*$; 
\item[(ZH)] (the "zero Hamiltonian condition")
$$
H(\xi_*(t),\pi(t),\eta_*(t))=0
$$ 
at all points of continuity $t$ of $\eta_*$; 
\item[(TC)] (the "transversality condition") 
$$
\pi(b)\in -C^\dagger\,;
$$ 
\item[(NT)] (the "nontriviality condition") 
$\pi(t)\neq 0$ for some $t$ (which implies
that $\pi(t)\neq 0$ for all $t$)\,.
\end{itemize}}

\vspace{2cm}
\begin{center}
\fbox{\fbox{
\begin{minipage}{10cm}
$$
\left.\begin{array}{ccc}
{\rm Theorem\,SEP-FT}&\Longrightarrow&{\rm Theorem\,SEP-VT}\\
\Downarrow&\,\,&\Downarrow\\
{\rm Theorem\,OC-FT}&\Longrightarrow&{\rm Theorem\,OC-VT}\end{array}\right.
$$
\end{minipage}}}
\end{center}





\cchapter{The geometric connection\label{secongeo}}

\noindent Both geometry and control are at least as hold as humankind and,
as we shall show in this section, these two areas  have been connected
from their very early days by the thread of 
optimal control, a large component of which is curve minimization.

We have already explained what control theory is about, and at this point 
we may just retain from our earlier discussion the simple statement 
that {\bem the essence of control problems is that they are problems of
design, in which it is desired to produce an object or a behavior so as to
satisfy certain specifications}. 

A similar question could, of course, be asked of ``geometry.'' 
Exactly what is geometry about?   Acording to E.~T.~Bell, in
\cite{belldev}, p.~301, 

\bqrm
Attempts to obtain from professional  geometers a statement of what
geometry is are likely to be only nebulously successful. 
Beyond agreeing, more or less, on ``geometry is the product of a particular
way of thinking,'' few geometers will commit themselves to anything less
hazy.
\eqrm

\noindent If, instead of asking geometers, we just take the simple-minded
approach of looking up the word in a dictionary, we will find
definitions such as 

\bqrm
[Geometry is] the branch of mathematics concerned with the properties and
relations of points, lines, surfaces, and solids in space.
\eqrm

\noindent Actually, this dictionary definition is not too bad
as a first
approximation to what mathematicians might want to say.   They would,
however, supplement it in a few ways: 

\medbreak
\bi
\item{} {\bem first}, by making it clear that 
``lines'' refers not only to ``straight lines'' but to more general curves
(exactly as ``surfaces'' does not mean ``planes'');
\item{} {\bem second}, by incorporating into the definition the
revolutionary 19th century  
discovery that in mathematics there is no such thing as ``space''
but, rather, there is a multiplicity of ``spaces,'' of which the space of
classical Euclidean geometry is one but not the only one;
\item{} {\bem third}, by taking into account the modern understanding that 
the concept of a truly general ``curve'' or ``surface'' or ``solid'' is 
much too broad, and what geometry really studies is some special classes of 
``curves, surfaces and solids,'' 
\item{} {\bem fourth}, by getting rid of the limitation to three
dimensions implicit in the reference to ``curves, surfaces and solids,'' 
and the statement that these objects live in ``space.''
\ei

\medbreak\noindent 
One would then end up with the following modified version of the standard
dictionary definition: 

\bqrm
[Geometry is] the branch of mathematics concerned with the properties and
relations of points and various special curves, surfaces, solids 
and other special higher-dimensional configurations and the 
study of the spaces where these configurations live.
\eqrm

\noindent The new definition is still far from transparent but, despite 
its evident shortcomings, it certainly represents an improvement 
over the previous attempt.  

The main problem with the new definition, which is bound to be 
a problem with {\bem any} definition of geometry, and is the reason 
for Bell's ``nebulous success,''   is the fact that nobody has ever been
able to say exactly what it is that distinguishes the special configurations
studied in geometry from completely general configurations such as 
arbitrary subsets of ``space.'' All that can be said
is that the true feature that singles out
these configurations and makes them special is that they are $\ldots$
geometric. So ``geometry'' is, ultimately, defined in a 
tautological way as the study of ``geometric objects.''

Even though it is far from clear what it is exactly that makes an object
``geometric,'' mathematicians feel and have always felt
that they can tell one when they see
it. For example, if you were to define an ellipse to be a 
curve that, relative to some system of Cartesian coordinates $x,y$ in the
plane has an equation of the form  $ax^2+by^2=1$, with $a<0$ and $b>0$,
then this would not be regarded\Ft{or, at least, would not have been regarded
before the introduction of analytic geometry by Descartes}  as
``geometric.''  If, on the other hand, you take the very same notion, and
characterize it instead as ``a curve which is the locus of all the points 
for which the sum of the distances to two fixed given points is constant,''
then the ``geometricity'' of the ellipse becomes apparent.

Although the extension of the term ``geometric'' appears to have been
fairly clear at each particular time, it has varied with time. 
Since ancient times, nobody ever doubted that straight lines and circles
were ``geometric curves.''  So were ellipses, hyperbolas and parabolas,
since they can be characterized ``geometrically'' as ``loci'' of points
satisfying certain ``geometric'' conditions, as we indicated 
above for the case of the ellipse. 

At the other extreme, the graph of a nonmeasurable function, or of
a continuous nowhere differentiable function, are objects that 
can definitely be talked about in 20th century mathematics, 
but they are  clearly not ``geometric
curves'' by most standards, although occasionally some of these objects can
acquire a ``geometric'' status if they become amenable to a particularly
simple description, as may be the case nowadays with some fractal sets. 

The ``geometricity'' of objects lying between these two extremes has been
somewhat unclear, and has certainly changed through the ages.  For example,
Descartes only called a curve ``geometric'' if it was algebraic\Ft{that is,
defined by a system of polynomial equations; for example, straight lines
and conic sections are ``geometric'' in this sense, but an exponential
curve is not, because $y=e^x$ is not a polynomial equation.}, thereby
denying admission to the world of geometry to transcendental loci such as
sinusoids, cycloids, tractrices, graphs of exponentials, clotoids,
logarithmic spirals, and other natural curves.  This was regrettable,
because some of these curves admit rather simple ``geometric'' descriptions.
(For example, the exponential curve $C$ is characterized by the property
that there is a straight line $L$ for which, if $P$ varies along $C$, and
$Q$, $R$ are the points in $L$ for which the line $PQ$ is tangent to $C$ at
$P$ and the line $PR$ is perpendicular to $L$, then the distance from $R$
to $Q$ is constant.)

With the advent of the calculus, a language became available in terms of
which these curves are simple and natural.  (For example, the exponential
function is the solution of the differential equation 
$y'=y$.)  So the notion of geometricity
shifted, and the geometric world was enlarged by the admission of many new
curves.  

From all this we have to conclude that ``geometricity'' is in the eyes of
the beholder, and the notion of a 
``geometric object'' is undergoing perpetual change. What makes an object
``geometric'' is that {\bem it admits a simple characterization}, with the
understanding that ``simplicity'' here is a time-varying concept, depending
on the mathematical language available at a particular moment in
history. 

Now, one of the most natural ways of singling out objects in a ``simple''
way has been, at least since the classical era, to require that they solve
some ``simple'' minimization or maximization problem\Ft{The key word here,
of course, is ``simple.''  A 20th Century mathematician would pedantically
observe that {\sl every} curve---a Peano curve as well as a straight line
segment---is the solution of {\sl some} minimization problem.}. On the
other hand, an important and very common type of specification for a design
problem is that something be minimized or maximized. In many design problems,
what is desired is to find a curve that achieves a minimum or a maximum of
some functional---that is, to solve an {\bem optimal control problem}---and
the solution frequently turns out to be a very special
curve, which becomes ``geometric'' by virtue of this special nature.
In this way, geometry emerges from the need to solve optimal control problems.

\ssection{Birds, bees, and the birth of geometry and optimal control}

\noindent {\em Homo sapiens} is not the first entity that has engaged 
in problem-solving and arrived at ``geometric'' curves and surfaces
as solutions of design problems. The animal kingdom already
exhibits many examples of optimal control problems and their geometric
solutions\Ft{Naturally, one could even 
go farther in time and attribute an infinite age to our fields, 
based on the fact that they have existed for all eternity in God's mind.
Here, we are only trying to establish the much weaker 
claim that these fields are at least as old as humankind, and for that 
extremely modest purpose we have no need of such a hypothesis.}. 
For example, J.~L.~Coolidge, in his 1940 book \cite{coolidge} (page
1), asked

\bqrm
What is the earliest example of puposive geometrical construction?
\eqrm

\noindent and argued that

\bqrm
This is a perfectly intelligible question if we avoid metaphysical difficulties
over the problem of freedom of the will. If a pigeon fly straight rather 
than on a curve, once it has decided the direction of its home, we are safe
in saying that it has intentionally taken the shortest path and applaud its
sagacity, without inquiring as to whether the real reason was that it had 
an impulse to to straight, and none to bend. 
\eqrm

\noindent Whatever the true reason for the pigeon's behavior might be, 
what the pigeon is actually doing is doing is solving a {\em curve
minimization problem}, and obtaining a straight line segment---the most
important of all ``geometric'' configurations---as the solution\Ft{Nowadays
there exists a vast body of truly fascinating literature showing that many
aspects of animal anatomy, physiology, and behavior, can be explained by
assuming that ``Nature'' is in fact solving optimal control design
problems.  The ``algorithm'' used by Nature to solve these problems, is, of
course, natural selection.}

\gnu{NEED LOTS OF REFERENCES HERE!!!!!}

Bees have also been given credit for exceptional geometrical ability by
many human observers, some of whom appeared all too eager to attribute to
these industrious insects the loftiest of ideals and a profoundly
superanimal wisdom.  For example, Pappus of Alexandria ({\em ca}. 290-{\em
ca}. 350) wrote\Ft{Quoted from Coolidge \cite{coolidge}, p.~1} that

\bqrm
Presumably because they know themselves to be entrusted 
with the task of bringing from the gods, to the accomplished portion of
mankind a share of ambrosia in this form, they do not think it proper to
pour it carelessly on ground or wood, or any other irregular material; but
first collecting the sweets of the most beautiful flowers which grow upon
the earth, they make from them for the reception of the honey, the vessels
which we call honeycombs (with cells) all equal and similar, and contiguous
to one another, and hexagonal in form. And that they have contrived this by
virtue of a certain geometrical forethought we may infer in this way. They
would necessarily think that the figures must be such as to be contiguous 
to one another, that is to say, to have their sides in common in order that
no foreign matter could enter into the interstices between them and 
so defile the purity of the produce. Now there are three 
rectilinear figures which are capable of fulfilling this condition,
I mean regular figures which are equilateral and equiangular, for bees would
have none of figures which are not uniform. \ .\ .\ .\ There beoing then
three figures capable by themselves of exactly filling the space 
about the same point, the bees by reason of their instinctive wisdom 
chose for their construction the figure which has most angles, because they
conceived it would hold more honey than the other two. 
\eqrm

\noindent That is, the bees want their cells to be identical regular
polygons (``for bees would have none of figures which are not uniform'')
and they want them to tile the plane.  This leaves them with only three
options, namely, triangles, squares and hexagons, and the bees cleverly
choose the hexagons because they maximize area (``hold more
honey''). Pappus is not entirely precise here.  It is not true that any
hexagon will have a larger area than any triangle or square. What is true
is that, for regular polygons with $n$ sides {\bem and a given perimeter
$P$}, the enclosed area grows with $n$.  In particular, a hexagon of a
given perimeter will ``hold more honey'' than a triangle or square of the
same perimeter.  But, if we assume that our superintelligent bees start
from the natural observation that ``we are $M$ bees and we have an area $A$
that we can tile with our cells,'' tben it is quite obvious that, whether
they use hexagons or squares or triangles the area of each cell will be
$\frac{A}{M}$.  This means, of course, that if they had used a triangular
tiling then the perimeter of the cells would be larger than that of the
hexagonal cells. It is unclear to us exactly what process of reasoning may
have led Pappus' wise bees to focus on area maximization {\bem subject to a
fixed perimeter}.

On the  other hand, 

 \gnu{Must complete this section}

\ssection{Line segments} \label{sect-rope}
With {\em Homo sapiens}, the new dimension of {\em conscious awareness and
understanding} is added to the relationship between geometry and control.
Many aspects of human behavior---such as walking straight to go from point
$A$ to point $B$---are similar to Coolidge's pigeon example, and can be
explained in terms of animal-like problem-solving. But in addition there
are other aspects that can only be accounted for by attributing to humans
the {\em knowledge} that certain curves solve certain optimal control
problems. 

For example, in ancient Egypt and India the art of {\em rope-stretching}
was practiced in order to produce segments and other geometrical figures,
showing knowledge of the fact that a segment minimizes length.  In Egypt,
according to Democritus, geometric constructions were carried out with the
help of specialized workers---whom Democritus describes as experts in
``composing lines,'' and calls by the Greek word ``harpenodaptai,'' which
means ``rope-strethcers''---by means of pegs and cords. Their skills were
used to build altars and temples, where it was deemed necessary to produce
certain geometric shapes that would obey very precise specifications, such
as being made of perfectly straight segments or perfect circles. These
demands often originated with the Gods, who insisted that an altar or
temple be built so as to meet very strict requirements, failing which their
wrath would be unleashed upon the builders and their communities.  For
example, the problem of doubling the cube appears in a story told in the
Dialogue ``Platonikos'' of Erathosthenes. The Athenians asked the oracle at
Delos what they should do to be liberated from aplague that had struck
Athens in BCE.  The god Apollo replied throught the oracle that they had to
build an altar of exactly the same shape but twice the volume of the
existing one.  

The fascinating story of the ``ritual origins of geometry'' is told in a
series of papers (cf. \cite{seidenberg-a,seidenberg-b,seidenberg-c})
by A.~Seidenberg, and was incorporated by B.~L.~van der Waerden
in his 1983 study \cite{vander} of geometry and algebra in ancient
civilizations. 

The important point for us is that 

 \gnu{Must complete this section}

Curve optimization
problems had been studied and solved at least 
since the Greeks. The oldest of all is of course the one of determining the
shortest path joining two points, whose solution---which must have been
well known since very ancient times---is a straight-line segment. 





\ssection{The parallel postulate}

\noindent {\em Euclid} (ca. 330-275 BCE) stated and used the following
postulate, number V in his list:

\bigskip
\bqsl
If a straight line falling on two straight lines make the interior 
angles on the same side less than two right angles, 
the two straight lines, if prolonged indefinitely, 
meet on that side on which are the angles less than the two right angles.
\eqsl

\bigskip\noindent 
Euclid himself appears to have felt that this postulate was less evident
than the others, as shown by the fact that he avoids using it for as long
as possible. And after Euclid, according to R.~Bonola (\cite{bonola}, p.~2), 

\bigskip
\bqrm
Even the earliest commentators on Euclid's text held that
Postulate V. was not sufficiently evident to be accepted without proof, and
they attempted to deduce it as a consequence of other propisitions.
\eqrm

\bigskip\noindent
In the 17th century, the Fifth Postulate attracted the interest of many
mathematicians, several of whom (for example, P.~A.~Cataldi in 1603
and 1604, G.~A.~Borelli in 1658, G.~Vitale in 1680, and J.~Wallis in
1693, cf. \cite{cataldi,cataldi-ag}, \cite{borelli},cite{vitale},
\cite{wallis}) published works entirely devoted to the subject. 

A major effort to prove Euclid's postulate was by G.~Saccheri (1667-1733),
a contemporary of Johann Bernoulli, whose 1733 book 
\textit{Euclides ab Omni Naevo Vindicatus}---``Euclid 
vindicated from every flaw''---is an attempt to derive the 
postulate by contradiction. Saccheri assumes that all the other postulates 
are true but the Fifth is false, and sets out to  derive a contradiction. 
At one point, he claims to have done it, but his argument is incorrect,
as it depends on extending to the point at infinity certain properties 
that are true at finite distances. (Specifically, Saccheri assumes 
that if two different lines intersect at infinity, then they cannot
have a common perpendicular there, arguing that if they did then this
would be ``contrary to the nature of the straight line.'') 

\ssection{Circles and the isoperimetric problem} \label{subsec-isoper-aist}
The oldest curve optimization problem after the one one about segments
is the {\em isoperimetric problem}, also known as {\em Dido's
problem}, inspired by the mythical story told by Virgil (70-19 B.C) in the
{\em Aeneid} about the foundation of Carthage ({\it c.} 850 B.C.): the
question is (I)~{\em to find the plane curve of a given length that
encloses the 
largest possible area}.  (It is quite easy to see that this is equivalent to
the problem of (II)~{\em to determine the shortest of all plane curves that
enclose a given area}.)  The solution was known by the
Greeks to be the circle, although it took until the 19th century for this
to be proved in a way that meets our contemporary standards of rigor.
Some philosophers  were convinced that {\em this had to
mean something} in the grand design of the universe.
For example, Aristotle believed that ``all motion is either
rectilinear or circular or a combination of the two, because these are the
only `simple movements'$\,$'' (\cite{yourgrau}, p.~4).  He wrote ({\it
ibid.}) that:
\bqrm
\noindent 
if the motion of the heavens is the measure of all movements whatever in
virtue of being alone continuous and regular and eternal, and if, in such
kind, the measure is the minimum, and the minimum movement is the swiftest,
then, clearly, the movement of the heavens must be the swiftest of all
movements. {\mytenit Now of lines which return upon themselves the line
which bounds the circle is the shortest}; and that movement is the swiftest
which follows the shortest line. Therefore, if the heaven moves in a circle
and moves more swiftly than anything else, it must necessarily be
spherical. [Our Italics.]
\eqrm
So Aristotle thought\Ft{Aristotle was far ahead of its time: his 
clarity and precision are up to the most exacting standards of rigor
of the very best postmodern and deconstructionist writing of the 
late 20th century, and the technique of using words such as ``clearly'' and
``therefore'' to justify the most unintelligible nonsequiturs is still
widely used today in 
contemporary scientific publications.} that the motion of heavenly bodies is
determined by some kind of ``minimum'' principle, and the fact that this
motion was circular---as he believed it was---had to have something to do
with the isoperimetric property.  (The italicized assertion in the quotation 
is of course not true unless we impose some extra constraint, e.g. that the
area enclosed by the curve be a fixed constant, since otherwise the
shortest path would be reduced to a single point. So it is reasonable to
assume that Aristotle had the isoperimetric problem---in its 
Version II---in mind.)

It clear that the Greeks understood that these problems were intimately
related to geometry.  However, as we shall argue, one important aspect of
this relationship was not really grasped until the end of the 19th
century. This aspect, that minimization problems can be the source of new
geometries, was not truly appreciated until the era of Gauss and Riemann.
In modern terms, this point is a trivial remark, namely, that one can turn
around the fact that the segment is the shortest path between two points,
and make it the {\em defintion of segment}, using a more general
prescription to measure ``length.''  We will show how the failure to
notice this point---which looks so simple today---made Johann Bernoulli 
miss a discovery that would have have added much to his 
fame and glory.

\ssection{Shortest paths in geometrical optics}\label{subsec-hero}
In the years between the flourishing of science in classical Greece and the
late 17th century, interest in geometric optimization problems grew
for at least two reasons: (i)~because an increasing number of problems of
this kind were solved, and the solutions posed nontrivial challenges to
mathematicians, since in some cases they were far from obvious and some
of the sharpest minds got them wrong, (ii)~because new evidence supported
the intuition that these problems were in some deep way related to the
workings of the physical world.

As we have already pointed out in \S \ref{subsec-era-spec}, 
Hero of Alexandria showed that when a
light ray emitted by an object is reflected by a mirror it follows a
path from the object to the eye which is the shortest of all possible
such paths.  In Hero's setting, which involved a single medium and
therefore a constant speed of light, ``shortest'' was equivalent to
``fastest.''  This was no longer the case in the work of Fermat
(1601-1665), who formulated the general principle that {\sl light rays
follow the fastest---i.e., minimum-time---paths}, and showed how this
explained not only Hero's observation about reflection but also
\snell{}' law of refraction.  (See Scholium \ref{schol-snell},
in page \pageref{schol-snell}; 
we shall soon see that Fermat's principle played a crucial role in Johann
Bernoulli's solution of the brachistochrone problem, cf.
Chapter \ref{controbrach} below, especially \S \ref{subsec-johann-sol} in 
page \pageref{subsec-johann-sol}.) 

These observations were no longer speculative guesses about the distant
heavens, but empirically verifiable assertions about the world around
us, which in fact made nontrivial testable predictions. (For example,
as we indicated \S Scholium \ref{schol-snell},
Descartes had an alternative explanation for \snell{}' law, which 
required that light travel {\it faster} in a denser medium,
whereas Fermat's principle yielded the correct law of refraction only
if light was assumed to travel more slowly in a denser medium. Huygens
(1629-1695) subsequently provided a justification of Fermat's
principle by deriving it from wave optics.)

There is, addition, an important ``geeometric'' aspect to Fermat's 
least time principle. To understand this, let us just retain
from  Hero's theory about reflection the following observation,
the we shall call ``Hero's geometric principle'': 

\bi\item[]\bi
\item[(HGP)] {\sl the path of the light rays is determined by ``pure
geometry,'' and light follows the shortest paths}.
\ei\ei


\noindent Then Fermat's principle can be interpreted in two ways, 
as saying {\sl either} 
\bi
\item[(1)] that Hero's geometric principle no longer applies,
since we now need to take the physical properties of the medium into
account, 
\ei
{\sl or}
\bi
\item[(2)] that Hero's geometric principle still applies, and light rays
still follow the shortest paths, but we now have to define ``length'' in
terms of a new, more refined geometry, in which the physical properties of
space influence the geometry, and shortest paths can be curved or bent.
\ei
Under the second interpretation, Fermat's principle would still say that
light follows a shortest path---i.e., a ``geodesic''---but in a
different, non-Euclidean geometry.
With hindsight, we now know that Interpretation 2 is the most fruitful.
For example, G.~Lochak writes in \cite{lochak}, p.~57--58 [our translation]:

\bqrm
[Fermat's principle] anticipated the greatest revolution to be undergone by
geometry in the future, namely, the discovery of non-Euclidean geometries,
because it showed for the first time that the physical properties of a
medium might cause the shortest paths not to be straight lines.
\eqrm

\medbreak\noindent One might be tempted to suspect that authors like 
Lochak perhaps allow themselves to be carried away by their own 
20th century  hindsight about the geometric  
possibility, and may be putting too much of this hindsight into 17th
century minds, attributing to them ideas that they were far from ready 
to conceive. It appears reasonable to ask 
\bi
\item{} To whom exactly did Fermat's principle ``show'' what Lochak
says it did? 
\item{} Did it ``show'' it to Fermat himself?  
\item{} Did Fermat really think that his minimum-time paths were
``shortest'' rather than just ``fastest''? 
\ei
We prefer to remain agnostic as to what Fermat's actual beliefs may have
been, but point out, in defense of Lochak, that Leibniz, in his 
analysis of the refraction problem in the 1684 \textit{Nova mathodus}
paper, definitely uses the word ``shortest.'' This suggests to us that at
least Leibniz definitely {\bem was} entertaining the possibility that light
rays  in a medium with variable speed of light truly were ``shortest
paths'' in  some new geometry.   

\gnu{HAVE TO CHECK LEIBNIZ' ORIGINAL LATIN TO MAKE SURE.}

\section[$\quad$Johann Bernoulli misses an opportunity to solve the problem
of \null$\quad$Euclid's fifth postulate]
{Johann Bernoulli misses an opportunity to solve the problem
of Euclid's fifth postulate} \label{subsec-noneuclidean} 
As will be explained in Chapter \ref{controbrach}, Johann Bernoulli's
brachistochrone 
problem is equivalent to that of determining the light rays in a 
half-plane $\{(x,y):y\ge 0\}$, where the speed of light is $\sqrt{y}$. 
Johann Bernoulli  was eager to generalize  
his work on the brachistochrone by studying other problems of a similar
kind. One obvious class of such problems is obtained by trying
to determine the paths of light
rays that obey Fermat's minimum-time principle, for a medium with a
variable speed of light given by a function $c(x,y)$ other than 
$\sqrt{y}$.

Johann Bernoulli solved this problem for other functions $c$.  In
particular, he did it for $c(x,y)=y$ and for $c(x,y)=y^{1/3}$, and showed
in both cases that one can obtain simple formulae for the solutions.  When
$c(x,y)=y$, he found that the solution curves are either half-circles with
center in the $x$ axis or vertical half lines.

His result for $c\!=\!y$ put him, unknowingly,
within a hair's breadth from the solution to a problem that had defied the
efforts of mathematicians for 2000 years, and would only be solved 172
years later, in 1868.

The problem was that of the independence from the other axioms of
geometry of Euclid's famous fifth postulate about plane geometry.  The
postulate asserts that if $L$ is a line and $P$ a point not on it then
there exists a unique line through $P$ parallel to $L$, and was regarded
by many authors as much less believable than all the others.
Since Euclid wrote his {\em Elements} (ca. 300 B.C.), 
enormous amounts of energy were spent in efforts to derive it from the
other axioms.  The ``proofs'' were usually by contradiction: one
assumes that all the other postulates hold but the fifth one does not,
and sees what happens, hoping that a contradiction will eventually
emerge.  Although all these proofs were wrong, because at some point
they incorrectly stated that something was a contradiction and
therefore the fifth postulate had been proved, they provided valuable
knowledge.  Indeed, the proofs really were extensive explorations of a
non-Euclidean universe, notwithstanding the authors' firm conviction
that such a universe was a logical impossibility, and their
unawareness that they were in fact laying the foundations for
non-Euclidean geometry.  Eventually, the proofs grew longer and
longer, and the strange properties of a non-Euclidean world became so
familiar and started to make so much sense, that the dominant view
shifted.  In the 1820s, Taurinus~(1794-1874) and Gauss~(1777-1855)
finally realized that the contradiction was nowhere to be found.
Lobachevski~(1793-1856) in 1829, and J. Bolyai~(1802-1860) in 1832,
openly advocated the view that non-Euclidean geometry was simply
another geometry, free from contradiction and worth studying.  This,
however, did not settle the question, for merely asserting with
conviction that a logical system is without contradiction does not
prove that it is.  The problem was finally solved in 1868 by
Beltrami~(1835-1900), who gave a {\em model} for hyperbolic geometry,
a non-Euclidean geometry in which parallels are not unique.  Finally,
in 1882, Poincar\'e~(1854-1912) gave another, much simpler model.

In the Poincar\'e model, the ``plane'' is the open half-plane
$\{(x,y):y>0\}$. A ``line'' is a half-circle with center in the $x$ axis,
or a vertical half line. One can then easily see that given any two points
$A$, $B$ in the ``plane'' there is a unique line joining them. The distance
between $A$ and $B$ is the ``length'' of the unique ``line segment''
joining them, except that ``length'' is not the standard length of a curve.
{\em The ``length'' of an arc is the travel time of a particle that moves
along the arc with velocity $y$.}

In other words, {\em the ``lines'' of the Poincar\'e model are none other than
the solutions of one of  Johann Bernoulli's modified versions
of the brachistochrone problem, with $c=y$.}

The cycloids that occur in the original brachistochrone problem
(cf. Chapter \ref{controbrach} below) have many of the incidence properties of
Euclidean lines (see, for example, statement (*)
in Chapter \ref{controbrach}  below), and therefore
come close to satisfying all Euclid's postulates other than the
fifth. But they do not yet provide a good model for non-Euclidean
geometry because they ``go to infinity'' (i.e.,  approach the $x$ axis)
with finite length.  The half-circles of the Poincar\'e model have all
the desired incidence properties and also have infinite length, so
they furnish a perfect model where all the Euclidean axioms other than
the fifth are true.  A simple way to understand why these two cases
are so different is to observe that in both of them one is studying
minimum-time paths for a control system $\dot x=\varphi(x,y)u$, $\dot
y=\varphi(x,y)v$, with control constraint $u^2+v^2=1$.  When
$\varphi(x,y)\equiv\sqrt{|y|}\,\,$\Ft{\label{absofoot} Since one
usually requires that the state space of a control system be an open
subset of $\rr^n$ or a smooth manifold, it is more natural to regard
the brachistochrone and related problems as defined in the {\mit whole
plane} rather than in a half-plane.  This is why we are using the
absolute value of {\mit y}.  We will pursue this point in 
\ref{refbrach}, where we will study the ``reflected brachistochrnoe 
problem.''}---the cycloid case---the $x$ axis
consists of equilibria, but since the function $\varphi$ is not
Lipschitz it is reasonable to expect that one can join points in the
$x$ axis to points strictly above it in finite time.  When
$\varphi\equiv y$---the circle case---the function $\varphi$ is
Lipschitz, so there is no way to connect in finite time a point not on
the $x$ axis to a point on the $x$ axis.

In modern terms, the circles of Johann Bernoulli's brachistochrone-like
problem are the geodesics of a Riemannian metric on the upper half 
plane given by $ds^2=y^{-2}(dx^2+dy^2)$. It is interesting to note that
Johann Bernoulli had made 
significant contributions to the early stages of the development of these
ideas, by studying the curves of minimum length on a surface, which later
came to be known as ``geodesics.''  (The word ``geodesic'' was coined much
later by Liouville (1809-1892).)

We also note that Johann Bernoulli lived at a time when the problem of
Euclid's parallel postulate was very much at the center of
mathematical research.  For example, G. Saccheri published in 1733 his
book {\em Euclides ab omni naevo vindicatus} (``Euclid vindicated from
every flaw''), where he pushed very far the development of
non-Euclidean geometry, only to end up dismissing it by claiming that
a particular conclusion he had reached had to be rejected because it
was ``repugnant to the nature of a straight line.''

To those of us who know that it can take years of hard work to understand
the solution of an important open problem such as Fermat's last theorem,
the question of the independence of Euclid's postulate is a mathematical 
rarity. It is a problem that mathematicians took about 2200 years to solve,
but the solution is so simple that it can be explained to any undergraduate
with a moderate understanding of plane analytic geometry.

Johann Bernoulli had essentially all the tools he needed to solve the
problem. Why didn't he?   What he lacked was the understanding 
of the {\em geometric} significance of the cycloids and half circles
that he had found as solutions of his minimization problems.    He
certainly saw these curves as ``geometric,''  but failed to see that
they could serve as the ``lines'' of a different geometry. 

\section[$\quad$The missing link: curve minimization as a rich source
of new \null$\quad$geometries]{The missing link: curve minimization as a 
rich source of new geometries} 

``Geometry'' is, tautologically, the study of
``geometric objects,'' but  
it is far from clear what it is exactly that makes an object ``geometric.''
Since ancient times, nobody ever doubted that straight lines and circles
were ``geometric curves.''  So were ellipses, hyperbolas and parabolas,
since they can be characterized ``geometrically'' as ``loci'' of points
satisfying certain ``geometric'' conditions (e.g., for the ellipse, that
the sum of the distances to two given points is a constant) or as conic
sections.  On the other hand, the graph of a nonmeasurable function, or of
a continuous nowhere differentiable function, are clearly not ``geometric
curves'' by most standards, although occasionally some of these objects can
acquire a ``geometric'' status if they become amenable to a particularly
simple description, as may be the case nowadays with some fractal sets. The
geometricity of objects lying between these two extremes has been somewhat
unclear, and has certainly changed through the ages.  For example,
Descartes only called a curve ``geometric'' if it was 
algebraic\Ft{that is, defined by a system of polynomial equations; for example,
straight lines and conic sections are ``geometric'' in this sense, 
but an exponential  curve is not,
because $y=e^x$ is not a polynomial equation.}, thereby
denying admission to the world of geometry to transcendental loci such
as sinusoids, cycloids, tractrices, graphs of exponentials, clotoids,
logarithmic spirals, and other natural curves.  This was regrettable,
because some
of these curves admit rather simple geometric descriptions.
(For example, the  
exponential curve $C$ is characterized by the property that
there is a straight line $L$
for which, if $P$ varies along $C$, and $Q$, $R$ are the points in $L$ for
which the line $PQ$ is tangent to $C$ at $P$ and the line $PR$ is
perpendicular to $L$, then the distance from $R$ to $Q$ is constant.)

With the advent of the calculus, a language became available in terms of
which these curves are simple and natural.  (For example, the exponential
function is the solution of $y'=y$.)  So the notion of geometricity
shifted, and the geometric world was enlarged by the admission of many new
curves.  

So ``geometricity'' is in the eyes of the beholder, and the notion of a
``geometric object'' is undergoing perpetual change. What makes an object
``geometric'' is that it admits a simple characterization, with the
understanding that ``simplicity'' here is a time-varying concept, depending
on the mathematical language available at a particular moment in
history. 

Within this framework, one of the most natural ways of singling out objects
in a ``simple'' way has been, at least since the classical era, to require
that they solve some ``simple'' minimization or maximization problem.  (The
key word here, of course, is ``simple.''  A 20th Century mathematician
would pedantically observe that {\sl every} curve---a Peano curve as well
as a straight line segment---is the solution of {\sl some} minimization
problem.)

As we pointed out above, Greek philosophers such as Aristotle
had a distinct feeling that there had to be a deep meaning to the fact that
the circle was the solution of the isoperimetric problem.

We now understand that even the simple statement that ``a line segment is
the shortest path joining its two endpoints'' is rich in profound
implications.  All it takes is to rewrite it as ``a {\it line segment} is
the shortest distance between two points,'' and it becomes clear that the
length minimization property can be regarded as a {\it definition} of line
segment rather than a proposition that happens to be true about ``real''
line segments.  Once this momentous step is taken, one can move on and
study general geometries in which ``length'' is measured in some other way,
e.g. as travel time in a medium with a variable speed of light, and one can
investigate whether the ``lines'' in this geometry behave like Euclidean
lines.

So maybe Johann Bernoulli could have solved an important 2000-year old
problem, thereby succeeding in his passionate quest for fame and glory
surpassing those of his brother and his son (cf. Scholium \ref{famtime}
in page \pageref{famtime}),
if only it had occurred to him
to look for the hidden meaning of a trivial statement about segments, or
to read it aloud with just a slight change of emphasis.

\ssection{Covariance and invariance, from Lagrange to
Riemann} We saw in Chapter \ref{frompri} that Lagrange was proud of 
having freed
mechanics from geometry by doing everything analytically.  Little did he
know that one of his discoveries anticipated Riemann's revolutionary
creation of differential geometry.

Lagrange allowed the configuration of a collection of $N$ particles to be
represented in terms of completely general systems of $3N$
coordinates. This was a decisive step in the process that would free
geometry from the constraint of having to deal with 3-dimensional Euclidean
space.  ``To appreciate how all was new in this work [the {\em
M\'ecanique Analytique}], let us point out that, when Lagrange described a
motion in terms of a system of Cartesian coordinates in space, this was
such a nontrivial step that he devoted two pages to trying to 
persuade the reader'' (Lochak, \cite{lochak}, p.~49).

He then showed that the  Euler-Lagrange system (\ref{eulagfirst})  has
the same form in ``curvilinear'' or ``generalized'' coordinates, so that 
(\ref{eulagfirst}) is invariant under 
{\em arbitrary}, nonlinear coordinate changes. This was the
first example of what would later become the cornerstone of differential
geometry, namely, the principle of ``coordinate democracy,'' according to
which all coordinate systems, ``rectangular'' as well as ``curvilinear,''
have to be treated equally. 

Lagrange saw the significance of this invariance property:\Ft{For
a striking illustration of what Lagrange had in mind here, think of how 
much simpler it is to derive the equations of motion of a body under an
inverse-square attracting force if one first proves that the motion must 
take place in a plane, and then does the whole calculation in polar
coordinates, using the invariance of the Euler-Lagrange equations.}  
\bqrm
It is perhaps one of the principal advantages of our method that 
it \label{beforehand} expresses the equations of every problem in the 
most simple form
relative to each set of variables and that it enables us to see
beforehand which variables one should use in order to facilitate the
integration as much as possible. [Quoted from \cite{yourgrau}, p.~33.]
\eqrm

In 1854, sixty-six years after the publication of Lagrange's treatise,
Riemann gave his famous lecture, sketching the brilliant ideas that would
become the basis of modern differential geometry.    It turns out that one
of these ideas was precisely that of {\em invariance under general
coordinate changes}, whose importance Lagrange
had presciently anticipated.

Riemann worked on spaces of any number $n$ of dimensions, whose points are
represented, locally, by $n$ coordinates $x^1,\ldots,x^n$. In addition, he
finally took the step of turning around the trivial statement about the
segment being the shortest path between two points, and used the
length-minimization property to {\em define} the concept of a ``line
segment,'' now called a ``geodesic.''

The length of an arbitrary curve $t\mapsto \xi(t)=(x^1(t),\ldots,x^n(t))$
is measured using the {\it arc-length} element
$ds^2=\sum_{ij}g_{ij}(x)dx^idx^j$, 
where $(g_{ij})_{i,j=1}^n$ is an $n$ by $n$ matrix of smooth functions 
of $x$, assumed to be positive definite for each $x$. 

The functions $g_{ij}$ define the metric relative to a particular coordinate
system $(x^1,\ldots,x^n)$.
If we change to another coordinate system $\tilde x$, related to 
$x$ by $\tilde x = \Phi(x)$, $x=\Psi(\tilde x)$, then the coefficients
$\tilde g_{\alpha\beta}$ expressing the metric in the new system are related 
to the $g_{ij}$ by 
\begin{equation}\label{itsays}
\tilde g_{\alpha\beta}(\tilde x)=
\sum_{ij}g_{ij}(x)\cdot\frac{\partial x^i}{\partial \tilde x^\alpha}
\cdot\frac{\partial x^j}{\partial \tilde x^\beta}\,.
\end{equation}
The metric itself should be thought of as a 
``coordinate free object,'' represented in each coordinate system 
by an array $\{g_{ij}\}$ of functions.   These arrays are related by the
transformation (\ref{itsays}).     More generally, {\em covariant 
quantities are 
``objects'' that for each coordinate system are represented 
by an array of functions, in such a way that when one changes coordinates 
these arrays transform according to some specified law.}   There are
different types of such quantities, depending on the transformation law, 
and among them there is a class of quantities called {\em tensors}. 
\begin{remark}{\rm 
In modern differential geometry, one seeks to define covariant
objects intrinsically, rather than as ``arrays of functions that transform 
in a certain way.''    This is a much better and more powerful 
language,  exactly as, say, the language of linear maps on abstract vector
spaces is superior to the language of matrices. But to do justice to 
this idea one needs much more algebra, so we will keep our discussion
simple by sticking with the old-fashioned way of
talking about covariance. \proofend
}\end{remark}
The metric is an example of a tensor, known as the {\bem Riemann metric
tensor}.  A ``flat metric'' is one whose metric tensor, relative to some
coordinate system, is the identity matrix.  (Ordinary Euclidean space is of
course flat.) It is easy to see that if a metric is flat, and one changes
coordinates, then in the new coordinate system the $g$ matrix will in
general no longer be the identity. In other words, the fact that a metric
is flat may not be apparent if the metric is presented in ``curvilinear''
coordinates.  This raises the question of how to characterize flatness
intrinsically, in a coordinate-free way.

Riemann answered the question by introducing the {\bem curvature tensor}
$R$.   In each coordinate system, $R$ has $n^4$ components $R_{ijk}^\ell$,
labelled by four indices.  (The number of truly independent components
is much smaller, because there any many relations among the $R_{ijk}^\ell$.)
When coordinates are changed as before, the transformation law for $R$ is:
\begin{equation}
\tilde R_{\beta\gamma\delta}^\alpha(\tilde x)=
\sum_{ijk\ell}R_{jk\ell}^i(x)\cdot\frac{\partial \tilde x^\alpha}{\partial x^i}
\cdot\frac{\partial x^j}{\partial\tilde x^\beta}\cdot\frac{\partial x^k}
{\partial\tilde x^\gamma}\cdot\frac{\partial x^\ell}{\partial\tilde
x^\delta}\,.
\end{equation}
So $R$ (like any tensor), has the property that  
{\em if $R\equiv 0$ in {\it some} coordinate system, then 
$R\equiv 0$ in every coordinate system.} 
In other words, the vanishing of $R$ is 
an {\it invariant property}.

Moreover, $R$ can be computed by explicit formulas in terms of $g$.
Precisely, one should first find the inverse $g^{-1}$ of the matrix
$g=(g_{ij})$. Using $g^{ij}$ to denote its entries, so that
$\sum_jg^{ij}g_{jk}=\delta_k^i$ (where $\delta_k^i=0$ if $k\neq i$,
$\delta_i^i=1$), it turns out that 
\begin{equation}
R_{jk\ell}^i=\frac{\partial \Gamma_{j\ell}^i}{\partial x^k}-
\frac{\partial \Gamma_{jk}^i}{\partial x^\ell}
+\sum_{s}\Bigl(\Gamma_{sk}^i\Gamma_{j\ell}^s
-\Gamma_{s\ell}^i\Gamma_{jk}^s\Bigr)\,,
\end{equation}
where the $\Gamma_{ij}^k$ are the so-called 
{\em  Christoffel symbols}: 
\begin{equation}\label{forforcs}
\Gamma_{jk}^i=\frac{1}{2}\sum_{\ell}g^{i\ell}\Biggl(
\frac{\partial g_{\ell k}}{\partial x^j}+
\frac{\partial g_{\ell j}}{\partial x^k}-
\frac{\partial g_{jk}}{\partial x^\ell}\Biggr)\,.
\end{equation}
These formulas are of course familiar to those readers acquainted 
with differential geometry, and probably appear impenetrable and unmotivated 
to everybody else.  But it should be clear from them that {\em if a metric is
flat then $R$ vanishes,} because for a flat metric all the $\Gamma_{ij}^k$
vanish. Moreover, since the vanishing of $R$ is an intrinsic property, we
can test a metric $g$ for flatness in {\em any} coordinate system,
by computing $R$ and seeing whether or not it vanishes. Actually, 
things are even nicer, because it turns out that
the vanishing of $R$ is in fact a sufficient as well a necessary condition
for flatness.  

\ssection{Invariants and Klein's Erlangen program}In
1872, F. Klein formulated his  {\em Erlangen program}, whose main idea is
that each geometry is 
characterized by the {\em group of transformations} which preserve the
relevant properties of the geometry.  Associated to such a group there is
usually a collection of {\em invariants} and a class of {\em covariant
objects}. For example, for Euclidean space 
the transformation group is the orthogonal group,  an obvious and
important invariant is the distance between two points, and 
the components of a vector are covariant quantities. 
For the study of general differentiable
manifolds, the natural ``group'' is really a {\em pseudogroup}, namely, the
set of diffeomorphisms, and differential geometry has produced a huge 
zoo of covariant objects: tensors of various kinds (an in particular vector
fields and differential forms), jet spaces, connections, etc. For control
systems, many of the properties of interest (e.g. the structure of 
minimum time trajectories, feedback stabilizability, feedback
linearizability) are invariant under this general pseudogroup, so one
should try to characterize them in a coordinate-free way, using 
invariants. A {\em complete set of invariants} for a class of properties 
is a set of invariants that complete determines these properties. For
example, if we are interested in the solvability of a system 
of linear equations $Ax=b$,  a complete set of invariants 
is provided by a pair of integers, namely, the rank of $A$ and 
the rank of the ``augmented matrix'' $[A;b]$. \proofend


\cchapter[$\quad$Beyond the maximum principle, geometrically]
{Beyond the maximum principle, geometrically\label{beyond}}

\noindent As we have shown, the work on necessary conditions for
optimality began 300 years ago, reached new heights with the maximum
principle, and is still continuing.  But, valuable as they are, the
results of this particular line of research are in a sense
preliminary.  Necessary conditions for optimality are tools for
studying the properties of optimal curves, but in the end it is these
properties that one really wants to understand.  An undue emphasis on
general necessary conditions for optimality for their own sake can
lead to sterile near-tautologies, amounting to the assertion that ``a
necessary condition for a trajectory to be optimal is that it be
optimal,'' skillfully packaged to convey an impression of depth and
novelty.  To avoid falling into this trap, the search for general
conditions has to be coupled with the study of challenging concrete
problems.  These can be general (e.g.  ``what regularity properties,
if any, are true of optimal trajectories for all possible smooth
problems?'')  or can involve the complete understanding of the
solution of a specific problem. We will give examples of both kinds of
questions below.

A large body of work exists on the properties of optimal curves for the
classical calculus of variations.  But, within the much broader context of
optimal control theory, little is known, and very exciting problems
remain open.  For example, the answers to some of the most basic questions
about the general regularity properties of optimal trajectories\Ft{There
are, of course,  many other hard and important questions
not touched upon in this book. For example, for finite-dimensional
deterministic optimal control, there are problems
having to do with optimal synthesis, the
structure of reachable sets, controllability, computational methods,
conditions involving state space constraints, etc. And there is an even
larger area involving infinite-dimensional and stochastic systems.} are 
not known.

Part of the problem here is to find the right language for talking about
these properties. More precisely, one would like to do as in other
mathematical theories where properties of the solutions of a problem 
are sought, and identify {\em invariants} of a problem that can be computed
in terms of the data and provide the desired information about the
solutions.  

\ssection{Covariance and invariance in control theory: the Lie
\mbox{bracket}} By now, the reader may be asking what our discussion of 
covariance and invariance in Chapter \ref{secongeo} had to do with
optimal control. The answer is that it has a lot to do with optimal control
and, more generally, with {\em control}, optimal or not.   

To see this, let us recall that the main aspect that makes optimal control 
stand out and be different from the calculus of variations is the 
presence of a nontrivial dynamics, given by Equation (\ref{dyna}).   
In the special but very important case of an {\em autonomous system},
i.e., when $f$ does not depend on $t$, the dynamics (\ref{dyna}) is really a
family ${\bf f}=\{f(\cdot,u)\}_{u\in U}$ of vector fields, 
parametrized by $u\in U$.  (From now on, we write $f_u(q)$ rather than 
$f(q,u)$, to emphasize that $f_u$ should be regarded as a vector field.)

Vector fields are, of course, covariant\Ft{In the general sense of
covariance defined earlier.  When one draws a distinction between
``covariant'' and ``contravariant'' objects, as is usually done in
differential geometry, vector fields are of course contravariant, and 
covectors such as the ``adjoint vector'' of the maximum principle
are covariant.}  objects, given with respect to each coordinate system by a
set of component functions that transform according to a specific law when
one changes coordinates.  
Differential geometry tells us that, if we want to understand the
properties of a family ${\bf f}$ of vector fields, {\em we should operate in a
``coordinate-democratic'' spirit, following the ideas of Lagrange and Riemann.}
This means that we should look for other {\em covariant quantities} that are
related to ${\bf f}$ in a natural way.

It turns out that, if $f$ and $g$ are two vector
fields, then there is a natural covariant object derived from 
them,\Ft{Besides the trivial ones, such as the linear combinations of
{\mit f} and  {\mit g}.}  namely,
the {\em Lie bracket} $[f,g]$.  This is defined in coordinates 
by  the formula 
\begin{equation}
[f,g]=\frac{\partial g}{\partial x}.f-\frac{\partial f}{\partial x}.g\,.
\end{equation}

\medbreak
\begin{center} 
\fbox{\fbox{\begin{minipage}{11cm}
\begin{center}
THE FORMAL DEFINITION OF THE LIE BRACKET
\end{center}
Let us regard $\rr^n$ as a space of {\sl column} vectors, so the members
of $\rr^n$ are $n$-dimensional columns (or, equivalently,
$n$ by $1$ matrices)
$x=\left[\begin{array}{c}x^1\\\vdots\\x^n\end{array}\right]$
whose entries $x^1,x^2,\ldots,x^n$ are real numbers. 
If $\Omega$ is an open subset of $\rr^n$, 
and $f$ is a vector field  on $\Omega$ (that is, a map
from $\Omega$ to $\rr^n$), we regard $f$ as a column of functions,
so that $f(x)=\left[\begin{array}{c}f^1(x)\\\vdots\\f^n(x)
\end{array}\right]$ for $x\in\Omega$. Also, we write
$D_i$ for the partial differential operator 
$\frac{\partial\phantom{f^1}}{\partial x^i}$. If $f:\Omega\mapsto\rr^n$
is of class $C^1$, we use $Df$ to denote 
the Jacobian matrix of $f$, so $Df$ is a square matrix of
functions, and  $Df(x)=\left[\begin{array}{ccc}
D_1f^1(x)&\ldots&D_nf^1(x)\\
\vdots&\vdots&\vdots\\
D_1f^n(x)&\ldots&D_nf^n(x)\end{array}\right]$
for $x\in\Omega$. 

If $f,g:\Omega\mapsto\rr^n$ are vector fields of class $C^1$, 
the {\em Lie bracket} $[f,g]$ of $f$ and $g$ is another
vector field (of class $C^0$) defined by 
$$
[f,g](x)=Dg(x)\cdot f(x)-Df(x)\cdot g(x)\quad
{\rm for}\quad x\in\Omega\,.
$$ 
If we write $h=[f,g]$, and  $h(x)=\left[\begin{array}{c}h^1(x)\\\vdots\\h^n(x)
\end{array}\right]$, then the components $h^j$ of $h$ are given, for 
$j=1,\ldots,n$, by
$$
h^j(x)=\sum_{i=1}^n\Big((D_ig^j)(x)f^i(x)-
(D_if^j)(x)g^i(x)\Big)\quad
{\rm for}\quad x\in\Omega\,.
$$
\end{minipage}}}
\end{center}
\medbreak

\gnu{Here we must insert a box with the following simple example showing
how the Lie bracket shows up. If you have $m$ dollars in your 
bank account (that is, you have $m(t)$ dollars at time $t$, 
earning interest continuously at a rate $\rho$), and in addition
at time $t$ you spend $s(t)$ dollars per unit time (where 
$s(t)$ can be negative, in which case you are earning money), then
$m$ evolves according to 
$$
\dot m(t)=\rho m(t)-s(t)\,.
$$
The function $t\mapsto s(t)$ is the control. The vector fields 
$m\mapsto f(m)=\rho m$ and $m\mapsto\rho m-1$ describe,
respectively, 
what happpens when you are spending nothing and when you are 
spending $1$ dollar per unit of time. {\em These two behaviors
do not commute,} because if you spend your money initially then this
money will not earn interest, but if if you wait and spend it later
then it will earn interest until you spend it. 
Mathematically, this is reflected 
in the fact that $[f,g]\neq 0$.}

\gnu{At this point we should also include Vel Jurdjevic's famous
pedagogical example: the operations of jumping into a swimming
pool and filling the pool with water do not commute.}


Then $[f,g]$ is a vector field, i.e., a tensor of exactly the
same type as $f$ and $g$.  This suggests that  $[f,g]$ should have an
intrinsic, coordinate-free characterization, and indeed it does, since
\begin{equation}
[f,g](q)=\lim_{t\to 0}\frac{\delta(t)-q}{t^2}\,,
\end{equation}
where
\begin{equation}
\delta(t)=qe^{tf}e^{tg}e^{-tf}e^{-tg}\,.
\end{equation}
(Here $t\mapsto qe^{tX}$ denotes, for a vector field $X$, the integral curve
of $X$ that goes through $q$ when $t=0$.) 

Exactly as in the case of Riemann's metric tensor $g$, a family $\{f_u\}$
of vector fields may have interesting invariant properties which are
immediately detected in a particular coordinate system but are much harder
to ascertain if the system is presented in different coordinates.

But certain things computable in terms of the $f_u$ are independent of the
choice of coordinates. For example, the fact that two vector fields $f$,
$g$ commute (i.e., $[f,g]=0$), is intrinsic.   Moreover, this property
has a very direct control-theoretic meaning:   $[f,g]\equiv 0$ if and only
if, for all control functions $u(\cdot)$, $v(\cdot)$, and all
$\bar q$, the solution of 
\begin{equation}
\dot q(t)=u(t)f(q(t))+v(t)g(q(t))\,,\,\,\,\,\,q(0)=\bar q\,, 
\end{equation}
is always given, for small $t$, by
\begin{equation}
q(t)=\bar qe^{U(t)f}e^{V(t)g}\,, 
\end{equation}
where $U(t)=\int_0^tu(s)ds$, and $V(t)=\int_0^tv(s)ds$.  In other words,
the effect of applying controls $t\mapsto u(t)$ and $t\mapsto v(t)$ during 
a time interval $[0,T]$ is completely determined by two numbers, namely,
the total amounts $U(T)$ and $V(T)$ of $u$-control and $v$-control that
have been applied. It does not matter whether all the $u$-control was 
applied first, or the $v$-control was applied first, or both $u$ and $v$
were evenly distributed on the interval.     By contrast, if $[f,g]\neq 0$,
then more than two numbers are needed to determine the final effect of
applying two controls. For example, if $f$ and $g$ are the vector fields 
in the $x,y$ plane $\rr^3$ with components $(1,0,0)$ and $(0,1,x)$,
respectively, so that $[f,g]$ has components $(0,0,1)$, then 
a simple calculation shows that, to know $q(t)$ given $q(0)$, we need three
independent numbers, namely, $U(t)$, $V(t)$, and the iterated integral 
$W(t)=\int_0^t U(s)v(s)ds$. 

It then turns out that 

\medbreak\noindent{\bf (INV)}\ {\em for real-analytic autonomous control
systems $\dot q=f_u(q)$, the Lie bracket relations among the vector fields
$f_u$ at a point $\bar q$ form a complete set of invariants for the
properties of the time-optimal trajectories through $\bar q\,$.}

\medbreak\noindent The meaning of ``complete set of invariants'' has
already been explained. 
A precise definition of ``Lie bracket relations''
would require more algebra than we care to develop here.  Let us
just show the main idea 
for the case of two vector fields $f$, $g$, without giving a
complete definition. A {\em  Lie bracket
relation} between $f$ and $g$ at a point $q$ is a formal linear combination
of brackets of two indeterminates $F$, $G$, such that, when one plugs in
$f$ and $g$ for $F$, $G$, and evaluates at $q$, the result is $0$. For
example, the expression $F+[F,G]+3[F,[G,F]]$ is a Lie bracket relation
between $f$ and $g$ at $q$ iff $f(q)+[f,g](q)+3[f,[g,f]](q)=0$. 

\begin{remark}{\rm 
Actually, a stronger result is true, but we need more terminology to state
it. For a system $\{f_u\}_{u\in U}$ of real-analytic vector fields,
let $L$ be the Lie algebra of vector fields generated by the $f_u$.  
Then {\em the Lie bracket relations at a 
point $q$ form a complete set of invariants for all properties of the system 
that are preserved by local diffeomorphisms of the maximal integral manifold 
$S$ of $L$ through $q$.}

What this means, in more pedestrian terms, is that the Lie bracket 
relations ``see'' everything that happens on the integral manifold 
but do not see what happens outside.  For example, the property that 
$S$ is a neighborhood of $q$ is {\em not} determined
by the Lie bracket relations, because this property has to do with how 
$S$ sits in relation to the whole space, and is not
about something that happens within $S$ itself. But 
the structure of the time-optimal trajectories through $q$ {\em is}
determined by the Lie bracket relations at $q$, because all the
trajectories through $q$ stay in $S$.

Clearly, (INV) is a special case of this.  Another important special case 
arises when the system has the ``accessibility property'' at $q$, that is, 
$S$ is a neighborhood of $q$. In that case {\em all} local properties 
near $q$ that are diffeomorphism-invariant---for example, asymptotic
stabilizability by state feedback, local equivalence to a linear system,
local feedback linearizability---are determined by the Lie
bracket relations.

Naturally, properties that are not diffeomorphism-invariant---for example,
convexity of reachable sets---need {\em not} be determined by the 
Lie bracket relations. \proofend
}\end{remark}

\ssection{Control, Lie brackets, zoology and medieval bestiaries} For
the reasons described in the preceding remark, Lie brackets play a
crucial role in optimal control theory in many ways and, more
generally, they are the key objects in the theory of nonlinear
finite-dimensional control.  They play a central role in nonlinear
controllability, feedback linearization, and system equivalence.  Just
to give a simple illustration having nothing to with optimality, let
us consider a control theory problem totally analogous to that of the
characterization of flatness for a metric.  In control theory, ``linear
control systems''---i.e., systems of the form $\dot q=Aq+Bu$, evolving
in $\rr^n$, with $u$ taking values in $\rr^m$, and $A$, $B$ matrices of
the appropriate sizes---play a crucial role, comparable in importance to
that of the flat metrics within the class of general Riemannian metrics.
So it is important to know when a system that looks nonlinear because it
is presented in ``curvilinear
coordinates,'' is in fact linear, in the sense that one can make it
linear by changing coordinates.  The answer turns out to be a
criterion involving Lie brackets, analogous to the vanishing of
Riemann's tensor $R$.

Precisely, for a system $\dot
q=f(q)+u_1g_1(q)+\ldots+u_mg_m(q)$, $q\in\rr^n$, with $C^\infty$
vector fields $f$, $g_1,\ldots,g_m$, and a point $\bar q$ in $\rr^n$,
a necessary condition for the system to be equivalent near $\bar q$ to
a linear system is that all the iterated brackets of members of the
set $\{f,g_1,\ldots,g_m\}$ involving two $g_i$'s should vanish
near $\bar q$. Under an extra technical condition (``strong 
accessibility,'' that is, the requirement that the Lie brackets 
${\rm ad}_f^k(g_i)(\bar q)$, $k\ge 0$, $i=1,\ldots,m$ span 
the whole space) this condition is sufficient as well.

Using such a criterion, one can 
see, for example, that a system such as 
\begin{equation}\label{syslin}
\dot x_1=x_2+2x_2x_3\,,\,\,\,\,\,\,\,\,\,\,\,\,\,\dot x_2=x_3-x_1x_2+x_2^3\,,
\end{equation}
$\phantom{aaaaaaaaaaaaaaaaa}\dot x_3=u+x_2^2+x_1x_3-x_1^2x_2+2x_1x_2^3\,,$

\medskip
\noindent which superficially looks very nonlinear, is in fact 
a linear system in disguise, whereas a system such as
\begin{equation}\label{sysnon}
\dot x_1=u_1\,,\,\,\,\dot x_2=u_2\,,\,\,\,\dot x_3=u_1x_2-u_2x_1\,,
\end{equation}
whose nonlinearity appears to be much milder, is in fact highly nonlinear.

So {\bem Lie brackets help us organize our knowledge of nonlinear
systems in terms of their deep structural properties.} This 
way of classifying systems is as superior
to the na\"\i{}ve approach based on a superficial counting of the
number of nonlinearities as the animal taxonomy of modern zoology
is to the classifications found in medieval bestiaries.  Exactly as a
dolphin is structurally closer to a zebra than to a shark---in spite of
the very visible ways, such as  shape and habitat, in which a dolphin
resembles a shark more than a zebra---System (\ref{syslin}) is 
much closer to a linear system than System (\ref{sysnon}).  Lie brackets
enable us to discover this, and then one can find abundant confirmation by
looking at the properties of both systems. (For example, (\ref{syslin}) is
globally stabilizable by a smooth state feedback, but (\ref{sysnon}) is
not even locally stabilizable near any point.) 

\begin{center}
\fbox{{
\begin{minipage}{12cm}
{\bem Control with Lie brackets is to control without Lie brackets
like the taxonomy of modern zoology is to that of a
medieval bestiary.}
\end{minipage}}}
\end{center}

\ssection{Using Lie brackets to understand optimal
trajectories}For optimal control problems, the effort to understand the
properties  
of optimal trajectories in terms of reasonable invariants has begun 
only recently, and most of the work remains to be done. 

Just to give the
flavor of some of these results, we concentrate on one problem that has 
attracted a lot of attention in recent years, namely, that of minimum time
control.  Suppose, for example, that we are looking at minimum time control
of systems in $\rr^n$ of the form $\dot x=f(x)+ug(x)$, with $|u|\le 1$,
where $f$ and $g$ are smooth vector fields.
Here the control space $U$ is the interval $[-1,1]$.  
The Hamiltonian is $H=\varphi+u\psi-p_0$, where $\varphi=\langle p,f(x)\rangle$
and $\psi=\langle p,g(x)\rangle$.  Then $u$ will equal $1$ when 
$\psi>0$, and $-1$ when $\psi<0$.  A na\"\i{}ve genericity argument may
suggest the wrong conclusion that $\psi=0$ is a ``rare'' event, which will 
happen at most at isolated points, so that in fact the optimal trajectories
will be ``bang-bang,'' i.e., such that $u$ is piecewise constant with 
values $1$ and $-1$ and finitely many jumps.  This, however, is quite wrong.
\begin{example}{\rm \label{nobang}
Let $Q=\rr^2$, and assume that $q\!=\!(x,y)\!\in\! Q$ evolves according to 
$\dot x\!=\!1\!-\!y^2$, $\dot y\!=\!u$, $|u|\!\le\! 1$, so 
$f$, $g$ are the vector fields $(x,y)\mapsto(1-y^2,0)$, $(x,y)\mapsto(0,1)$, 
respectively.  (Equivalently, $f=(1\!\!-\!\!y^2)\frac{\partial\,\,}{\partial x}$ 
and $g=\frac{\partial\,\,}{\partial y}$.)  Suppose  we want a trajectory  
that goes from $(0,0)$ to  $(1,0)$ in minimum time.  It is obvious 
that the solution is given by $u(t)\equiv 0$, $y(t)\equiv 0$, 
$x(t)\equiv t$, $0\le t\le 1$, and the optimal time is $1$.  
So the optimal control is ``singular''---i.e., takes values in the 
open interval $]\!\!-\!\!1,1[\,\,$---and therefore drastically fails 
to be bang-bang. And one can easily verify  
that this example is ``stable under perturbations'' in any reasonable sense
of the word.  So it is not true that optimal controls are always, or even 
generically, bang-bang.  \proofend
}\end{example}

In view of our previous remarks, it is to be expected that the {\em
Lie brackets} of the vector fields $f$ and $g$ will play a crucial
role in determining the properties of a solution.  Not surprisingly, a
close analysis of the problem, applying (MP), shows that this is
indeed so.

For example, the ``singular'' control of Example \ref{nobang} is closely
related to a property of the Lie brackets of the vector fields 
$f$ and $g$. Precisely, {\em the singular trajectory exists and is contained 
in the $x$ axis because $[f,g]$ vanishes on the $x$ axis}. 

Since the computation of Example \ref{nobang}, as presented,  does not
explicitly display any
Lie brackets, we will now make the Lie brackets stand out by looking, more
generally, at systems of the form $\dot x=f(x)+ug(x)$, with $|u|\le 1$,
without specializing to a particular pair of vector fields $f$, $g$. 

If one computes the derivative of $\psi$ along an extremal $\gamma$, the
result is $\rho$ evaluated along $\gamma$, where $\rho=\langle
p,[f,g]\rangle$.  So, if $n$ happens to be $2$, and $g$ and $[f,g]$ are
linearly independent at each point, then $\psi$ and $\dot\psi$ cannot
vanish simultaneously, for otherwise $p$ would be orthogonal to $g(x)$ and
$[f,g](x)$, so $p$ would vanish, which is a contradiction. (Recall that
(MP') says that for variable time problems $H$ must vanish. Since
$H=\langle p,f(x)\!+\!u\,g(x)\rangle\!-\!p_0$, it is clear that $p\!=\!0$
implies 
$p_0\!=\!0$, contradicting (NT).)  Now, if a function and its derivative do not
vanish simultaneously, it follows that the zeros of the function are
isolated. In our case, this implies that the optimal controls must be
bang-bang.  So we have proved a simple theorem on the structure of optimal
trajectories: {\em for a minimum time problem in $\rr^2$ arising from a
control system $\dot x=f(x)+ug(x)$, with $|u|\le 1$, if $g$ and $[f,g]$ are
linearly independent at each point, then all optimal trajectories are
bang-bang.}  The reader can verify that in Example \ref{nobang} the Lie
bracket $[f,g]$ vanishes along the $x$ axis, so the trajectory of that
example is contained in the ``singular set'' of points where $g$ and
$[f,g]$ fail to be linearly independent, in perfect agreement with our
theorem.

\ssection{Implementing  Lagrange's covariance idea in optimal 
control} Our elementary arguments about Lie brackets and minimum time
control provide a good illustration of how Lagrange's principle 
of ``coordinate democracy'' applies in the optimal control setting,
and  why a truly intrinsic formulation of the maximum principle, on
manifolds, is needed. 

To understand optimal trajectories, one needs to study certain
functions, such as the ``switching function'' $\psi$.  We can think of
$\psi$ as a ``generalized component'' of the (co)vector $p$, in the sense
that the ordinary components $p_i$ of $p$ are the inner products of $p$
with the coordinate vector fields $\frac{\partial\,\,}{\partial q^i}$, and
$\psi$ is the product of $p$ with the vector field $g$. The adjoint
equation gives us an explicit formula for the functions $\dot p_i$.  A
truly ``democratic'' version of the maximum principle should treat all
generalized components equally, that is, should give us in one swoop the
derivative of {\em every} function $\langle p,X(q)\rangle$, for {\em every}
vector field $X$.

With such a version one could, in each case, work with the generalized
components of $p$ that are most natural for the problem under study,
such as, for example, the ``switching function'' $\langle
p,g(q)\rangle$ for a system $\dot q=f(q)+ug(q)$.  (Another example
will be given in our differential geometric analysis of the
brachistochrone in Chapter \ref{controbrach}.)  This would be, we believe,
true in spirit to Lagrange's idea that it is good to have a method
that ``{\bf\em expresses the equations of every problem in the most
simple form relative to each set of variables and ... enables us to
see beforehand which variables one should use in order to facilitate
the integration as much as possible}.''

An intrinsic version, treating all ``generalized components''
(sometimes known as ``momentum functions'') equally, can be formulated in 
a number of ways, e.g. using Poisson brackets. 
For a general version of the maximum principle on manifolds, 
using generalized components as above (and also for an alternative 
version involving connections along curves), we refer the reader
to \cite{susscofree}. The crucial point of the version given in
\cite{susscofree} is---considering, for simplicity, the case
of the minimum time problem for a control system of the form
$\dot q=f(q,u)$---that the adjoint equation is rephrased as the
following statement (the ``intrinsic adjoint equation''):
\bi
\item[(IAE)] {\sl 
for every smooth vector field $X$, the derivative 
with respect to $t$ of the function 
$t\mapsto \langle p(t),X(q_*(t))\rangle$ equals
$\langle p(t),[f_u,X](q_*(t))\rangle$, where we define
$f_u(q)=f(q,u)$}
\ei
The point of using (IAE) is that it treats the  inner product 
of with $X$---i.e., ``the $X$ component of $p$''---equally 
for all vector fields $X$, thus enabling us to follow Lagrange's
prescription (cf. Page \pageref{beforehand}) 
that one should express 
``the equations of every problem in the most simple
form relative to each set of variables and ... enables us to see beforehand
which variables one should use in order to facilitate the integration as
much as possible,'' by working in each case with the 
functions of the form $\langle p,X\rangle$ that are most suited to the problem.


\ssection{Why the language of Lie brackets is natural
for optimal control} The preceding discussion, based on rather
trivial examples, is meant 
to illustrate what we have in mind when we talk about
``understanding the qualitative properties of trajectories'' and why
we say that the appropriate language for
discussing this question and formulating conditions is that of Lie
brackets of vector fields.  

The reader should think of the following analogy.  When one tries to solve
a simple $2\times 2$ system of linear equations
such as $2x+3y=5$, $6x-2y=4$, one may
fail to notice that the number $2.(-2)-3.6$ is important.  If, however, one
``does it with letters'' (as a Calculus student would say), 
i.e., sets out to solve a {\bem general} system
$a_{11}x+a_{12}y=b_1$, $a_{21}x+a_{22}y=b_2$, then one discovers
immediately that the number $a_{11}a_{22}-a_{12}a_{21}$ determines the
properties of the solutions, so it becomes natural to gives this number a
name such as ``determinant,'' and to look for a generalization of the
concept to higher dimensions.  Similarly, when one applies the maximum
principle to study a particular problem such as the one of Example
\ref{nobang}, one is in fact computing a Lie bracket, although this fact
may go unnoticed.  But when one ``does it with letters,'' e.g. by working
with a 
general problem of the form $\dot x=f(x)+ug(x)$, leaving $f$ and $g$
unspecified, then the Lie bracket $[f,g]$ stands out immediately as the
object that ``determines'' what happens, exactly as the expression
$a_{11}a_{22}-a_{12}a_{21}$ did in the linear algebra example.

It is a general fact that, when we apply the maximum principle to minimum
time problems, we get conclusions of the form ``if such and such thing is
true of the Lie brackets of the vector fields involved, then the optimal
trajectories have such and such properties.''  This can be ascertained in
all kinds of examples, provided that one makes sure that the computations
are always done ``with letters.''  But the occurrence of the brackets is due
to a profound {\em a priori} reason, namely, Statement (INV) of the
previous section.

In recent years, ever since the advent of ``differential geometric control
theory,''  much has been done to determine, in a systematic way, how
structural properties of a system, embodied in their Lie bracket relations,
relate to properties of the optimal trajectories.  For example, it has been
known for a long time that for linear minimum time optimal control 
the ``bang-bang property'' holds. From a general nonlinear perspective,
one can show that the bang-bang property holds when certain Lie brackets 
$B_i$ can be expressed as linear combinations of some other brackets. Linear
systems just happen to be those for which the $B_i$ vanish.  This provides 
an explanation of the linear bang-bang theorem from the nonlinear
perspective, as well as a generalization.

\cchapter[$\quad$Some open problems]{Some open problems\label{threeopen}}

\noindent  We supplement our introduction to what may lie beyond the maximum 
principle by challenging the reader with three open problems. Our challenges
are not like the one issued by Johann Bernoulli, since we do not know the
answers to our questions. 
\bques
Consider a minimum time problem corresponding to a controlled dynamics
of the form $\dot q=f(q,u)$, where (i)~$u$ takes values in a set $U$ which 
is a closed cube, or a ball, or, more generally, a compact subanalytic
subset of $\rr^m$, (ii)~$q$ takes values in an open subset $Q$ of $\rr^n$, 
(iii)~$f$ is real-analytic as a function of $q$ and $u$.   Suppose 
a trajectory-control pair $\gamma_*=(q_*,u_*)$ (with $u$ measurable 
and $U$-valued) of this system, defined on the interval 
$[0,T]$, is a solution of the minimum time problem with initial state 
$\bar q$ and terminal state $\hat q$. Suppose this problem has a unique
solution (except for time translates of $\gamma_*$).  {\em Does it follow that
\bi
\item[(1)] $u$ is real-analytic on the complement of a closed subset 
$C$ of $[0,T]$ of measure zero,
\vspace{-0.2cm} 
\ei
or, perhaps, even more strongly, that 
\bi
\vspace{-0.2cm} 
\item[(2)] $u$ is real-analytic on the complement of a countable closed 
subset $C$ of $[0,T]$?
\ei
}
The requirement that the minimum time problem have a unique solution 
is added to exclude trivial cases where every control is optimal.
For example, for the system $\dot x=1$, $\dot y=u$, $|u|\le 1$, evolving 
in $\rr^2$, it is clear that any trajectory going from
$(x_1,y_1)$ to $(x_2,y_2)$ must do so in time $x_2-x_1$, so all trajectories 
are optimal, and in particular there is no limit to the pathology 
of an optimal control. 

The problem, as stated, is open even for simple systems of the form of the
form $\dot q=f(q)+ug(q)$, $|u|\le 1$, in $\rr^3$. For these simple systems,
the answer in $\rr^2$ is known: (2) follows, and in fact, even more
strongly, $C$ can actually be taken to be finite.  Examples in $\rr^3$ show
that the answer there is not as simple as for $\rr^2$, since there are
problems for which the unique optimal control has infinitely many points 
of discontinuity.

It is known in general that the following weaker conclusion does follow: 
\bi
\item[(0)] {\em $u$ is real-analytic on an open dense subset of
$[0,T]$.}
\ei
This result depends crucially on real analyticity, and is not valid for
$C^\infty$ systems. In fact, if $u_*\!:\!\![0,1]\!\!\mapsto\!\![\!-\!1,1]$
is a completely arbitrary measurable function,
then one can always find
a pair $(f,g)$ of $C^\infty$ vector fields in $\rr^3$ and 
two points $\bar q, \hat q\!\in\!\rr^3$, such that $u_*$ is the unique
solution  
of the problem of steering $\bar q$ to $\hat q$ in minimum time by a
trajectory of $\dot q=f(q)+ug(q)$, $|u|\le 1$.
So for $C^\infty$ systems the only general answer to the question 
whether there are any nontrivial regularity properties shared by 
all uniquely optimal controls is that there are no such properties. 
For the real-analytic case, the answer is ``yes,'' since (0) is an example
of such a property. The question is whether (0) can be strengthened to (1),
or perhaps even (2), or whether there are counterexamples. 

For more details on this problem, the reader may consult
\cite{sussreg1,sussreg2}. \proofend
\eques
\bques
Let $f_1,\ldots,f_m$ be $C^\infty$ vector fields on an open subset $Q$ of 
$\rr^n$.  Let $\Lambda$ be the Lie algebra of vector fields generated by 
$f_1,\ldots,f_m$. Suppose $\{g(q)\!:\!g\!\in\!\Lambda(q)\}\!=\!\rr^n$ for
every $q\!\in\!Q$.  Consider the minimum time problem for the 
control system $\dot q=u_1f_1(q)+\ldots+u_mf_m(q)$,   with 
$u=(u_1,\ldots,u_m)\in U$, where $U$ is the closed unit ball in $\rr^m$. 
Assume $f_1(q),\ldots,f_m(q)$ are linearly independent at each point. 

{\em Are all minimum time trajectories smooth?}

This is the problem of the smoothness of sub-Riemannian minimizers. 
Several papers with incorrect answers have been published. 
By applying the maximum principle, one can easily show that 
the ``normal extremals'' are smooth, but the regularity 
properties of abnormal extremals are not well understood. 
Examples of optimal abnormal extremals have been given in the literature,
but they are all smooth. Nonsmooth abnormal extremals are easy 
to produce, but none so far have been proved to be optimal. 

For more details on this problem and a long list of references, the reader 
is referred to \cite{sussliu}. \proofend
\eques
Finally, for those readers who find the preceding two questions too 
general and abstract, we close with a very concrete problem.
\bques
Consider the minimum time problem for the system in $\rr^4$ whose law of 
evolution is 
$\dot x=\cos{\alpha}$, $\dot y=\sin{\alpha}$, $\dot\alpha=\beta$,
$\dot\beta=u$, $|u|\le 1$.   {\em  What is the structure of the optimal
trajectories? In particular: do  all optimal controls for this problem 
have at most countably many points of nonanalyticity?}

The above system is a ``unicycle'' whose angular acceleration is being
controlled. A simple computation using the maximum principle yields 
three ``basic'' families of optimal trajectories, namely, 
$P$, $N$ and $Z$.  The $P$ and $N$ trajectories, corresponding to 
$u=1$ and $u=-1$, respectively, are clotoids (also known as ``Cornu
spirals'').  The $Z$ trajectories correspond to $u=0$ and only occur when 
$\beta=0$, in which case the unicycle is moving along a straight line 
in the $(x,y)$ plane.    A more careful analysis of the 
conclusions arising from the maximum principle shows that there cannot be a
junction from a $P$ or an $N$ trajectory to a $Z$ trajectory.
On the other hand, the possibility of  ``infinite chattering'' 
(i.e., an alternation of infinitely many $P$ and $N$ pieces,
on intervals $[t_j,t_{j+1}]$, with $t_j<t_{j+1}$ and 
$\lim_{j\to\infty}t_j=\bar t<\infty$) cannot be excluded, and it is natural
to conjecture that such an infinite chattering occurs.  

In fact, if the fixed endpoints  
$\bar q=(\bar{x},\bar{y}, \bar\alpha,\bar\beta)$,
$\hat q=(\hat{x},\hat{y},\hat\alpha,\hat\beta)$  are such that 
$(\bar{x},\bar{y})$ and $(\hat{x},\hat{y})$ are very far,
then one would expect that the optimal trajectory connecting them 
will involve rectilinear motion most of the time.  However, rectilinear
motion alone cannot drive $\bar q$ to $\hat q$ unless the angles 
$\bar\alpha$, $\hat\alpha$ are right---that is, 
$\bar\alpha=\hat\alpha$ and the line $t\mapsto 
(\bar{x}+t\cos{\bar\alpha},\bar{y}+t\sin{\bar\alpha})$ passes through
$(\hat{x},\hat{y})$---and $\bar\beta=\hat\beta=0$. 

So in general one expects that the optimal trajectory will involve an
initial maneuver to position the vehicle in the appropriate line, followed
by a long stretch of rectilinear motion, followed by a terminal maneuver to
get the vehicle to the desired target position.  The initial and terminal
maneuvers should involve concatenations of clotoids, but cannot consist of
{\em finite} concatenations, for a finite set has a first and a last
element, and then there would be a forbidden junction between a clotoid and
a line.  The next simplest
guess is that they involve infinite chattering as described above, but this
has not been proved, and even more pathological possibilities, such as an
uncountable set of control switchings, could in principle occur. 

For a general discussion of infinite chattering, we refer the reader to the
book \cite{zeli}.  Some partial results
on the unicycle with angular acceleration control are given in
\cite{sussangular}.\proofend
\eques

\cchapter[$\quad$The brachistochrone, 300 years ago and today]
{The brachistochrone, 300 years ago and today\label{controbrach}}

\noindent We now look at the brachistochrone problem from various
perspectives, starting with Johann's Bernoulli's analysis,
\Ft{Jakob's 
method was quite different fromm Johann's, and at first sight seemed
clumsier, but in the long run it has turned out to be more akin to the
mainstream ideas of the calculus of variations, Hamilton-Jacobi theory, and
dynamic programming, and is therefore widely considered to be of great
historical importance in the development of optimal control, as shown in
our earlier quotation from Stillwell.  It will not, however, be discussed
here, due to lack of space. Goldstine's book \cite{goldstine} give 
an excellent account.} and then proceeding on to the calculus of variations
and optimal control solutions, and the differential-geometric approach,
showing in each case how the later method 
yields better information than its predecessors.  

%\ssection[$\quad$Johann Bernoulli's challenge]
%{Johann Bernoulli's challenge}\label{subsec-thechal}

\ssection{Johann Bernoulli's challenge}\label{subsec-thechal}

The June 1696
challenge took the form of an ``Invitation to all Mathematicians to
solve a new problem'':
     
\bqsl If in a vertical plane two points $A$ and $B$ are given, then it
is required to specify the orbit $AMB$ of the movable point $M$, along
which it, starting from $A$, and under the influence of its own
weight, arrives at $B$ in the shortest possible time. So that those
who are keen of such matters will be tempted to solve this problem, is
it good to know that it is not, as it may seem, purely speculative and
without practical use. Rather it even appears, and this may be hard to
believe, that it is very useful also for other branches of science
than mechanics.  In order to avoid a hasty conclusion, it should be
remarked that the straight line is certainly the line of shortest
distance between $A$ and $B$, but it is not the one which is traveled
in the shortest time. However, the curve $AMB$---which I shall
disclose 
if by the end of this year nobody else has found it---is very
well known among geometers. 
\eqsl 

\gnu{IN THE EARLIER VERSION WE HAD ``DIVULGE.''
I LIKE ``DISCLOSE'' BETTER. IT'S CERTAINLY BETTER ENGLISH.}

\noindent Later, Johann Bernoulli changed his original plan to disclose 
the solution by the end of 1696 and, following a suggestion of Leibniz,
extended the deadline until Easter 1697.

\gnu{PUT PICTURE OF PAGE WITH JOHANN'S LATIN TEXT AROUND HERE}
\bigbreak

\label{par-sixmaths}
Responses came from the best mathematical minds of the time.  Besides
Johann's own solution, there was one by Leibniz, who had communicated
it to Johann in a letter dated June 16, 1696; another one by Johann's
elder brother Jakob; one by Tschirnhaus; one by l'H\^{o}pital and,
finally, one by Newton, which was presented to the Royal Society on
February 24, 1697 and published, anonymously and without proof, in the
{\em Philosophical Transactions}.  (The identity of the anonymous
author was clear to Johann Bernoulli, since he thought that you can know
{\em ex ungue leonem}, i.e., that you can know the lion from its claws.)  
The May 1697 issue of {\em
Acta Eruditorum} contains articles with Johann's and Jakob's
solutions, contributions by Tschirnhaus\Ft{See Scholium \ref{schol-tschirn}.}
and l'H\^{o}pital\Ft{See Scholium \ref{schol-lhopi}.}, a short
note by Leibniz, remarking that he would not reproduce his own
solution, since it was similar to that of the Bernoullis, and Newton's
anonymous paper, reprinted from the {\em Philosophical Transactions}.

\scholium{The cycloid's double role as the brachistochrone and the tautochrone}

As is apparent from Johann Bernoulli's announcement, he was under the
impression that the problem was new. However, Leibniz knew better: Galileo
in his book on the {\em Two New Sciences} in 1638 formulated the
brachistochrone problem and even suggested that the solution was
a circle.  (Galileo had showed---correctly---that an arc of a circle
always did better than a straight line---except, of course, when $B$
lies on the same vertical line as $A$---but he then wrongly concluded from
this that the circle is optimal.

\comment{One can 
wonder whether he may still have been under the influence of the
Aristotelian belief that all motions take place along circles or
lines\Ft{Kepler had discovered in 1609 that planetary orbits were
elliptical, and this should have had a devastating effect on the ``lines
and circles'' orthodoxy. But Kepler was an astrologer who owed his fame
during his lifetime mainly to his 1597 theory on the relationship between
the planets and the five regular polyhedra.  Kepler's work on planetary
orbits was mostly ignored until about 1645, fifteen years after Kepler's
death and three years after Galileo's death and Newton's birth, and it
wasn't until Newton grew up and derived Kepler's laws from his
inverse-square law of gravitation that Kepler's achievement was duly
appreciated.  Galileo was polite towards Kepler, whom he regarded as an
ally in the struggle for the Copernican system, and thanked him for sending
his work, but he did not read it.}}

Johann Bernoulli considered the fact that Galileo had been wrong on two
counts, in thinking that the catenary was a parabola, and that the
brachistochrone was a circle, as definitive evidence of the superiority of
differential calculus (or the {\em Nova Methodus} as he called this new
set of techniques,  following Leibniz).

He was thrilled by his discovery that the
brachistochrone was a cycloid. This curve had been introduced by Galileo
who gave it its name: {\em related to the circle}. Huygens had discovered
that the the cycloid has a remarkable property: it is the only curve with
the property that a body falling under its own weight is guided by this
curve in such a way that it will oscillate with a period that is
independent of the initial point where the body is released.\Ft{Contrary to
what Galileo thought, the circle has this property only approximately: the
period of oscillation of a pendulum is a function of its amplitude.}
Therefore Huygens called this curve, the cycloid, the {\em tautochrone}
from $\tau\alpha\stackrel{,}{\upsilon}\tau \acute{o}\varsigma$: equal,
and $\chi\rho \acute{o}\nu o\varsigma$: time).

Johann Bernoulli was amazed and somewhat puzzled, it
seems, by the fact that the same curve had these two remarkable properties
related to the time traveled on it by a body falling under its own weight:


\bqrm
Before I end I must voice once more the admiration that I feel for the
unexpected identity of Huygens' tautochrone and my brachistochrone.
I consider it especially remarkable that that this coincidence can take
place only under the hypothesis of Galilei [HJS\&{}JCW: 
that is, ``that the velocities
of falling bodies are to each other as the square roots of the altitudes
traversed''], so the we even obtain from this
a proof of its correctness. Nature alwasy tend to act in the simplest way,
and so here lets one curve serve two different functions, while under any
other hypothesis we would need two curves, one for tautochrone
oscillations, the other for the most rapid fall. If, for example, the
velocities were as the altitudes, then both curves 
would be algebraic, the one a circle, the other one a straight line.
\eqrm

\ssection{Why was the brachistochrone so important in 1696-97?}
\label{subsec-specu}

Perhaps the crucial fact about Johann Bernoulli's question is that
we are told, very clearly, that {\bem the solution is 
a curve that everybody knows, and yet there appears to be no way 
to guess that the solutions is a cycloid by means of some 
intuitive geometric argument.} 

This situation should be compared with that arising in another famous
curve minimization problem, the {\bem isoperimetric problem}. In the
isoperimetric problem, we are asked to find the simple closed curve
that encloses the largest possible area among all curves of a given
length, and the answer also turns out to be ``a curve that everybody
knows,'' namely, a circle. But in this case it is fairly easy to guess
directly that the answer is a circle, even though it is a delicate
matter to give a rigorous proof.  For example, it suffices to observe
that the question is rotationally symmetric, so the answer ought to be
a rotationally symmetric curve, and the only such curve is the
circle\Ft{This argument is not a rigorous proof for at least two
reasons.  First, it assumes existence of solutions. The second and
most serious flaw in the argument is that it requires uniqueness of
the solution, up to translations.  Indeed, it might conceivably happen
that the solution is not unique, and in that case the only conclusion
one can draw from rotational symmetry is that every rotation will map
a solution to a solution, which may be different.  Consider, for
example, the problem of {\em minimaizing}---rather than
maximizing---enclosed area. If we insist on working in the space of
simple closed curves, then the problem has no solution. If we work in
the slightly larger space of closed curves that are not necessarily
simple, then the problem has many solutions, since any curve that
encloses zero area is a solution. Rotations map solutions to
solutions, so the family of all solutions has rotational symmetry, but
the individual solution curves are not rotationally symmetric.}.  This
argument is rather convincing, and makes it quite obvious that the
solution must be a circle.

Therefore, if Johann Bernoulli had challenged ``the best mathematical
minds'' of his time 
\gnu{CHECK TO SEE IF THESE WERE JB'S EXACT WORDS}
\noindent to solve the isoperimetric problem, it would not have made sense for
him to say, as he did in his brachistochrone challenge, that ``the
curve---which I shall disclose if by the end of this year nobody else
has found it---is very well known among geometers.''  Everybody would
have guessed immediately that the mysterious curve was the circle.

In the case of the brachistochrone, {\bem there is no way to
guess}\Ft{Even now, 300 years later, we are not aware of any simple
geometric argument that might lead to the cycloid without using
Calculus.}.  The problem was posed and solved at a time when the
Calculus was being invented, and cannot be solved without using
Calculus.  So it provides an impressive illustration of the power of
the Calculus to solve problems that cannot be solved in any other way,
and to discover truths about the world that can be stated and
understood in ``well known'' terms\Ft{This is an important point.
Compare the brachistochrone, for example, with Newton's 1685 work on
the shape of a body with minimal drag. In this case, the solution also
requires the Calculus, but once we have solved the problem all we find
is a curve defined by some strange formula.  This curve was not ``well
know to geometers,'' and would not have had any particular
significance to mathematicians. who would only retain from going
through Newton's solution the fact that the solution is ``some
curve.''  In the case of the brachistochrone, one does not the
Calculus to explain what the solution is, since cycloids were already
well understood curves, but one does need it to discover that the
solution is a cycloid.}, but can only be arrived at by using this new
mathematical tool.

\ssection{Formulation of the brachistochrone problem} We now
present the formulation of the brachistochrone problem in modern
mathematical language, as it is usually presented in modern books.
(We will make a case below for a different formulation.)
 Choose $x$ and $y$ axes in the plane with the
$y$ axis pointing downwards. Use $(a_1,a_2)$ and $(b_1,b_2)$ to
denote, respectively the coordinates of the end points $A$ and
$B$. Fix a number $E\in\rr$. A path $f:[0,T]\rightarrow \rr^2$,
defined on an interval $[0,T]$, and having components $f_1(t),f_2(t)$,
is said to be a {\em feasible trajectory} (or {\em feasible path}) if

\bi
\item[(i)] $f(0) = A$, $f(T)=B$, and $f$ is Lipschitz continuous,
\item[(ii)] $\frac{1}{2}(|\dot f_1(t)|^2 + |\dot f_2(t)|^2) = E+gf_2(t)$
for almost all $t\in [0,T]$, 
\ei
where $g$ is the gravitational constant.

Condition (i) states that the path $f$ must start at $A$ and end at
$B$. The number $E+ga_2$ is the initial kinetic energy of the body.  (Often,
only the case $E+ga_2=0$ is considered, corresponding to the body being dropped
at time $0$ with zero initial speed.)  Condition (ii) reflects conservation of
energy: at each instant $t$, the kinetic energy of the body must equal its
initial kinetic energy $E+ga_2$ plus the potential energy $g(f_2(t)-a_2)$
due to its loss of
height. (The law that a body which has fallen from a height $h$ has kinetic
energy proportional to $h$ was due to Galileo, and was
well known in Bernoulli's time.)

\medskip

A feasible path $f:[0,T]\rightarrow \rr^2$
is said to be {\em optimal} if there exists no feasible path 
$\tilde f:[0,\tilde T]\rightarrow \rr^2$ for which 
$\tilde T<T$. A {\em brachistochrone} is a 
curve in $\rr^2$ traversed by an optimal feasible 
path, i.e., a subset $B$ of $\rr^2$ of the form 
$$
B= \{(x,y) \in \rr^2:\mbox{ there exists } t\in [0,T^\ast] \mbox{,  such
that } (x,y) = f^\ast(t)\}\,,
$$
where $f^\ast:[0,T^\ast]\mapsto{}\rr^2$ is an optimal feasible path.

Obviously, a feasible path must be entirely contained in the closed
half-plane
\begin{equation}
H_+(\eta)=\{(x,y):y\ge-\eta\}\,,\,\,\,\,\,\,\,{\rm  where\,\,\,\,\,\,}
\eta=\frac{E}{g}\,,
\end{equation}
since, for $f_2(t)$ to be $<-\eta$,  the kinetic energy would have to be
negative, which is of course impossible.

The following two facts are true, though neither one is completely obvious:
\bi
\item[(1)] an optimal feasible path exists for any $A,B\in H_+(\eta)$,
\item[(2)] the path of (1) is unique.  
\ei 
One fact which is truly obvious is that {\sl the solution
cannot always be a straight line}, a possibility that 
Johann Bernoulli rightly warns against.
For example, consider the extreme  case when
$E\!=\!0$ and $a_1\!=\!a_2\!=\!b_2\!=\!0\!\neq\!b_1$. We will show that in
this situation there is a 
feasible path that goes from $A$ to $B$ in finite time. Since 
the straight-line segment $S$ from $A$ to $B$ 
is horizontal, (ii) implies that the speed of motion along $S$ vanishes,
and then it will follow that $S$ cannot be an optimal path, because the motion
along $S$ takes infinite time. To construct a feasible path from $A$ to
$B$ we follow a circle $C$ with center on the $x$ axis.  The circle can
be parametrized by $x$, taking values in the interval $[0,b_1]$. The speed
$v$ of motion along $C$ will be proportional to $\sqrt{y}$, and hence
bounded away from $0$ as long as $x$ is bounded away from $0$ and $b_1$. Near
$A$, we can also use $y$ as a parameter, and then $y\downarrow 0$ as
$x\downarrow 0$, and $v$ behaves like $\frac{dy}{dt}$ and also like
$\sqrt{y}$, so $\frac{dt}{dy}\sim\frac{1}{\sqrt{y}}$, showing that $t$
stays finite as $x$ and $y$ approach $0$, since the function
$y\mapsto{}\frac{1}{\sqrt{y}}$ is integrable near $y=0$.
A similar argument shows that $t$
stays finite near $B$. So $C$ joins $A$ to $B$ in finite time. \proofend

\ssection{Johann Bernoulli's solution}\label{subsec-johann-sol} It 
turns out that
the brachistochrone is a cycloid.  When $E=0$, it is the 
curve described by a point $P$ in a circle that rolls without slipping on
on the $x$ axis, in such a way that $P$
passes through $A$ and then through $B$, without hitting 
the $x$ axis in between.  (It is easy to see that this defines the cycloid
uniquely.)

\bigskip
\[
\epsfbox{cycloid.eps}
\]
\figurecap{The cycloid\label{cycloid-fig}}

Johann Bernoulli's ingenious derivation of the brachistochrone has been the
subject of numerous accounts, but since this event plays a crucial
role in our own tale, we will briefly tell the story again.

Johann Bernoulli based his derivation on Fermat's minimum time principle.
If we imagine for a moment that instead of dealing
with the motion of a moving body we are dealing with a light ray, Condition
(ii) above gives us a formula for the ``speed of light'' $c$ as a function
of position: $c=\sqrt{2E+2gy}$.  From now on, let us change coordinates so
that $E=0$.  (This amounts to moving the $x$ axis upward to a height
$\eta$. Then $\eta$ is now equal to $0$, so our feasible paths live
in the half plane $H_+\equaldef H_+(0)$.) Also, let us rescale---or, if the
reader so 
prefers, ``change our choice of physical units''---so that $2g=1$. Then our
problem is exactly equivalent to that of determining the light
rays---i.e., the minimum-time paths---in a plane medium where the speed of
light $c$ varies continuously as a function of position according to the
formula $c=\sqrt{y}$.  It is at least intuitively clear that, if we
discretize our problem by dividing the half-plane $H_+$ into horizontal
strips $S_k=\{(x,y):y_k\le y\le y_{k+1}\}$ of height $\delta$, for
$k=0,1,\ldots$, where $y_k=k\delta$, and treating $c$ in each strip $S_k$
as a constant $c_k$ (by, say, setting $c_k=\sqrt{y_{k+1}}$), then the light
rays for the discretized problem should approach those for the original
problem as $\delta\downarrow 0$.  

%\bigskip
%\[
%\epsfbox[height=14cm, width=10cm]{layer.eps}
%\]
%\figurecap{Bernouli''s discretization\label{discrete-fig}}

\bigskip
\[
\epsfysize=11cm\epsfbox{layer.eps}
\]
\figurecap{Bernouli''s discretization\label{discrete-fig}}


The light rays of the discretized problem
can be studied using \snell{}' law.  Clearly, the paths will be straight-line
segments within each individual strip, and all that needs to be done is to
determine how these rays bend as they cross the boundary between two
strips. The answer is provided by the laws of optics as developed by
\snell{}, Descartes, Fermat, Leibniz and Huygens.

\bigskip
\[
\epsfxsize=12cm\epsfbox{snellius.eps}
\]
\figurecap{\snell's law\label{snellius-fig}}

\snell{} had observed that, if two media---in the plane---are 
separated by a line $L$, and a light ray is refracted at the
boundary between them, then the  ratio of the sines of the incidence angles 
is constant. Fermat subsequently showed that this is precisely what
happens when light is assumed to follow a minimum-time path. The 
law relating the incidence angles to the velocities of propagation
is due to Leibniz and Huygens, and implies the law of \snell{}. It says that
$$
\frac{\sin \theta_1}{\sin \theta_2} = \frac{v_1}{v_2}\,\,\,\,\,\,\,
\mbox{ or, equivalently,  }\,\,\,\,\,\,
\frac{\sin \theta_1}{v_1} = \frac{\sin \theta_2}{v_2}\,,
$$
where $\theta_i$ is the angle of the line joining $P_i$ to $P'$ 
with the line perpendicular to $L$. 

Johann Bernoulli used this law to conclude that the quantity $\frac{\sin
\theta_k}{\sqrt{y_{k+1}}}$ will be a constant, since 
in each strip $S_k$ the speed of our light ray is $\sqrt{y_{k+1}}$.
Passing to the limit as $\delta\downarrow 0$, we conclude that
the sine of the angle $\theta$ between the tangent to the brachistochrone
and the vertical axis must be proportional to $\sqrt{y}$. Since $\sin
\theta=\frac{dx}{\sqrt{dx^2+dy^2}}$, we find that
$\frac{dx^2}{dx^2+dy^2}=Ky$, where $K$ is a constant. Then
$\frac{dx^2+dy^2}{dx^2}=\frac{1}{Ky}$, i.e., $1+y'(x)^2=\frac{C}{y}$,
where $C=\frac{1}{K}\,$. So the curve described by expressing the
$y$-coordinate of the brachistochrone as a function of its $x$-coordinate
will satisfy the differential equation
\begin{equation}\label{diffeq}
y'(x) = \sqrt{\frac{C-y(x)}{y(x)}}\,,
\end{equation}
with C a constant. The curves parametrized by 
\begin{equation}\label{eqcycloid}
x(\varphi)  =  x_0+\frac{C}{2}(\varphi - \sin \varphi) \,,\,\,\,\,\,\,\,\,\,\,
y(\varphi) =  \frac{C}{2} (1 - \cos \varphi)\,,\,\,\,\,\,\,\,\,\,\,
0\le\varphi\le 2\pi\,, 
\end{equation}
satisfy this differential equation. It is easily seen that these
equations specify the cycloid generated by a point $P$ on a circle of 
diameter $C$ that rolls without slipping on the
horizontal axis, in such a way that $P$ is at $(x_0,0)$ when
$\varphi=0$.   Moreover, it is also easy to check that 
\bi
\item[(*)] given two points $A$ and $B$ in $H_+$ there is exactly one curve
in this family passing through $A$ and $B$.  \ei (The parameter count for
(*) is obviously right: (\ref{eqcycloid}) describes a 2-parameter family of
curves; requiring that the curve pass through $A$ fixes one of the
parameters, and then asking that it pass through $B$ fixes the other.
Proving (*) requires a bit more work, but is still quite easy.) 

\ssection{Remarks on Johann Bernoulli's argument}  The argument that we
have presented is the one of Johann Bernoulli, and 
Equation (\ref{diffeq}) is the one that he wrote in his paper, followed
by the statement ``from which I conclude that the {\em Brachistochrone}
is the ordinary {\em Cycloid}.''   (Johann Bernoulli actually wrote
$dy=dx\sqrt{\frac{x}{a-x}}$, but he was using $x$ for the vertical
coordinate and $y$ for the horizontal one. Cf. \cite{struik}, p.~394.) 
In present day notation, it is customary to adopt the convention that 
the symbol $\sqrt{r}$ stands for the {\em nonnegative} square root 
of $r$, but it is obvious that Johann Bernoulli did not have this in mind. 
What he meant was, clearly, what we would write as 
\begin{equation}\label{plumin}
y'(x) = \pm\sqrt{\frac{C-y(x)}{y(x)}}\,,
\end{equation}
or, equivalently, 
\begin{equation}\label{diffeqac}
y(x)(1+y'(x)^2)={\rm constant}\,.
\end{equation}
In particular, the solution curves should be allowed to have a negative slope,
contrary to what a ``modern'' interpretation of (\ref{diffeq}) might
suggest.  But $y'$ should stay continuous, so that a switching from a 
$+$ to a $-$ solution of (\ref{plumin}) is not permitted. 

Even with the more accurate rewriting (\ref{diffeqac}),  the differential
equation derived by Johann Bernoulli also has spurious solutions,
not given by (\ref{eqcycloid}). Indeed, for any $\bar y\!>\!0$, the 
constant function $y(x)\!=\!\bar y$ is a solution, corresponding to 
$C\!\!=\!\!\bar y$.   More generally, one can take an ordinary cycloid
given by  (\ref{eqcycloid}), follow it up to $\varphi\!\!=\!\!\pi$---so that
$dy/dx\!\!=\!\!0$---then follow the constant solution $y(x)\!\!=\!\!C$ for an
arbitrary time $T$, and 
then continue with a cycloid given by (\ref{eqcycloid}).   
Such paths are, indeed, compatible with Huygens' law of refraction.  

\begin{remark}{\rm 
The existence of the spurious solutions was pointed out by Brook Taylor
(1685-1731) in 1715. 

Taylor is known to mathematicians as the discoverer
of ``Taylor's expansion,''  the inventor of integration by parts, 
and the creator of the calculus of finite differences.  

Like many other mathematicians of his time, he was involved in a priority
dispute with Johann Bernoulli, involving Taylor's 1708 solution of the 
problem of the center of oscillation, that went unpublished until 
1714. \proofend
}\end{remark}

\ssection{The calculus of variations approach} 
The spurious solutions, and all the other problems, such as the apparent
arbitrariness of the requirement that $y'$ be continuous, can be eliminated
in a number of ways. For example, one can prove directly that the spurious
trajectories are not optimal.  Or one can use, as an alternative to Johann
Bernoulli's method, the calculus of variations approach, based on writing
the Euler-Lagrange equation (\ref{eulagfirst}).

It is easy to see that the brachistochrone problem can be put in the 
``standard'' form (\ref{standard}), provided we {\bem postulate}\Ft{With
optimal control, this ``postulate'' becomes a {\em
provable conclusion}, as shown below.}   that 
it suffices to consider curves  in the $x,y$ plane that are graphs of
functions $y\!=\!y(x)$ defined on  the interval 
$[a_1,b_1]$.     Then Constraint~(ii) can be written (taking $E\!=\!0$
and $2g\!=\!1$ as before) as  $dx^2\!+\!dy^2\!=\!y\,dt^2$, which gives
$dt=\frac{\sqrt{dx^2+dy^2}}{\sqrt{y}}=L(y,\dot y)dx$, where
$L(y,u)=\frac{\sqrt{1+u^2}}{\sqrt{y}}$,  and we are using $x$ rather than
$t$ for the time variable, and writing $\dot y$ for $dy/dx$.   So 
Johann Bernoulli's problem becomes that of minimizing the integral
$\int_{a_1}^{b_1}L(y(x),\dot y(x))dx$ subject to 
$y(a_1)=a_2$ and $y(b_1)=b_2$.

This gives the equation 
\begin{equation}\label{truediffeq}
1+y'(x)^2+2y(x)y''(x)=0\,, 
\end{equation}
which is {\em stronger} than (\ref{diffeqac}), since (\ref{diffeqac}) is
equivalent to $y'+{y'}^3+2yy'y''=0$, i.e., to $y'(1+{y'}^2+2yy'')=0$, whose
solutions are those of (\ref{truediffeq}) plus the spurious solutions found
earlier. 

It is easy to verify that the solutions of the Euler-Lagrange equation
(\ref{truediffeq}) are {\em exactly} the curves given by (\ref{eqcycloid}),
without any extra spurious solutions, showing that,
for the brachistochrone problem,  {\bem the
Euler-Lagrange method gives better results than Johann Bernoulli's 
approach.}  

We will soon  see that {\bem optimal control is
even better.}  

\ssection{The brachistochrone as an optimal control
problem} We can formulate Johann Bernoulli's question as an optimal
control problem, in which the motion 
takes place in the $x,y$ plane, and the dynamics is given by
\begin{equation}\label{braccontrol}
\dot{x}=u\sqrt{|y|}\,,\,\,\,\,\,\dot{y}=v\sqrt{|y|}\,, 
\end{equation}
where the control is a 2-dimensional vector $(u,v)$ taking values in
the set $U=\{(u,v):u^2\!+\!v^2\!=\!1\}$ (cf. Footnote \ref{absofoot}). 

The Hamiltonian $H(x,y,u,v,p,q,p_0,t)$  is then given (using
$\alpha={\rm{sgn}}\,y$) by the formula 
$H=(p_1u+p_2v)\sqrt{\alpha y}-p_{0}$, and the application of (NCO) gives
the conditions 
\begin{equation}\label{eqforuv}  
u=\frac{p_1}{|p|}\,, \,\,\,\,\,\,v=\frac{p_2}{|p|}\,, 
\end{equation}
where $|p|=\sqrt{p_1^2+p_2^2}$, as well as the differential equations
\begin{equation}\label{eqpdot}
\dot{p_1}=0\,,\,\,\,\,\,\,\,\dot{p_2}=-\alpha\,\frac{p_1u+p_2v}{2\sqrt{\alpha{}y}}=-\frac{\alpha|p|}{2\sqrt{\alpha y}}\,.
\end{equation} 
(Notice that $|p(t)|\neq 0$. Indeed, (MP') tells us that $H=0$. So 
$|p|=0$ would imply $p_0=0$, contradicting (NT).)

If the constant $p_1$ vanishes, then $\dot x\equiv 0$, so we get a vertical
line. Otherwise, $\dot x$ is always $\neq 0$, showing that we can use 
$x$ to parametrize our solution.  Since 
\begin{equation}
y'(x)=\frac{dy}{dx}=\frac{\dot y}{\dot x}=\frac{v}{u}=\frac{p_2}{p_1}\,,
\end{equation}
we have 
\begin{equation}
1+y'(x)^2=\frac{|p|^2}{p_1^2}
\end{equation}
and 
\begin{equation}\label{eqfory2prime}
y''(x)=\frac{1}{p_1}\cdot\frac{dp_2}{dx}
=\frac{\dot p_2}{p_1\dot x}\,. 
\end{equation}
But (\ref{braccontrol}) and (\ref{eqforuv}) imply that 
\begin{equation}
\dot x=\frac{p_1\sqrt{\alpha y}}{|p|}\,,
\end{equation}
and then (\ref{eqpdot}) and (\ref{eqfory2prime}) yield
\begin{equation}
y''(x)=-\frac{|p|^2}{2 yp_1^2}\,. 
\end{equation}
So 
$2yy''=-\frac{|p|^2}{p_1^2}=-(1+{y'}^2)$, and then 
\begin{equation}
1+{y'}^2+2yy''=0\,, 
\end{equation}
which is exactly 
Equation (\ref{truediffeq}). As we showed before, this leads
to the cycloids, with no ``spurious solutions.''  Notice that this
argument does not involve any 
discretization or any use of \snell{}' law of refraction.

Notice also that in our control argument {\em we have not postulated
that the solution curves could be represented as graphs of functions
$y(x)$.  We have proved it!} (Remember that in the calculus of variations case
this was an extra assumption.)

This is one example showing that, for the
brachistochrone problem, 
{\bem the optimal  control method gives better results than the
classical calculus of variations.}

\ssection{Optimality proofs and the existence problem} 
\label{subsec-existence} 

For a second example showing the superiority of the optimal control
method over 
the classical calculus of variations, for the specific case of the
brachistochrone problem, we turn to the question of the rigorous proof of
the optimality of Bernoulli's cycloids. Clearly, no argument based only on
necessary conditions for optimality will ever prove that a trajectory is
optimal, because it could happen, for example, that there are no optimal
trajectories, in which case the statement that ``every optimal trajectory
is a cycloid'' would be vacuously true.  If we really want
to prove the optimality of Bernoulli's cycloids, an extra step is needed.

In addition, even the classical calculus of variations and optimal control
proofs that every optimal trajectory is a cycloid 
are of limited validity: they  only work for trajectories that are entirely
contained  
in the open half-plane $\hoo_+\!\!=\!\!\{(x,y)\!:\!y\!>\!0\}$.

To give a truly rigorous proof 
of the optimality of the cycloids, valid not only when $A$ and $B$ are 
both in $\hoo_+$ but also when they are in the closed half-plane 
$H_+\!=\!\{x,y)\!:\!y\!\ge\! 0\}$, one needs a more elaborate argument. 

One possibility (cf. Bliss \cite{bliss}) is to use Hamilton-Jacobi theory.
Another, much simpler approach, is to prove first 
the {\em existence} of an optimal trajectory. Indeed, once this is known
for all $A,B\!\in\!H_+$, then it is clear that any optimal trajectory 
$q_*\!:\![0,T]\!\mapsto\!H_+$ going from $A$ to $B$ must be such that its
restriction 
$\xi\restr[\alpha,\beta]$  to
every subinterval $[\alpha,\beta]$ of $[0,T]$ is also optimal 
for the problem $P(\alpha,\beta)$ with endpoints $q_*(\alpha)$,
$q_*(\beta)$.    
(This is a rather obvious point, known in optimal  control theory as the
``principle of optimality,'' and noticed in the case of the
brachistochrone by Jakob Bernoulli.)  If one assumes that 
$q_*(\alpha)$ and  $q_*(\beta)$ belong to $\hoo_+$ for $0<\alpha<\beta<T$, 
then the necessary conditions for optimality give a unique candidate 
for the solution of $P(\alpha,\beta)$, namely, the unique curve in the
family given by 
(\ref{eqcycloid}) that goes through 
$q_*(\alpha)$ and  $q_*(\beta)$. 
So $\xi$ is such that $\xi\restr[\alpha,\beta]$  is given by
(\ref{eqcycloid}) whenever $0<\alpha<\beta<T$, 
and this easily implies that $\xi$ itself is a cycloid
given by (\ref{eqcycloid}).   
To complete the argument, one has to exclude the possibility of a solution 
of the problem with endpoints $A$, $B$ which intersects the $x$ axis
at some other point. This can be done by a simple qualitative argument 
that we omit.
(The key lemma is this: suppose $L$ is a horizontal line in the upper
half-plane $H_+$.   Let $P$, $Q$ be distinct points of $L$. Then the
straight-line segment joining $P$ to $Q$ is faster than any trajectory 
{}from $P$ to $Q$ that lies entirely below $L$.)

So everything hinges upon the existence result.  It turns out that
this is not at all trivial if the brachistochrone problem is
reformulated in terms of the classical calculus of variations, because
the Lagrangian $L(y,u)=\frac{\sqrt{1+u^2}}{\sqrt{y}}$ has a
singularity at $y=0$.\Ft{We thank F. H. Clarke for bringing this point
to our attention.} (Existence theorems for classical calculus of
variations problems that include the brachistochrone can be found in
Cesari \cite{cesari}, \S 14.3 and \S 14.4. But they are not at all simple.)

In the control theory setting, however, the existence question is
easily settled by a completely trivial application of Ascoli's
theorem: given $A,B\in H_+$, let $T$ be the infimum of the times of
all trajectories going from $A$ to $B$; let $\{\xi_j\}$ be a sequence
of trajectories from $A$ to $B$, defined on intervals $[0,T_j]$, such
that $T_j\downarrow T$. Using Ascoli's theorem, extract a subsequence
that converges uniformly to a curve $\xi:[0,T]\mapsto H_+$.  In
principle, $\xi$ is just a trajectory of the ``convexified'' system,
in which the evolution equations are those of (\ref{braccontrol}) but
the control constraint is $u^2+v^2\le 1$ rather than $u^2+v^2=1$.
However, it is easy to see that $\xi$ is a true trajectory of the
nonconvexified system because, if the inequality $u(t)^2+v(t)^2\le 1$
was strict on a set of positive measure, then it would be possible to
find an even faster trajectory from $A$ to $B$ by just reparametrizing
$\xi$.  So we see, once again, that {\em in the control setting a
question that is hard from the point of view of the calculus of
variations can become easy or even trivial.}

The key point in the above argument is that there is at least one
trajectory going from $A$ to $B$ in finite time, as can easily be
verified. The argument would not work if the speed of light was $y$ rather
than $\sqrt{y}$, for in that case there is no path joining $A$ and $B$ in
finite time, unless both $A$ and $B$ belong to $\hoo_+$ or are
equal. 

\ssection{Hestenes' ``variation of the brachistochrone
problem''}\label{hestebrach}  In 1956, M. Hestenes wrote
(in \cite{hestenespap}, p.~64):

\bqrm

... it is interesting to speculate on what
type of brachistochrone problem the Bernoulli brothers might have
formulated had they lived in modern times.  An interesting variation
of the brachistochrone problem can be described as follows:
Consider an idealized rocket ship moving in a vertical plane. The ship is 
to be considered as a particle acted upon by a gravitational force $g$
and a thrust of constant magnitude $F$ and variable inclination $\varphi$ ...
There are no other forces acting on this ship. The pilot steers the ship 
by controlling the inclination $\varphi$, that is, by controlling the 
direction of the thrust. The problem is to find a path of least time
under suitable initial and terminal conditions. Analytically, one
seeks to find among all functions 
$$
x=x(t)\qquad y=y(t)\qquad \varphi=\varphi(t)\qquad 0\le t\le T
$$
subject to the constraints (with unit mass)
$$
\ddot x= F \cos\varphi\qquad\ddot y= F \sin\varphi-g
$$
and with $x$, $y$, $\dot x$, $\dot y$, prescribed at $t=0$ and $t=T$, 
one which minimizes the time of flight $T$. 

It can be shown that the path of least time has the following
interesting property: there is a line $L$ with direction fixed in
space, and with position fixed relative to the ship, such that if a
particle $M$ is moving along $L$ with constant speed, the ship will
traverse a path of least time by keeping the thrust aimed at the
particle $M$.

\eqrm

\noindent Hestenes later on proceeds to solve the problem. 	Let us 
present a solution based on optimal control.  The dynamical equations
can be rewritten as 
$$
\dot x= z\qquad \dot z=F \cos\varphi\qquad\dot y= w\qquad \dot w
=F \sin\varphi-g\,.
$$
If $[0,T]\ni t\mapsto (x_*(t),z_*(t),y_*(t),w_*(t))\in\rr^2$ is an 
optimal trajectory, corresponding to a control function 
$[0,T]\ni t\mapsto\varphi_*(t)\in\rr$, then the maximum principle
yields a nonnegative constant $\pi_0$ and
and a vector-valued function 
$$
[0,T]\ni t\mapsto\pi(t)=(\pi_x(t),\pi_z(t),\pi_y(t),\pi_w(t))\,.
$$
The Hamiltonian is given by 
$$
H(x,z,y,w,\varphi,p_x,p_z,p_y,p_w,p_0)=
p_x\,z+p_z\,F\cos\varphi+p_y\,w+p_w\,(F\sin\varphi-g)-p_0\,.
$$
The adjoint equation becomes
$$
\dot\pi_x=0\qquad\dot\pi_z=-\pi_x\qquad\dot\pi_y=0\qquad
\dot\pi_w=-\pi_y\,.
$$
Then, if we write $\omega(t)=(\pi_z(t),\pi_w(t))$, 
we have 
$$
\omega(t)=\omega_0+t\omega_1\,,
$$
where $\omega_0$ and $\omega_1$ are constant two-dimensional vectors.

The Hamiltonian maximization condition then says that
the vector
$$
{\bf u}(t)=(\cos\varphi(t),\sin\varphi(t))
$$
is given by
\begin{equation}\label{hestenescontrol}
{\bf u}(t)=\frac{\omega(t)}{\|\omega(t)\|}\,.
\end{equation}
The Hamiltonian must vanish along our trajectory.
Therefore, $\pi(t)$ can never vanish, for if $\pi(t)=0$ 
it would follow that $\pi_0=0$, contradicting the nontriviality
condition. Hence $\omega(t)$ cannot vanish identically. 
So $\omega$ is a linear function which is not identically equal to zero.
Therefore there is at most one value of $t$ such that
$\omega(t)=0$. It follows that the function 
${\bf u}$ of (\ref{hestenescontrol}) is well defined 
except possibly at one point $\bar t$. Moreover, if such a $\bar t$ exists,
then $\omega(t)=(t-\bar t)\omega_1$, so 
$$
{\bf u}(t)=\left\{\begin{array}{rll}
-\omega_1&{\rm if}&t<\bar t\\
\omega_1&{\rm if}&t>\bar t.\end{array}\right.
$$
The ``line $L$'' of Hestenes' ``interesting property''
is the line $L_t$ with direction $\omega_1$, with a point $\lambda_t$
having a ``position fixed relative to the ship'' given by
$$
\lambda_t=(x_*(t),y_*(t))+\omega_0\,.
$$
So in fact $L_t$ is moving along with the ship.
If a point $q_t$ on $L_t$ moves with constant velocity $\omega_1$, 
starting at $\lambda_0$ at time $0$, then the point's 
position at time $t$ is $\lambda_t+t\omega_1$. Therefore
the vector from the ship's position to $q_t$ is 
precisely $\omega(t)$. We have shown that the direction of the thrust
is $\omega(t)$, from which Hestenes' ``geometric property'' follows.


\ssection{The reflected  brachistochrone problem}\label{refbrach}  

\noindent So far, our analysis of the brachistochrone problem was
carried out in a half-plane, by looking only for optimal trajectories
that are entirely above the $x$ axis, as in Johann Bernoulli's
original problem.

Let us pursue Hestenes' idea that ``it is interesting to speculate on
what type of brachistochrone problem the Bernoulli brothers might have
formulated had they lived in modern times.''  An obvious question for
the brothers to ask would have been that of finding out what happens
if we work in the whole plane.  This is in fact a much more natural
mathematical setting for the brachistochrone problem.  We shall refer
to this new question as the {\em reflected brachistochrone problem.}
(It is because we were thinking all along of this extension that we
wrote $\sqrt{|y|}$ rather that $\sqrt{y}$ in (\ref{braccontrol})). 

Solving this more general problem amounts to
finding  the light rays when the medium is {\sl the whole
plane}, and the speed of light is $\sqrt{|y|}$.  Notice that this
problem is ``completely controllable,'' in the sense that any two
points $A$, $B$ of $\rr^2$, even if they lie on opposite sides of the
$x$ axis, can be joined by a feasible path.  The right-hand side of
(\ref{braccontrol}) vanishes along the $x$ axis, but this does not
prevent the existence of feasible paths crossing the $x$-axis, because
the function $\sqrt{|y|}$ is not Lipschitz near the $x$ axis.  (If the
function was Lipschitz, then by the usual uniqueness theorem of
ordinary differential equations, every solution going through a point
on the $x$ axis would have to be a constant curve.)  However the same
non-Lipschitz feature that makes the system controllable also renders
the maximum principle inapplicable, in its classical and nonsmooth
versions, including the Lojasiewicz version, since all these require a
Lipschitz reference vector field.

Suppose we want to find the light rays for a medium which is the
whole plane, with ``speed of light'' equal to $\sqrt{|y|}$.   Can the necessary
conditions help us find the solution?  
Suppose, for example, that we want to find an optimal trajectory from $A$
to $B$, where $A$ lies in the upper half-plane and $B$ is in the lower
half-plane. Then one can show, first of all, that an optimal trajectory
$\xi$ exists, using Ascoli's theorem as before. Next, using the usual necessary
conditions for optimality, e.g. the Euler-Lagrange equation or the
classical version of the maximum principle, one shows that any portion of 
an optimal curve which is entirely contained in the closed upper half plane
or in the closed lower half plane is a cycloid given by (\ref{eqcycloid}),
or a reflection or such a cycloid with respect to the $x$ axis. 
Next, one sees that $\xi$ cannot traverse the $x$ axis more than once. 
(This requires an elementary qualitative lemma that we leave as exercise.) 
So we know that $\xi$ consists of a cycloid
$\Gamma_1$  going from $A$ to a point $X$ 
in the $x$ axis, followed by a second cycloid $\Gamma_2$ 
going from $X$ to $B$.  It remains to find $X$.  

It turns out that {\bem the version of \cite{suss3} applies},
since this result does  not require Lipschitz continuity---or even
continuity---of the right-hand side, and works as long as the 
reference trajectory arises from a semidifferentiable flow. 
The result is that {\bem the cycloids $\Gamma_1$, $\Gamma_2$
must be such that the quantity 
\begin{equation}
C=|y(x)|(1+y'(x)^2)\,.
\end{equation}
which is constant along each $\Gamma_i$, actually 
has the same value for both curves}. This extra condition determines $X$. 
We refer the reader to \cite{suss3} for the details. 

\gnu{JAN: Actually, I am not sure that ``the details'' are done in my
Kobe CDC paper. (I'd have to check. I cannot do it right now.)  And I
am not sure that, even if the details turn out to be done there, it is
a good idea to ``refer the reader.'' Maybe we should put in the full
computation. What do you think? If you are in favor, I volunteer to do
it. I probably have the whole thing on line somewhere, so it should
not be too much work, if I can find it.}

The ``extra condition'' has a simple geometric meaning. Recall that
the cycloids of Johann Bernoulli's problem are generated
by a circle rolling along the $x$ axis. The extra condition then
says that 

\begin{center}
\fbox{\begin{minipage}{11cm}
{\bem the rolling circles that generate the upper and lower
cycloids must have equall radii.}
\end{minipage}}\end{center}



\gnu{JAN: wouldn't it be nice to put a figure here of the two cycloids,
displaying the similarity of this situation with that of Snell's
law, with the segments replaced by cycloids?}

\ssection{A modest claim}  If we are going to ``speculate''
as Hestenes did, it is natural to ask not only what questions
the Bernoulli brothers would have asked if they had lived 
in modern times, but also what kinds of answers they would have liked.

This is of course a hard question to answer, and anything we say 
will have to be highly subjective. Granted this, 
we are firmly convinced that {\bem the Bernoulli brothers would have liked 
the ``interesting property'' of the solution of Hestenes' problem
(which is almost certainly the reason why Hestenes chose to 
emphasize this property) and also that they would have found 
elegant the geometric condition of our solution of the reflected
brachistochrone problem.} We believe this should be obvious 
to all mathematical readers but, since {\em de gustibus non est disputandum},
we shall make no attempt to give reasons for this purely
aesthetic assertion.

\gnu{JAN: I think this is a very important point, and the case we are
making seems extremely convincing to me, but I am of course far from
a neutral objective observer. Please think it over. If you agree that is 
very convincing, then maybe you can find a better way to say what I am trying to say, or you can add some more stuff on this?}

\section[$\quad$Hamilton's principal function for the
brachistochrone problem, \null$\quad$and the differential-geometric 
connection]{Hamilton's principal function for the
brachistochrone problem, and the differential-geometric 
connection}\label{brachprinc}

\noindent The mathematical formulation of the brachistochrone problem,
as presented so far, is not completely natural, because it takes it
for granted that we know that energy is conserved.  More precisely,
``energy conservation'' is ``assumed'' in at least two conceptually
different ways.  First, {\bf\em the fact of energy conservation is a
prerequisite for even stating the problem}.  This is clearly not
intellectually satisfactory, because it would be much better to state the
problem first, and then use energy conservation to solve it
(or, even better, to let the mathematical analysis discover
energy conservation for us). {\em Second}, {\bf\em the problem of 
``finding the minimum time curve from
$A$ to $B$'' does not make sense for general $A$ and $B$, unless the
value of the energy $E$ is specified}. The problem usually studied in
textbooks is another one, namely, that of ``finding the minimum-time
curve from $A$ to $B$ subject to an extra constraint $E=\bar E$, for a
given $\bar E$.''  If this constraint is not given, then we really
have, for a given $A$, $B$, a one-parameter family of problems, one
for each value of $\bar E$. (Alternatively, we may want to look at the
{\em unconstrained} problem, in which we are also minimizing over all
values of $\bar E$.  But this is clearly not what Johann Bernoulli or
anyone else who studied the problem had in mind, and in addition is
easy to solve: the infimum of the times is zero, and there is no
minimizer.)

\ssection{What is Hamilton's principal function for the
brachistochrone?}
Another way to see why there is a problem is to ask {\bem what is
the state space of the brachistochrone problem, and what is its
``principal function'' in Hamilton's sense?} Or, using the precise
definitions of the appendix, {\bem what is, exactly, the
controlled dynamical system with cost that corresponds to the
brachistochrone problem?} Let $\mathcal{ C}=(S,U,f,\mathcal{ A},J)$ 
be this system.  It is clear that $J$ should be time.  
But what, exactly, should we choose for $S$? Can we take 
$S=\rr^2$? This would mean that one can specify arbitrary points
$A$ and $B$ and define $V_{\mathcal{ C}}(A,B)$ unambiguously. 
But we know that this is not so. There is a well defined value
$V^E(A,B)$ for each value of $E$, but these values are in general different,
and the infimum of all of them is zero.

A common way to get around this difficulty is by stipulating that we are
only interested in dropping the particle with zero initial kinetic
energy.  (One can even interpret Johann Bernoulli's statement of the
problem, quoted in Chapter \ref{controbrach},  
as suggesting this possibility.) This, however, will not do,
because it would have the following unnatural consequence: {\em if we
pick two points $CD$ in an optimal arc $\Gamma$ going from $A$ to
$B$, and use $\Gamma(C,D)$ to denote the piece of $\Gamma$ from $C$ to
$D$, then $\Gamma(C,D)$ will not in general be an optimal arc from
$C$ to $D$}. (The reason should by now be obvious: the kinetic energy
of the particle that started falling with zero velocity at 
$A$ will not in general have zero velocity at $C$.)  In modern
terminology, {\em the problem with zero initial kinetic energy does
not satisfy the principle of optimality}.\Ft{Jakob Bernoulli's
solution of the brachistochrone problem makes heavy use of the fact
that a portion of a minimizer is itself a minimizer.  So Jakob cannot
possibly have been thinking  that one should only consider 
the case of zero initial kinetic energy.}   

Perhaps the state space is bigger than the plane?  In other words, it
might be the case that one can specify the position and something else at
the starting time and a similar object at the terminal time.  This
will not do either. We cannot, for example, specify the initial and
terminal positions and velocity vectors, or even just the positions
and the lengths of the velocities.  What can be specified is $A$, $B$,
and the energy---or something yielding equivalent information,
e.g. the length of the initial velocity vector---and from this there 
is no way to derive a state space $S$ such that $V$ is defined on $S$. 

\gnu{WE NEED ONE MORE SUBSECTION HERE. Topic: the Hestenes problem and
the brachistochrone are closely related. In both cases we are moving
in $\rr^2$ and controlling acceleration.  In the BP we control
acceleration by requiring that acceleration is that of gravity plus
something perpendicular to the velocity (D'Alembert's virtual force
principle).  In Hestenes acceleration=gravity plus thrust.  So in both
cases the problem should have $\rr^4$ as its state space.  Then how
come the BP is formulated in $\rr^2$? It should not be!!!  And how
come for the BP we specify initial and terminal positions, but for
Hestenes we specify initial and terminal positions AND VELOCITIES?
Isn't that an anomaly?  Then we should give a first hint of the answer
that is coming later: the BP is also a 4-dimensional problem, really,
but it is anomalous in two ways: first, because there is a nontrivial
conserved quantity---the energy---so the state space is foliated by
3-dimensional manifolds such that every trajectory is contained in one
of these manifolds. Second, because for the BP the controls are
unbounded, even within a single leaf existence of an optimal control
joining any two points does not follow.  These two anomalies conspire
to make the BP look two-dimensional and to conceal its true
4-dimensional nature.  It should be obvious from the above that I
think I know what I want to say, but if you want to give try your hand
and write something, go ahead.}

\gnu{Maybe we should make a complete study of the general
acceleration control problem in $\rr^2$ of which both Hestenes
and BP are particular cases? Compute the Lie algebra in general and 
show how it degenerates for the BP? What do you say?}

\gnu{One the previous GNUs are taken care of, I'd like to go back to
the beginning of the book and put in a few remarks on how the BP is a
``degenerate problem'' and how this fact has obscured its true meaning
and how it fits within a general theory.}



\ssection{Lie brackets, integrals of motion, and the
differential-geometric perspective} A much better way to pose the
problem is to write down the equations of motion that truly correspond to
Johann Bernoulli's problem as he stated it in June 1696.  One should then 
let the mathematical analysis lead us to the discovery of energy
conservation and the simplification resulting from it.  This will lead us
naturally to a four-dimensional problem whose analysis is best carried out
from the differential-geometric perspective, by means of Lie-algebraic
calculations.

The true equations of motion for the brachistochrone are 
\begin{equation}\label{trueeq}
\dot{x}=v,\,\,\,\,\,\,\,\dot{y}=w\,,\,\,\,\,\dot{v}=uw\,,\,\,\,\,\,\,\,\,
\dot{w}=-uv-g\,.
\end{equation}
Here $x$ and $y$ are the horizontal and vertical coordinates of our moving
point, and $v$, $w$ are the components of the velocity vector.   
The requirement that the point is freely falling ``under the influence
of its own weight'' means that the force effectively acting on the point 
should be equal to a vector proportional to $[0,-g]$ plus a 
``virtual force'' that does no work, i.e., is perpendicular to the 
velocity.   Equation (\ref{trueeq}) captures these requirements, by 
introducing a virtual force vector of the form $[uw,-uv]$, where 
$u$ is an arbitrary ``control,'' taking values in $\rr$.

Using $q=[x,\,y,\,v,\, w]^{\dagger}$, we get a familiar equation, namely, 
\begin{equation}\label{fami}
\dot{q}=F(q)+uG(q)\,,
\end{equation}
where $F=[v,w,0,-g]^\dagger$, and 
$G=[0,0,w,-v]^\dagger$. 

This is a {\em 4-dimensional system}, with state space $\rr^4$.  

\ssection{The diamond Lie algebra}  Having expressed our
equations of motion in the form (\ref{fami}), the first thing that one
should do, from the differential-geometric perspective is to compute the
iterated Lie brackets of $F$ and $G$, and determine the 
{\bem Lie algebra sturcture of our problem.}    It turns out that 
the four vector fields $F$, $G$, $[F,G]$ and $[F,[F,G]]$ are linearly
independent over $\rr$ (that is, there exists no nontrivial linear
combination of them with constant coefficients that vanishes 
identically), and in addition the relations 
\begin{equation}\label{gfgf}
[G,[F,G]]=F\,,\,\,\,\,[F,[F,[F,G]]]=[G,[F,[F,G]]]=0\,
\end{equation}
hold. It then follows easily that, if $L$ denotes the Lie algebra of vector
fields generated by $F$ and $G$, then $L$ is four-dimensional.  As an
abstract Lie algebra, $L$ is isomorphic to the {\em diamond Lie algebra} $G_4$,
a solvable Lie algebra that plays an important role in representation
theory (cf. \cite{berna}, p. 59).   This Lie algebra has a basis 
$H,P,Q,E$, with the commutation relations $[H,P]=Q$, $[H,Q]=-P$,  
$[P,Q]=E$, $[H,E]=[P,E]=[Q,E]=0$.  (To construct an isomorphism $\Phi$
from $G_4$ to $L$, define $\Phi(P)=[F,G]$, $\Phi(Q)=F$, $\Phi(H)=G$,
$\Phi(E)=-[F,[F,G]]$.   A concrete realization of $G_4$ as an algebra
of differential operators---acting of smooth functions on the real 
line---can be defined by ``quantizing the harmonic oscillator,'' i.e., by 
taking $P=\frac{d\,\,}{dx}$, $Q={\rm multiplication\,\,\,by\,\,\,}x$, 
$E={\rm identity}$, $H=\frac{1}{2}(-P^2+Q^2)
=\frac{1}{2}\Big(-\frac{d^2\,\,}{dx^2}+x^2\Big)$, 
so $H$ is the Hermite operator.   


\ssection{Energy conservation} Although  
$F$, $G$, $[F,G]$ and $[F,[F,G]]$ are linearly independent over
$\rr$, a simple computation shows that 
\begin{equation}\label{eqobse}
G=\psi_1F+\psi_{2}[F,G]+\psi_{3}[F,[F,G]]\,\,\,\,\, {\rm
and}\,\,\,\,\,[G,[F,G]]=F\,, 
\end{equation}
where the $\psi_i$ are smooth functions
of $q$, given by 
\begin{equation}
\psi_1=\frac{v}{g}\,,\,\,\,\,\psi_2=-\frac{w}{g}\,,\,\,\,\,
\psi_3=-\frac{v^2+w^2}{2g^2}\,.  
\end{equation}
\medbreak\noindent{}From these facts  one can easily show that
every iterated Lie bracket of $F$ and $G$ is a linear combination of $F$,
$[F,G]$, and $[F,[F,G]]$ with smooth coefficients.  Since $F(q)$,
$[F,G](q)$, and $[F,[F,G]](q)$ are linearly independent at each point
$q\in\rr^4$, we can conclude that the {\em distribution}
$\Lambda$ generated by $L$ is 3-{\it dimensional} at each point. 
(By definition, $\Lambda(q)\equaldef\{X(q):X\in L\}$.) 
This means---using, for example, Frobenius' theorem on the existence of
integral manifolds---that, at least locally, there is a nontrivial integral of
motion, i.e., a function with nonzero gradient which is constant along all
integral curves of $F$ and $G$, and then also along all solutions of
(\ref{fami}). This integral of motion can be easily computed and turns out
to be the energy $E$, given by 
\begin{equation}
E(x,y,v,w)=\frac{v^2+w^2}{2}+gy\,.
\end{equation} 
Notice that $E$ is a smooth function on $\rr^4$, with nowhere vanishing
gradient. Therefore the level hypersurfaces $S_{\bar E}$ of $E$---i.e.,
the sets $E=\bar E$, for $\bar E\in\rr$---are smooth $3$-dimensional
manifolds. (The set $S_{\bar E}$ is given by the equation
$y=\frac{2c-v^2+w^2}{2g}$, which shows that each $S_{\bar E}$ is the graph
of a smooth function $(x,v,w)\mapsto y$.)

\ssection{Na\"{\i}ve application of the maximum
principle}The fact that our 4-dimensional state space $\rr^4$
is foliated by 3-dimensional ``leaves'' $S_E$ has profound
implications for the optimization problem.  Indeed, 
{\em if we try to find the optimal trajectories by applying the maximum
principle, we get} {\bem nothing}. This is a consequence of more
general result, namely, that, 

\medbreak\noindent{\bf (\#)} {\em  whenever a control system satisfies a
nontrivial ``holonomic constraint'' such as $E={\rm constant}$, then the
maximum principle is uninformative, because {\bem every trajectory is an
extremal}}.  

\medbreak\noindent(The word ``nontrivial'' here means that $\nabla E$ 
never vanishes.) This explains why in our case the maximum principle,
na\"{\i}vely applied, yields nothing.   

\ssection{Application of the maximum principle on
manifolds}Using the integral of motion $E$, we can regard $\rr^4$ as
foliated by the 3-dimensional level hypersurfaces of $E$, and conclude that
every trajectory is contained in one of the leaves $S_{\bar E}$.  If
$\bar{E}\in\rr$, then the maximum principle on manifolds (cf.
Chapter \ref{beyond} and \cite{susscofree}) 
can be applied to the problem restricted to
$S_{\bar{E}}$, and the conclusion turns out to be exactly the same as that
for the unrestricted problem, except that now the nontriviality condition
says that $p_*(t)$ cannot be orthogonal to $S_{\bar E}$ at $q_*(t)$.  (In
differential-geometric terms, $p_*(t)$ is really a {\em covector} on
$S_{\bar E}$ at $q_*(t)$, and has to be nontrivial as such. Equivalently,
if one insists on regarding $p_*(t)$ as a vector in $\rr^4$, then it should
not be orthogonal to the tangent space to $S_{\bar E}$ at $q_*(t)$.)
Naturally, one could also avoid using the maximum principle on manifolds by
using the fact that $S_{\bar E}$ is identified with $\rr^3$ in obvious way,
since it is the graph of a smooth function $y(x,v,w)$. This would amount to
eliminating $y$ from the equations by substituting for it its expression in
terms of the other variables, for each fixed $\bar E$.

The Hamiltonian is $H(q,u,p,p_0)=\varphi(p,q)\rangle+u\psi(q,p)-p_0$, where
the functions $\varphi$, $\psi$ are defined by $\varphi(p,q)=\langle
p,F(q)\rangle$ and $\psi(q,p)=\langle p,G(q)\rangle$.  Since $u$ is
completely unrestricted, $H$ cannot have a maximum as a function of $u$
unless $\psi(q,p)=0$.  So $\psi$ must vanish along every lifted extremal.
Differentiation of $\psi=0$ yields $\rho=0$, where $\rho(q,p)=\langle
p,[F,G](q)\rangle$. A further differentiation---together with the fact
that $[G,[F,G]]=F$---yields $\theta+u\varphi=0$,
where $\theta(q,p)=\langle p,[F,[F,G]](q)\rangle$.
This gives $u=-\varphi^{-1}\theta$, i.e.,
\begin{equation}\label{fio} 
u=-\frac{\langle p,[F,[F,G]](q)\rangle}{\langle p,F(q)\rangle}\,.
\end{equation} 
This, together with the equalities $\psi=\rho=0$, determine $u$ as a
function of $q$, i.e., provide an optimal control in the form of a feedback
law  $u=u(q)$.  In fact, (\ref{eqobse}) implies that
$\psi=\psi_1\varphi+\psi_{2}\rho+\psi_{3}\theta$ along our trajectory.
Since $\psi=\rho=0$,  and $\theta=-u\varphi$, we get (using the fact that
$\varphi\neq 0$, because $p$ is nontrivial as a linear functional
on the span of $F$, $[F,G]$ and $[F,[F,G]]$, so that 
$(\varphi,\rho,\theta)\neq(0,0,0)$, while on the other hand 
$\psi=\rho=0$, so that if $\varphi=0$ implies $\theta=0$)
$u=\psi_3^{-1}\psi_1$, i.e.,
\begin{equation}
u=\frac{2vg}{v^2+w^2}\,.  
\end{equation}
It is then clear that $u$ is a smooth function on the complement 
of the set
\begin{equation}
\Sigma=\{(x,y,v,w):v^2+w^2=0\}\,.
\end{equation}
The ``singular'' set $\Sigma$ corresponds, naturally, to the points where 
the kinetic energy vanishes, which as we know from our other analyses 
is where some singular behavior occurs.  

If we define 
\begin{equation}
Z(q)=F(q)-\psi_1(q)G(q)\,,
\end{equation}
we see that {\em $Z$ is a smooth vector field on $\rr^4\backslash \Sigma$
whose trajectories are the extremals in the sense of the maximum
principle.}

The reader is urged to carry out the remaining computations explicitly
as an exercise, and verify that the final result is once again 
Johann Bernoulli's family of cycloids.   

\begin{remark}{\rm 
The preceding calculations provide another illustration of the
implementation of {\bem Lagrange's idea of choosing the best possible 
coordinates for a particular problem}.  Notice how everything is made 
much simpler by the fact that, using the intrinsic version of the
adjoint equation, we can do all our computations directly for the 
switching function $\langle p,G\rangle$ and other functions of the same
type, without ever writing the components of $p$. \proofend
}\end{remark}

\ssection{The complete synthesis and Hamilton's 
principal function for the 4-dimensional brachistochrone 
problem} We haven't yet completely answered all our earlier 
questions and determined Hamilton's principal function $V$. 

It is clear that $V(A,a,B,b)=+\infty$ 
unless both $(A,a)$ and $(B,b)$ lie in the same energy
hypersurface, because if the energies are different then the points cannot
be joined by a trajectory, and the infimum of the empty set is $+\infty$.

On the other hand, if $(A,a)$, $(B,b)$ belong to the same level
hypersurface $S_{\bar E}$, then we expect that $V(A,a,B,b)$ will be finite.
Moreover, although it would be a ``rash conclusion'' to say that there must
be a minimizing curve---cf. our earlier quotation from Synge's paper, in 
Chapter \ref{fromto}---it is natural to expect that such a curve will exist,
and if it does not then we ought to be able to explain why.     

The classical analysis carried out earlier seems to suggest that, given
$A$, $B$ and $\bar E$, there will be only one optimal arc joining them. So,
if we give $A$ and $B$ but do not fix the energy, then there should be a
1-parameter family of such arcs (one arc for each value of $E$). Yet we
now seem to need a 3-parameter family, because we want to accomodate all
possible pairs $(a,b)$ (i.e., four parameters) as long as they give rise to the
same energy (one constraint, and $4-1=3$). Equivalently, {\bem given
$\bar E$, we ought to be able to choose the directions $\theta_a$,
$\theta_b$, of the vectors $a$ and $b$ arbitrarily}. (Their magnitudes are
determined by the fact that the energy at $A$ and $B$ must equal
$\bar{E}$.)  But {\bem given $A$, $B$ and $\bar{E}$ there is a unique
cycloid from $A$ to $B$ corresponding to the energy $\bar{E}$}, and in
particular the directions $\bar{\theta}_a$, $\bar{\theta}_b$ of this
cycloid at $A$ and $B$ are completely determined by $A$, $B$ and $\bar{E}$.

In other words, {\bem we ought to be able to choose freely the two angles
$\theta_a$, $\theta_b$, but it appears that we cannot do so.}

The solution of this apparent paradox is as follows.  Suppose we fix an
energy level $\bar E$.  Given any $A$, the set $C(A,\bar E)$ of those $a$'s
such that $(A,a)\in S_{\bar E}$ is either empty or a circle, except for the
degenerate case when $(A,0)\in S_{\bar E}$, in which case $C(A,\bar E)$ is
a point.  If $C(A,\bar E)$ is a circle, then through each point of
$\{A\}\times C(A,\bar E)$ there passes an integral curve of $Z$.  The set
of all points lying on such curves is 2-dimensional, and intersects the
1-dimensional set $\{B\}\times C(B,\bar E)$ at just one point. This means
that there is a unique pair $(\bar a,\bar b)$ such that $(A,\bar a)$ and
$(B,\bar b)$ lie on the same integral curve of $Z$.  The portion of that
curve joining $(A,\bar a)$ to $(B,\bar b)$ is, of course, our famous
cycloid.

So, then, what is the optimal trajectory joining $(A,a)$ to $(B,b)$
for general $a$, $b$? What is the value of $V(A,a,B,b)$?  The answer
is quite simple: the ``optimal trajectory'' is obtained by applying an
{\bem impulse control} (cf. Ch. \ref{secreasons}) to rotate $a$ to $\bar
%GNU Check the crossreference to Ch. \ref{secreasons}
a$ instantaneously, then following the cycloid from $(A,\bar a)$ to
$(B,\bar b)$, and then applying another impulse to rotate $\bar b$ to
$b$.  (These are not true trajectories, of course, but it is easy to
translate the above into a statement about a minimizing sequence of
true trajectories.)
The value $V(A,a,B,b)$ is therefore equal to 
$V(A,\bar a,B,\bar b)$. 

\ssection{Deriving \snell{}' law from the maximum 
principle}\label{snellderi}

\noindent We conclude our discussion by going back to an even earlier
episode in our story, namely, Fermat's minimum time principle and \snell{}'
law of refraction.  We have already explained how this law played a crucial
role in Johann Bernoulli's solution of the brachistochrone problem, which
then gave rise to the calculus of variations and optimal control.  It would
therefore be nice to end our discussion by ``closing the loop'' and showing
that the results attained during this 300-year old endeavor do apply, in
particular, to their venerable distant ancestor from geometrical optics.

So let us consider a medium such as the one we described when
discussing \snell{}' law, in which the speed of light is constant on
each of the two sides $H_-$, $H_+$ of a hyperplane $H$ in $\rr^n$, but
the constant values $c_-$, $c_+$, are different.  Let us ask the
obvious question, namely, whether \snell{}' law be derived from the
maximum principle.

It is easy to see that there is no way to formulate this problem
as a minimum time problem with a controlled dynamics 
$\dot q=f(q,u,t)$ in which $f$ is continuous with respect to $q$, since
$||f||$ has to jump as we cross $H$.

In our previous discussion in Chapters \ref{frompri} and \ref{nonsmoothcon}
we have explained how one
can get rid of requirements that $f$ and $L$ be differentiable, and
even continuous, with respect to $u$.  But here the situation is
different, since we want to be able to work with an $f$ that is
discontinuous with respect to $q$.

Once again, it turns out that our recent version of (MP) given 
in \cite{suss3} applies to this problem as
well. The reason is that in \cite{suss3} all that is needed is that the 
reference trajectory $q_*$ arise from a flow $\{\Phi_{t',t}\}$ consisting of
semidifferentiable maps, and there is no need for this flow to be generated by
{}from a continuous vector field. One can formulate our refraction 
problem in terms of the
dynamical law $\dot q=f(q,u,t)$, where the control $u$ is now
a pair $(u_-,u_+)$ of unit vectors, and 
$f(q,u,t)=c_-u_-$ for $q\in H_-$, $f(q,u,t)=c_+u_+$ for $q\in H_+$.
If $\bar q\in H_-$, $\hat q\in H_+$, it is easily seen, by applying 
the classical version of  (MP), that an optimal trajectory 
{}from $\bar q$ to $\hat q$ has to consist of a segment going from
$\bar q$ to some point $P$ in $H$, followed by another segment from $P$ to
$\hat q$.   The question is to find $P$.  It is easy to verify that,
if $q_*:[a,b]\mapsto\rr^n$ is a trajectory of this type, then 
$q_*$ is in fact a trajectory of a constant control 
$u=(u_-,u_+)$, such that both $u_-$ and $u_+$ point to the same side of
$H$.  Although the vector field $q\mapsto f(q,u)$ is discontinuous,
one can easily see that the corresponding flow maps 
$\Phi_{t,a}$ are continuously differentiable (and {\it a fortiori}
semidifferentiable) at $\bar q$, for all $t$ except the one such
that $\Phi_{t,a}(\bar q)\in H$. This suffices to make the maximum principle of 
\cite{suss3} applicable, and derive \snell{}' law. 


\cchapter[$\,$Rigorous solution of the isoperimetric problem]
{Rigorous solution of the isoperimetric problem}\label{justifisop}

\ssection{Example II: the isoperimetric problem}\label{subsec-isoper}. 

\subsection{Problem statement}

The standard statement of the isoperimetric problem is as follows:

\begin{itemize}
{\Large \item[(IP)] {\em Of all simple, closed, rectifiable curves of a given 
length $L$, find those that enclose the largest possible area.}}
\end{itemize}
To make this mathematically precise we would have, to begin with, to give
precise definitions of all the terms involved. We then have to give
a formula for the area enclosed by a simple closed curve, and solve the
maximization problem.  Some of these issues---those of 
defining ``simple,'' ``closed,'' ``rectifiable,'' and ``length''---
are easy to take care of, so we will do it here. 

Here we will show how to use the necessary conditions for an optimum to 
derive the classical result that the solution of the isoperimetric problem
exists and is a circle. If this is so, the radius $R$ of this 
circle is determined by the equation $2\pi R=L$, and then the enclosed
area is $\pi R^2$, i.e., $\frac{L^2}{4\pi}$. 

The solution presented here is not completely rigorous, because
\begin{itemize}
\item[1.] We will assume that the ``area enclosed'' by a simple, closed,
rectifiable curve $\xi$ is the line integral
\begin{equation}\label{areafor}
A(\xi)\equaldef
\frac{1}{2}\int_a^b(x(t)\dot y(t)-y(t)\dot x(t))\,dt\,,
\end{equation}
if $\xi:[a,b]\mapsto\rr^2$ and we write $\xi(t)=(x(t),y(t))$.
\item[2.] We will prove that the solution is a circle
under the assumption that a related maximization 
problem has a solution.
\end{itemize}
The precise justification of these two assumptions
will be given later, in \S\ref{justifisop}.

\subsection{Problem reformulations}

From now on, $L$ is a fixed positive real number. We let $\Gamma_L^1$ 
be the set of all curves 
$[0,1]\ni t\mapsto\gamma(t)=(x(t),y(t))\in\rr^2$ such that

\begin{itemize}
\item[1.] $\gamma$ is absolutely continuous;
\item[2.] $\gamma(1)=\gamma(0)$;
\item[3.] $\gamma$ has length $L$, that is, 
$$
\int_0^1\|\dot\gamma(t)\|=\int_0^1\sqrt{\dot x(t)^2+\dot y(t)^2}\,dt=L\,;
$$
\item[4.] $\gamma$ is simple (that is, the restriction of $\gamma$ to
the half-open interval $[0,1[$ is one-to-one);
\item[5.] $\gamma$ is oriented counterclockwise.
\end{itemize}
The precise meaning of the words ``oriented counterclockwise'' is as follows: 
if \mbox{$\gamma:[0,1]\mapsto\rr^2$} is continuous, simple, and such 
that $\gamma(0)=\gamma(1)$, then 
the Jordan Curve Theorem (cf. Theorem \ref{jordantheo} below)
implies that the complement of the set
$|\gamma|=\{\gamma(t):t\in[0,1]\}$ has two \mbox{connected} components,
one bounded and one unbounded; let ${\rm Ins}(\gamma)$ denote the bounded 
\mbox{connected} component of $\rr^2\backslash|\gamma|$; then
the winding number $\nu(\xi,p)$ is either 
equal to $1$ for all $p\in{\rm Ins}(\gamma)$ or
equal to $-1$ for all $p\in{\rm Ins}(\gamma)$, by 
Theorem \ref{theowinnum}. (The definitions of ``connected component''
and  ``winding number'' are given later, in \S\ref{secconnected}
and \S\ref{secwinding}.)


If $\gamma\in\Gamma_L^1$, then ${\rm Ins}(\gamma)$ is the ``region enclosed
by $\gamma$.'' We will admit that
\begin{itemize}
\item[(A1)] {\em If a curve 
$[0,1]\ni t\mapsto\gamma(t)=(x(t),y(t))\in\rr^2$ belongs to
$\Gamma_L^1$, then the area of the region ${\rm Ins}(\gamma)$ is equal to
$A(\gamma)$, where}
$$
A(\gamma)\equaldef\frac{1}{2}\int_0^1(x(t)\dot y(t)-y(t)\dot x(t))\,dt\,.
$$
\end{itemize}
(This is ``Green's formula,'' proved below, cf. Theorem
\ref{green}.)


To any curve $\gamma\in\Gamma_L^1$ we can associate the curve
$\xi_\gamma:[0,L]\mapsto\rr^2$ obtained from $\gamma$ by 
{\em reparametrization by arc-length}. Precisely, 
write $\gamma(t)=(x(t),y(t))$ for $t\in[0,1]$, and 
define a function $s_\gamma:[0,1]\mapsto[0,L]$ 
(the ``arc-length along $\gamma$'') by letting
$$
s_\gamma(t)=\int_0^t\sqrt{\dot x(u)^2+\dot y(u)^2}\,du\,.
$$
Then $s_\gamma$ is a monotonically nondecreasing continuous
function from $[0,1]$ onto $[0,L]$. Furthermore, if
$0\le t_1<t_2\le 1$ and $s_\gamma(t_1)=s_\gamma(t_2)$, then
$\int_{t_1}^{t_2}\sqrt{\dot x(u)^2+\dot y(u)^2}\,du
=s_\gamma(t_2)-s_\gamma(t_1)=0$, so $\dot x(u)\equiv \dot y(u)\equiv 0$
for almost all $t_1\le u\le t_2$, and then 
$\gamma(s_\gamma(t_2))=\gamma(s_\gamma(t_1))$. Therefore there
exists a unique map $\xi_\gamma:[0,L]\mapsto\rr^2$ such that
$\xi_\gamma(s)=\gamma(t)$ whenever $s=s_\gamma(t)$. If $0\le s_1<s_2\le L$, 
then we can pick $t_1,t_2\in[0,1]$ such that 
$s_1=s_\gamma(t_1)$ and $s_2=s_\gamma(t_2)$. Then $t_1<t_2$, and
\begin{eqnarray*}
\|\xi_\gamma(s_2)-\xi_\gamma(s_1)\|&=&\|\gamma(t_2)-\gamma(t_1)\|\\
&=&\|\int_{t_1}^{t_2}\dot\gamma(t)\,dt\|\\
&\le&\int_{t_1}^{t_2}\|\dot\gamma(t)\|\,dt\\
&=&s_\gamma(t_2)-s_\gamma(t_1)=s_2-s_1\,.
\end{eqnarray*}
Therefore $\|\xi_\gamma(s_2)-\xi_\gamma(s_1)\|\le|s_2-s_1|$
whenever $s_1,s_2\in[0,L]$. So {\em $\xi_\gamma$ is Lipschitz, 
with constant $1$.}  It follows that $\xi_\gamma$ is 
absolutely continuous and 
$\Big\|\frac{d\,\,}{ds}(\xi_\gamma(s))\Big\|\le 1$ 
for almost all $s\in[0,L]$. \label{pagelip}

Formally, the identity $\xi_\gamma\circ s_\gamma=\gamma$ implies
that 
$$\frac{d\,\,}{dt}(\gamma(t))=
\frac{d\,\,}{ds}(\xi_\gamma(s))\Big|_{s=s_\gamma(t)}
\cdot\frac{d\,\,}{dt}(s_\gamma(t))\,,
$$
and then
$$\Big\|\frac{d\,\,}{dt}(\gamma(t))\Big\|=
\Big\|\frac{d\,\,}{ds}(\xi_\gamma(s))\Big|_{s=s_\gamma(t)}
\Big\|\cdot\frac{d\,\,}{dt}(s_\gamma(t))\,,
$$
so that, making the change of variables $s=s_\gamma(t)$, we find
\begin{eqnarray*}
\int_0^L\Big\|\frac{d\,\,}{ds}(\xi_\gamma(s))\Big\|\,ds
&=&\int_0^1\Big\|\frac{d\,\,}{ds}(\xi_\gamma(s))\Big|_{s=s_\gamma(t)}
\Big\|\cdot\frac{ds_\gamma}{dt\,}(t)\,dt\\
&=&\int_0^1\Big\|\frac{d\,\,}{dt}(\gamma(t))\Big\|\,dt\\
&=&L\,.
\end{eqnarray*}
Therefore 
\begin{equation}\label{nonrig-uup}
\Big\|\frac{d\,\,}{ds}(\xi_\gamma(s))\Big\|=1\,\,\,{\rm
for\,\,\,almost\,\,\,all}\,\,\,s\in[0,L]\,.
\end{equation}
The above argument is of course nonrigorous. For a rigorous proof
of (\ref{nonrig-uup}), see Proposition \ref{rigochangev} below. 


Clearly, 

So our problem can be restated as follows:

{\Large\em 
\begin{itemize}
\item[(IP')] Find a curve $\gamma_*\in\Gamma_L$ 
that maximizes
the functional $\Gamma_L\ni\gamma\mapsto A(\gamma)\in\rr$.
\end{itemize}}

We associate to (IP') another problem involving a larger
class of curves. We let $\tilde\Gamma_L$ 
be the set of all curves $\gamma:[0,L]\mapsto\rr^2$ such that

\begin{itemize}
\item[1.] $\gamma$ is absolutely continuous, 
\item[2.] $\|\dot\gamma(t)\|\le 1$ for almost all $t\in[0,L]$,
\item[3.] $\gamma(L)=\gamma(0)=(0,0)$,
\end{itemize}
and consider the problem 
{\Large\em 
\begin{itemize}\item[]\begin{itemize}
\item[(IP'')] Find a curve $\gamma_*$ belonging to $\tilde\Gamma_L$ 
that \mbox{maximizes}
the \mbox{functional} $\tilde\Gamma_L\ni\gamma\mapsto A(\gamma)\in\rr$.
\end{itemize}\end{itemize}}

It is cleat that $\Gamma_L\subseteq\tilde\Gamma_L$. Therefore
\begin{proposition}
If (IP'') has a solution $\gamma_*$ such that 
$\gamma_*\in\Gamma_L$, then $\gamma_*$ is a solution of 
(IP'). \proofend
\end{proposition}

\subsection{Finding the solutions of (IP'')}

To find the solutions of  (IP''), we apply the necessary conditions
for optimality given in the conjecture.  Suppose that
$$
[0,L]\ni t\mapsto\gamma_*(t)=(x(t),y(t))\in\rr^2
$$ 
is a solution of (IP''). It then follows from the conjecture on necessary
conditions for an optimum that there exists
an \mbox{absolutely} \mbox{continuous} ``momentum'' function
$$
[0,L]\ni t\mapsto\mu(t)=(m_x(t),m_y(t))\in\rr^2
$$ 
such that the Hamilton equations
\begin{eqnarray}
\dot m_x(t)&=&-\frac{\partial H}{\partial x}(\gamma_*(t),\mu(t),
\dot\gamma(t))\,,\label{hamil1}\\
\dot m_y(t)&=&-\frac{\partial H}{\partial y}(\gamma_*(t),\mu(t),
\dot\gamma(t))\,,\label{hamil2}
\end{eqnarray}
and the Hamiltonian maximization condition 
\begin{equation}\label{hamio}
H(\gamma_*(t),\mu(t),\dot\gamma_*(t))=
\max\Big\{H(\gamma_*(t),\mu(t),w):w\in\rr^2,\|w\|\le 1\Big\}
\end{equation}
hold for almost all $t\in[0,L]$. Here $H$ is the Hamiltonian,
that is, the function
\begin{equation}\label{hamio2}
\rr^6\ni(x,y,p_x,p_y,u,v)\mapsto H(x,y,p_x,p_y,u,v)\equaldef
p_xu+p_yv+xv-yu\,.
\end{equation}
In view of (\ref{hamio2}), we can rewrite 
(\ref{hamil1}), (\ref{hamil2}), as
\begin{eqnarray}
\dot m_x(t)&=&-\dot y(t)\,,\label{hamil3}\\
\dot m_y(t)&=&\phantom{-}\dot x(t)\,.\label{hamil4}
\end{eqnarray}

Define
$$
\alpha(t)=m_x(t)-y(t)\,,\qquad\beta(t)=m_y(t)+x(t)\,.
$$
Then 
\begin{eqnarray}
\dot\alpha(t)&=&-2\dot y(t)\,,\label{hamil5}\\
\dot\beta(t)&=&\phantom{-}2\dot x(t)\,,\label{hamil6}
\end{eqnarray}
and
\begin{equation}\label{hamix}
H(\gamma_*(t),\mu(t),u,v)=\alpha(t)u+\beta(t)v\,\,.
\end{equation}
Therefore, if we define 
\begin{equation}\label{haminu}
\nu(t)=\sqrt{\alpha(t)^2+\beta(t)^2}\,,
\end{equation}
(\ref{hamio}) says that
\begin{equation}\label{hamioux}
\left.\begin{array}{c}
\dot x(t)=\frac{\alpha(t)}{\nu(t)}\,,\\
\,\,\,\\
\dot y(t)=\frac{\beta(t)}{\nu(t)}\,,\end{array}\right\}\quad{\rm if}\quad
\nu(t)\neq 0\,.
\end{equation}
If $\nu(t)=0$ then  (\ref{hamio}) allows the vector 
$\dot\gamma_*(t)$ to be an arbitrary member of
$\{w\in\rr^2:\|w\|\le 1\}$. We can combine both cases into the following
restatement of (\ref{hamioux}) which is also valid when $\nu(t)=0$:
\begin{equation}\label{hamiouxx}
\nu(t)\dot x(t)=\alpha(t)\qquad{\rm and}\qquad
\nu(t)\dot y(t)=\beta(t)\,.
\end{equation}
A simple calculation then shows that
$$
\frac{d\phantom{t}}{dt}(\nu^2)=
\frac{d\phantom{t}}{dt}(\alpha^2+\beta^2)=
2\alpha\dot\alpha+2\beta\dot\beta=
2\nu\dot x\cdot(-2\dot y)+
2\nu\dot y\cdot 2\dot x=0\,.
$$
Therefore {\em the function $\nu$ is constant.} Let $\bar\nu$ be the 
constant value of $\nu(t)$. If $\bar\nu=0$ then 
$\alpha(t)\equiv\beta(t)\equiv 0$, so 
$\dot\alpha(t)\equiv\dot\beta(t)\equiv 0$, and then 
(\ref{hamil5}) and (\ref{hamil6}) imply that 
$\dot x(t)\equiv\dot y(t)\equiv 0$, so $\gamma_*$ is a constant
curve and $A(\gamma_*)=0$. It is clear that there do exist curves
$\gamma\in\tilde\Gamma_L$ for which $A(\gamma)>0$. Therefore 
the fact that $A(\gamma_*)=0$ contradicts the assumption that
$\gamma_*$ is a solution of (IP''). So we can exclude the possibility 
that $\bar\nu=0$.

From now on we assume that $\bar\nu>0$. Then (\ref{hamioux}) implies that
$\dot\gamma$ is continuous, and 
\begin{equation}\label{hamiouxi}
\|\dot\gamma(t)\|=1\qquad {\rm for\,\,\,all}\,\,\,t\in[0,L]\,.
\end{equation}

Write 
$$
\xi(t)=\dot x(t)\,,\qquad\eta(t)=\dot y(t)\,.
$$
Then, using (\ref{hamil5}), (\ref{hamil6}), and (\ref{hamioux})
we can conclude that
$$
\dot\xi\equiv-\frac{2\eta}{\bar\nu} \quad{\rm and}\quad
\dot\eta\equiv\frac{2\xi}{\bar\nu}\,.
$$
Therefore 
$$
\ddot\xi+\frac{4\xi}{\bar\nu^2}\equiv\ddot\eta+\frac{4\eta}{\bar\nu^2}\equiv
0\,.
$$
This implies that 
$$
\eta(t)=c_1\cos\Big(\frac{2t}{\bar\nu}\Big)+c_2\sin\Big(\frac{2t}{\bar\nu}\Big)
$$
for some constants $c_1$, $c_2$. If we choose $t_0\in\rr$ and
$r\in[0,\infty[$ such that 
$$
c_1=r\cos\Big(\frac{2t_0}{\bar\nu}\Big)\,,\qquad
c_2=r\sin\Big(\frac{2t_0}{\bar\nu}\Big)\,,
$$
we find that
$$
\eta(t)=r\cos\Big(\frac{2(t-t_0)}{\bar\nu}\Big)\,.
$$
Then 
$$
\dot\xi(t)=\frac{\bar\nu\eta(t)}{2}=-r\sin\Big(\frac{2(t-t_0)}{\bar\nu}\Big)\,.
$$
Since $\|\dot\gamma(t)\|=1$, we have $r=1$, and we have shown that 
\begin{equation}\label{hamiouxio}
\xi(t)=-\sin\Big(\frac{2(t-t_0)}{\bar\nu}\Big)\qquad
{\rm and}\qquad
\eta(t)=\cos\Big(\frac{2(t-t_0)}{\bar\nu}\Big)\,.
\end{equation}
The fact that $\gamma(L)=\gamma(0)$ implies that $\bar\nu=\frac{L}{\pi k}$ 
for some nonzero integer $k$. Therefore 
\begin{equation}\label{hamiouxioo}
\xi(t)=-\sin\Big(\frac{2\pi k(t-t_0)}{L}\Big)\qquad
{\rm and}\qquad
\eta(t)=\cos\Big(\frac{2\pi k(t-t_0)}{L}\Big)\,.
\end{equation}
We then get, by integration, 
\begin{eqnarray*}
x(t)&=&\bar x+\frac{L}{2\pi k}\cos\Big(\frac{2\pi k(t-t_0)}{L}\Big)\,,\\
y(t)&=&\bar y+\frac{L}{2\pi k}\sin\Big(\frac{2\pi k(t-t_0)}{L}\Big)\,,
\end{eqnarray*}
where 
\begin{eqnarray*}
\bar x&=&\frac{-L}{2\pi k}\cos\Big(\frac{2\pi kt_0}{L}\Big)\,,\\
\bar y&=&\frac{L}{2\pi k}\sin\Big(\frac{2\pi kt_0}{L}\Big)\,.
\end{eqnarray*}
The above formulas define a family $\Big\{\gamma^{t_0,k}\Big\}_{t_0\in\rr,
k\in\zz}$ of curves belonging to $\tilde\Gamma_L$, \mbox{depending} on the real
parameter $t_0$ and the nonzero integer parameter $k$. If we let 
$\bar\gamma^{t_0,k}$ be the point with coordinates 
$\frac{-L}{2\pi k}\cos\Big(\frac{2\pi kt_0}{L}\Big)$, 
$\frac{L}{2\pi k}\sin\Big(\frac{2\pi kt_0}{L}\Big)$, then 
the curves $\gamma^{t_0,k}$ satisfy 
$$
\Big\|\gamma^{t_0,k}(t)-\bar\gamma^{t_0,k}\Big\|\equiv
\frac{L}{2\pi k}\,.
$$
Therefore {\em the locus $|\gamma^{t_0,k}|$ of $\gamma^{t_0,k}$ is a circle 
of radius $\frac{L}{2\pi k}$}. It is easy to verify that
{\em the map $[0,L]\ni t\mapsto\gamma^{t_0,k}(t)\in\rr^2$ is 
parametrized by arc-length and 
covers the circle $|\gamma^{t_0,k}|$ $|k|$ times.} Moreover,
{\em the orientation derived from this map is counterclockwise if
$k>0$ and clockwise if $k<0$.}

Let us now compute the numbers $A(\gamma^{t_0,k})$. We have
\begin{eqnarray*}
A(\gamma^{t_0,k})&=&\int_0^L(x(t)(\dot y(t)-y(t)\dot x(t))\,dt\\
&=&\int_0^L
\Big(\bar x+\frac{L}{2\pi k}\cos\Big(\frac{2\pi k(t-t_0)}{L}\Big)\Big)
\cos\Big(\frac{2\pi k(t-t_0)}{L}\Big)\,dt\\
&\,&-\int_0^L
\Big(\bar y+\frac{L}{2\pi k}\sin\Big(\frac{2\pi k(t-t_0)}{L}\Big)\Big)
\Big(-\sin\Big(\frac{2\pi k(t-t_0)}{L}\Big)\Big)\,dt\\
&=&\frac{L^2}{2\pi k}\,.
\end{eqnarray*}
The numbers $A(\gamma^{t_0,k})$ are therefore negative 
for $k<0$ (corresponding to the fact that for such values of $k$ the 
curve $\gamma^{t_0,k}$ is oriented clockwise) and positive for 
$k>0$. Furthermore, the largest of these values is 
$\frac{L^2}{2\pi}$, which is attained  for $k=1$. 

In addition, when $k=1$ the curves $\gamma^{t_0,k}$ belong to $\Gamma_L$, 
because they are simple, parametrized by arc-length, and 
oriented counterclockwise.  It follows that 
{\em the curves $\gamma^{t_0,1}$} are solutions 
of (IP') if they are solutions of (IP''). Moreover, if 
a solution of (IP'') exists, then it has to be 
one of the $\gamma^{t_0,1}$, because we have shown that 
the $\gamma^{t_0,k}$ are the only members of $\tilde\Gamma$ that satisfy 
the necessary conditions for an optimum. In that case, then all 
the $\gamma^{t_0,1}$ are solutions, because the functional $A$ has the 
same value for all of them.

Summarizing, we have shown
\begin{proposition}\label{comb1}
Assume that (IP'') has a solution. Then the solutions of (IP'') are 
the circles $\gamma^{t_0,1}$, for $t_0\in\rr$. Moreover, these circles
are also the solutions of the isoperimetric problem (IP'). In particular,
the area enclosed by the area-maximizing circles is 
$\frac{L^2}{4\pi}$. \proofend
\end{proposition}


\ssection{Existence of area-maximizers}

To conclude our complete solution of (IP'), we have to prove that
(IP'') has a solution. The proof is quite simple. The set 
$\tilde\Gamma_L$ is a subset of $C^0([0,L],\rr^2)$ of continuous
functions $\gamma:[0,L]\mapsto\rr^2$, which is a Banach space
in a well-known waym with the sup norm. We first prove the following 
result. 

\begin{proposition}
$\tilde\Gamma_L$ is a compact subset of $C^0([0,L],\rr^2)$.
\end{proposition}

\medbreak\noindent PROOF.\ Let 
$\{\gamma_j\}_{j=1}^\infty$ be a sequence of members 
of $\tilde\Gamma_L$. We want to extract a subsequence
$\{\gamma_{j(k)}\}_{k=1}^\infty$ that converges uniformly
to a limit $\gamma\in\tilde\Gamma_L$. 

Since every member $\gamma$ of $\tilde\Gamma_L$ is absolutely continuous
and satisfies $\|\dot\gamma(t)\|\le 1$ for a.e.~$t$, and 
$\gamma(0)=(0,0)$, it is clear that the sequence 
$\{\gamma_j\}_{j=1}^\infty$ is uniformly bounded and equicontinuous. 
So there exists a subsequence
$\{\gamma_{j(k)}\}_{k=1}^\infty$ that converges uniformly
to a limit $\gamma$, and our conclusion will follow if we prove that 
$\gamma\in\tilde\Gamma_L$. Since $\gamma_j(0)=\gamma_j(L)=(0,0)$ for every 
$j$, it is clear that $\gamma(0)=\gamma(L)=(0,0)$. Since 
$\gamma_j$ is absolutely continuous and $\|\dot\gamma_j(t)\|\le 1$ for
almost all $j$, it follows that 
$$
\|\gamma_j(t)-\gamma_j(s)\|=\|\int_s^t\dot\gamma_j(r)\,dr\|\le|t-s|
$$
for all $j$ and all $t,s\in[0,L]$.  Therefore 
$$
\|\gamma(t)-\gamma(s)\|\le|t-s|\quad{\rm for\,\,\,all}\quad
t,s\in[0,L]\,.
$$
This says that $\gamma$ is Lispchitz with constant $1$, so 
$\gamma$ is absolutely continuous and
$\|\dot\gamma(t)\|\le 1$ for a.e.~$t$. Therefore 
$\gamma\in\tilde\Gamma_L$. \proofend

\medbreak
We now prove the continuity of $A$. 
\begin{proposition}
The functional $A$ is continuous on $\tilde\Gamma_L$.
\end{proposition}

\medbreak\noindent PROOF.\ Let 
$\{\gamma_j\}_{j=1}^\infty$ be a sequence of members 
of $\tilde\Gamma_L$ that converges uniformly to
a limit $\gamma_\infty\in\tilde\Gamma_L$. We want to prove 
that 
\begin{equation}
\lim_{j\to\infty}\Big(A(\gamma_j)-A(\gamma_\infty)\Big)=0\,.
\end{equation}
Write 
$$
\gamma_j(t)=(x_j(t),y_j(t))\quad{\rm for}\quad j\in\{1,2,\ldots\}\cup
\{\infty\}\,,\,t\in[0,L]\,.
$$
Then 
$$
A(\gamma_j)-A(\gamma_\infty)=I_j^1-I_j^2\,,
$$
where
\begin{eqnarray*}
I_j^1&=&\int_0^L\Big(x_j(t)\dot y_j(t)-x_\infty(t)\dot y_\infty(t)\Big)\,dt\,,\\
I_j^2&=&\int_0^L\Big(y_j(t)\dot x_j(t)-y_\infty(t)\dot x_\infty(t)\Big)\,dt\,.
\end{eqnarray*}
We must therefore show that 
$$
\lim_{j\to\infty}I_j^1=\lim_{j\to\infty}I_j^2=0\,.
$$
Let us prove that $\lim_{j\to\infty}I_j^1=0$. (The proof that 
$\lim_{j\to\infty}I_j^2=0$ is similar.) Write
$$
I_j^1=\hat I_j+\tilde I_j\,,
$$
where 
\begin{eqnarray*}
\hat I_j&=&\int_0^L\Big(x_j(t)-x_\infty(t)\Big)\dot y_j\,dt\,,\\
\tilde I_j&=&\int_0^Lx_\infty(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt\,.
\end{eqnarray*}
We will show that 
$$
\lim_{j\to\infty}\hat I_j=\lim_{j\to\infty}\tilde I_j=0\,.
$$
The fact that 
$\lim_{j\to\infty}\hat I_j=0$ is trivial, because
$x_j\to x_\infty$ uniformly, and the functions $\dot y_j$ are uniformly 
bounded, since $|\dot y_j(t)|\le\|\dot\gamma_j(t)\|\le 1$.

To prove that $\lim_{j\to\infty}\tilde I_j=0$, we pick a real number $\eps$
such that $\eps>0$, and find $J$ such that $|\tilde I_j|\le\eps$
whenever $j\ge J$. Let $h$ be a function of class $C^1$ on $[0,L]$ such
that
$$
|x_\infty(t)-h(t)|\le\frac{\eps}{4L}\quad{\rm whenever}\quad t\in[0,L]\,.
$$
(The existence of such an $h$ follows, for example, from the
Stone-Weierstrass theorem, since $x_\infty$ is continuous.) Then 
$$
\tilde I_j=\int_0^L\Big(x_\infty(t)-h(t)\Big)
\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt
+\int_0^Lh(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt\,.
$$
The absolute value of the first integral is bounded 
by $\frac{\eps}{2}$, because 
$|x_\infty(t)-h(t)|\le\frac{\eps}{4L}$ 
and $|\dot y_j(t)-\dot y_\infty(t)|\le 2$. To estimate the second integral,
we first rewrite it using integrationby parts:
\begin{eqnarray*}
\int_0^Lh(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt
&=&h(L)\Big(y_j(L)-y_\infty(L)\Big)-h(0)\Big(y_j(0)-y_\infty(0)\Big)\\
&\,&\,\,\,-\int_0^L\dot h(t)\Big(y_j(t)-y_\infty(t)\Big)\,dt\\
&=&-\int_0^L\dot h(t)\Big(y_j(t)-y_\infty(t)\Big)\,dt\,,
\end{eqnarray*}
using the fact that $y_j(0)=y_\infty(0)=y_j(L)=y_\infty(L)=0$. 
Since $y_j\to y_\infty$ uniformly, and $\dot h$ is a fixed 
bounded function, the limit of 
$\int_0^Lh(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt$ as $j\to\infty$ is
zero. Therefore we can find $J$ such that 
$$
\Big|\int_0^Lh(t)\Big(\dot y_j(t)-\dot y_\infty(t)\Big)\,dt\Big|\le
\frac{\eps}{2}\quad{\rm whenever}\quad j\ge J\,.
$$
It then follows that 
$$
|\tilde I_j|\le\eps\quad{\rm whenever}\quad j\ge J\,,
$$
completing our proof. \proofend


\medbreak
It then follows from Propositions 3 and 4 that
\begin{proposition}\label{comb2}
The maximization problem (IP'') has a solution. \proofend
\end{proposition}


\ssection{The final result}

By combining Propositions \ref{comb1} and \ref{comb2}, we get

\begin{theorem}\label{comb3}
The solutions of the modified isoperimetric problem (IP'') are the circles
$\gamma^{t_0,1}$, for $t_0\in\rr$. \mbox{Moreover}, these circles are also
the solutions of the original isoperimetric isoperimetric problem (IP'). In
\mbox{particular}, the area \mbox{enclosed} by the area-maximizing circles is
$\frac{L^2}{4\pi}$. \proofend
\end{theorem}




\ssection{Polygonal loops}

If $n$ is a positive integer, a {\em plane polygonal loop with $n$ 
sides} (or {\em plane \mbox{$n$-polygonal} loop}) is a loop $\xi$ in $\rr^2$ 
such that there exist times $t_1,t_2,\ldots,t_{n}$ for which
\mbox{$0=t_1<t_2<\ldots<t_n<1$} having the property that
for $j=1,\ldots,n$ the map $[t_{j},t_{j+1}]\ni t\mapsto
\xi(e^{2\pi it})\in\mathbb{C}$ is linear. A plane $n$-polygonal loop 
is completeTly determined by the 
times $t_1,t_2,\ldots,t_{n}$ and the $n$ points 
$A_1,\ldots,A_n$ defined by letting 
$A_j=\xi(e^{2\pi it_j})$  for $j=1,\ldots,n$.
Furthermore, changing the choices of $t_1,t_2,\ldots,t_{n}$ results
in a reparametrization of $\xi$, which does not change 
the carrier $|\xi|$ nor the winding numbers $\nu(\xi,p)$ for 
$p\notin|\xi|$. We can therefore talk unambiguously 
about the carrier and the winding numbers of the 
``polygonal with vertices $A_1,\ldots,A_n$.''

A {\em plane triangle} is a plane $3$-polygonal loop.

The {\em vertices}
of a triangle 


A triangle $\xi$ is {\em nondegenerate}
if $|\xi|$ is not entirely contained in a straight line. 
It is easy to see that a triangle $\xi$ with vertices
$A_1,A_2,A_3$ is nondegenerate if and only
if the vectors $A_2-A_1$ and $A_3-A_1$ are linearly independent. 
If $\xi$ is a nondegenerate triangle, we say that
$\xi$ {\em oriented counterclockwise} if the determinant
of the $2$ by $2$ suqare matrix with columns
$A_2-A_1$ and $A_3-A_1$ is positive. 


\ssection{Rectifiable curves}

A {\em partition} of a compact interval $[a,b]$ is a finite sequence 
${\bf t}=(t_0,t_1,\ldots,t_N)$ such that 
$a=t_0<t_1<\cdots<t_{N-1}<t_N=b$. 

If $\xi:[a,b]\mapsto\rr^n$ is a curve, and 
${\bf t}=(t_0,t_1,\ldots,t_N)$ is a partition of $[a,b]$, then the 
{\em ${\bf t}$-approximate length} of $\xi$ is the number 
${\rm Le}(\xi,{\bf t})$ defined by
$$
{\rm Le}(\xi,{\bf t})\,\,\equaldef\,\,\sum_{j=1}^N
\|\xi(t_j)-\xi(t_{j-1}\|\,.
$$
The {\em length} ${\rm Le}(\xi)$ of $\xi$ is the supremum of the
numbers ${\rm Le}(\xi,{\bf t})$ over all the partitions ${\bf t}$ 
of $[a,b]$.  The curve $\xi$ is said to be {\em rectifiable}
if ${\rm Le}(\xi)$ is finite.

\ssection{Absolutely continuous curves}

\begin{proposition}\label{rigochangev-a}
Let $\gamma:[a,b]\mapsto\rr^n$ be an absolutely
continuous map. Then $\gamma$ is rectifiable
and ${\rm Le}(\gamma)=\int_a^b\|\dot\gamma(t)\|\,dt$.
\end{proposition}
\medbreak\noindent PROOF.\ Let ${\bf t}=(t_0,\ldots,t_N)$ be a
partition of the interval $[a,b]$. Then 
$\gamma(t_j)-\gamma(t_{j-1})=\int_{t_{j-1}}^{t_j}\dot\gamma(t)\,dt$,
so $\|\gamma(t_j)-\gamma(t_{j-1})\|\le
\int_{t_{j-1}}^{t_j}\|\dot\gamma(t)\|\,dt$. Therefore
$$
{\rm Le}(\gamma,{\bf t})=\sum_{j=1}^N\|\gamma(t_j)-\gamma(t_{j-1})\|
\le\sum_{j=1}^N\int_{t_{j-1}}^{t_j}\|\dot\gamma(t)\|\,dt
=\int_{a}^{b}\|\dot\gamma(t)\|\,dt\,.
$$
Hence 
\begin{equation}\label{neatly}
{\rm Le}(\gamma)\le\int_{a}^{b}\|\dot\gamma(t)\|\,dt\,,
\end{equation}
and in particular $\gamma$ is rectifiable. To complete our proof,
we have to show that ${\rm Le}(\gamma)\ge\int_{a}^{b}\|\dot\gamma(t)\|\,dt$.

Fix a positive number $\eps$. Then pick positive numbers
$\beta$ and $\lambda$ such that $2\beta+2\lambda(b-a)<\eps$,
and choose $\alpha$ such that $\alpha>0$ 
and $\int_E\|\dot\gamma\|<\beta$ whenever 
$E$ is a measurable subset of $[a,b]$ such that
${\rm meas}(E)<\alpha$. Using Lusin's theorem, select a compact 
subset $K$ of $[a,b]$ such that 
${\rm meas}([a,b]\backslash K)<\alpha$ and the restriction
of $\dot\gamma$ to $K$ is continuous.  Let 
$\delta$ be such that $\delta>0$ and
$\|\dot\gamma(t_2)-\dot\gamma(t_1)\|<\lambda$
whenever $t_1,t_2\in K$ and $|t_2-t_1|<\delta$. 
Let $E=[a,b]\backslash K$.

Now pick a partition ${\bf t}=(t_0,\ldots,t_N)$
of $[a,b]$ such that 
$t_j-t_{j-1}<\delta$ for $j=1,\ldots,N$. Then
$$
\int_a^b\|\dot\gamma(t)\|\,dt-\sum_{j=1}^N
\|\gamma(t_j)-\gamma(t_{j-1})\|
=\sum_{j=1}^N I_j\,,
$$
where $I_j=I_j^1-I_j^2$, $I_j^1=\int_{t_{j-1}}^{t_j}\|\dot\gamma(t)\|\,dt$,
$I_j^2=\|U_j\|$, and $U_j=\int_{t_{j-1}}^{t_j}\dot\gamma(t)\,dt$.
Write $\mu_j={\rm meas}(K\cap[t_{j-1},t_j])$.

Fix $j$, and assume that $K\cap[t_{j-1},t_j]\neq\emptyset$.
Then 
$$
\int_{K\cap[t_{j-1},t_j]}\dot\gamma(t)\,dt=
U_j-\int_{[t_{j-1},t_j]\cap E}\dot\gamma(t)\,dt\,,
$$
so 
$$
\Big\|\int_{K\cap[t_{j-1},t_j]}\dot\gamma(t)\,dt\Big\|\le
\|U_j\|+\int_{[t_{j-1},t_j]\cap E}\|\dot\gamma(t)\|\,dt\,.
$$
Pick $\hat t\in K\cap[t_{j-1},t_j]\neq\emptyset$.  Then
$$
\mu_j\dot\gamma(\hat t)=
\int_{K\cap[t_{j-1},t_j]}\dot\gamma(t)\,dt+
\int_{K\cap[t_{j-1},t_j]}(\dot\gamma(\hat t)-\dot\gamma(t))\,dt\,,
$$ 
from which it follows that 
\begin{eqnarray*}
\mu_j\|\dot\gamma(\hat t)\|&\le&
\|\int_{K\cap[t_{j-1},t_j]}\dot\gamma(t)\,dt\|+
\int_{K\cap[t_{j-1},t_j]}\|\dot\gamma(\hat t)-\dot\gamma(t)\|\,dt\\
&\le& \|U_j\|+\int_{[t_{j-1},t_j]\cap E}\|\dot\gamma(t)\|\,dt
+\lambda\mu_j\,.
\end{eqnarray*}
On the other hand, if $t\in K\cap[t_{j-1},t_j]$, then
$$
\|\dot\gamma(t)\|=\|\dot\gamma(t)-\dot\gamma(\hat t)+\dot\gamma(\hat t)\|
\le\|\dot\gamma(t)-\dot\gamma(\hat t)\|+\|\dot\gamma(\hat t)\|
\le\lambda+\|\dot\gamma(\hat t)\|\,.
$$
Therefore
$$
\int_{K\cap[t_{j-1},t_j]}\|\dot\gamma(t)\|\,dt
\le\lambda\mu_j+\mu_j\|\dot\gamma(\bar t)\|
\le \|U_j\|+\int_{[t_{j-1},t_j]\cap E}\|\dot\gamma(t)\|\,dt
+2\lambda\mu_j\,.
$$
If we add $\int_{[t_{j-1},t_j]\cap E}\|\dot\gamma(t)\|\,dt$ to both sides,
we get
\begin{eqnarray}
I_j^1&=&\int_{t_{j-1}}^{t_j}\|\dot\gamma(t)\|\,dt\nonumber\\
&\le&\lambda\mu_j+\mu_j\|\dot\gamma(\hat t)\|\nonumber\\
&\le &\|U_j\|+2\int_{[t_{j-1},t_j]\cap E}\|\dot\gamma(t)\|\,dt+
2\lambda\mu_j\,.\label{hoforj}
\end{eqnarray}
This inequality was proved assuming that $K\cap[t_{j-1},t_j]\neq\emptyset$,
but is obviously true as well if the set
$K\cap[t_{j-1},t_j]$ is empty, because
in that case $\int_{[t_{j-1},t_j]\cap E}\|\dot\gamma(t)\|\,dt=I_j^1$.
Therefore (\ref{hoforj}) holds for all $j$. Hence
$$
I_j=I_j^1-\|U_j\|\le 2\int_{[t_{j-1},t_j]\cap E}\|\dot\gamma(t)\|\,dt+
2\lambda\mu_j\,.
$$
If we add over all $j$, we find
$$
\sum_{j=1}^NI_j\le2\int_E\|\dot\gamma\|+2\lambda(b-a),
\le 2\beta+2\lambda(b-a)<\eps\,.
$$
Therefore $\int_a^b\|\dot\gamma(t)\|\,dt\le\eps+\sum_{j=1}^N\
\|\gamma(t_j)-\gamma(t_{j-1})\|\le\eps+ {\rm Le}(\gamma)$.
Since $\eps$ is arbitrary, we have shown that 
\begin{equation}\label{neatly-a}
\int_a^b\|\dot\gamma(t)\|\,dt\le{\rm Le}(\gamma)\,.
\end{equation}
In view of (\ref{neatly}) and (\ref{neatly-a}), 
our conclusion is proved.\proofend

\ssection{Parametrization by arc length}

If $\xi:[a,b]\mapsto\rr^n$ is a curve, 
then we can define the {\em arc-length function}
$s_\xi$ of $\xi$ to be the function 
$s_\xi:[a,b]\mapsto\rr\cup\{+\infty\}$ given by 
$$
s_\xi(t)={\rm Le}(\xi_t)\,,
$$
where, for $t\in[a,b]$, we write $\xi_t$ to denote the restriction
of $\xi$ to $[a,t]$, i.e., the curve
$[a,t]\ni\tau\mapsto\xi(\tau)\in\rr^n$.  
\begin{proposition}
Suppose that $a,b\in\rr$, $a\le b$, and $\xi:[a,b]\mapsto\rr^n$ is a
\mbox{rectifiable} curve, and $L={\rm Le}(\xi)$. Then
the arc length $s_\xi$ is
a continuous \mbox{monotonically} \mbox{nondecreasing} function from $[a,b]$
to $\rr$ that satisfies $s_\xi(a)=0$ and $s_\xi(b)=L$.
\end{proposition}

\medbreak\noindent PROOF.\ The facts that 
$s_\xi(a)=0$ and $s_\xi(b)=L$ follow trivially from the definition
of length. If $a\le t<t'\le b$, and ${\bf t}=
(t_0,t_1,\ldots,t_N)$ is a partition of $[a,t]$, then 
${\bf t}'=(t_0,t_1,\ldots,t_N,t')$ is a partition of $[a,t']$, and 
\mbox{${\rm Le}(\xi_{t'},{\bf t}')={\rm Le}(\xi_{t},{\bf t})+
\|\xi(t')-\xi(t)\|$}, 
so ${\rm Le}(\xi_{t},{\bf t})\le{\rm Le}(\xi_{t'},{\bf t}')
\le s_\xi(t')$. \mbox{Taking} the supremum over all the partitions
${\bf t}$ of $[a,t]$, we find
$s_\xi(t)\le s_\xi(t')$. This proves the \mbox{monotonicity} 
of $s_\xi$, from which it follows that the values of $s_\xi$ 
are finite, since $s_\xi(b)<\infty$.

To prove that $s_\xi$ is continuous, we fix a $t\in[a,b]$
and let 
$$
\lambda_-=\lim_{\tau<t,\tau\uparrow t}s_\xi(\tau)\,,\qquad
\lambda_+=\lim_{\tau>t,\tau\downarrow t}s_\xi(\tau)\,.
$$ 
(If $t=a$ we let $\lambda_-=0$. If $t=b$ we let $\lambda_+=L$.)

We will prove that $\lambda_+=\lambda_-$. Let $\alpha=\lambda_+-\lambda_-$.
Fix a positive \mbox{number} $\eps$, and then choose
a positive $\delta$ such that $\|\xi(\tau)-\xi(t)\|\le\eps$ 
whenever $\tau\in[a,b]$ and $|\tau-t|\le\delta$. Then 
pick $t'\in[a,b]$ such that
$t\le t'$ and \mbox{$s_\xi(t')\le\lambda_++\eps$}. 
Choose a partition ${\bf t}'=(t_0,t_1,\ldots,t_N)$ of 
the interval $[a,t']$
such that \mbox{${\rm Le}(\xi_{t'},{\bf t}')\ge s_\xi(t')-\eps$}.
Without loss of generality, we may assume that $t$ itself
is one of the partition points, and $t_j-t_{j-1}<\delta$ for all $j$,
for otherwise we could modify 
${\bf t}'$ by adding new partition points, making sure that 
$t$ is one of them, and the inequality
${\rm Le}(\xi_{t'},{\bf t}')\ge s_\xi(t')-\eps$ would still
hold.   Let $j\in\{0,\ldots,N\}$ be such that
$t=t_j$. 

Suppose first that $0<j<N$. We claim that, 
if ${\bf \hat t}=(t_0,\ldots,t_{j+1})$, then 
\begin{equation}\label{isgu}
{\rm Le}(\xi_{t_{j+1}},{\bf \hat t})\ge s_{\xi}(t_{j+1})-\eps
\ge \lambda_+-\eps\,.
\end{equation}
Indeed, the second inequality of (\ref{isgu}) follows from the definition
of $\lambda_+$, the monotonicity of $s_\xi$, and the fact that 
$t_{j+1}>t$. If ${\rm
Le}(\xi_{t_{j+1}},{\bf \hat t})<s_{\xi}(t_{j+1})-\eps$, then there
would exist a number $\rho$ and a partition ${\bf \hat u}=(u_0,\ldots,u_M)$ of
$[a,t_{j+1}]$ such that 
$$
\rho>\eps\,,\quad {\rm Le}(\xi_{t_{j+1}},{\bf \hat
t})<s_{\xi}(t_{j+1})-\rho\,,\quad{\rm and}\quad
{\rm Le}(\xi_{t_{j+1}},{\bf \hat u})\ge s_{\xi}(t_{j+1})-(\rho-\eps)\,.
$$
Then 
${\rm Le}(\xi_{t_{j+1}},{\bf \hat u})>\eps+ {\rm Le}(\xi_{t_{j+1}},{\bf \hat
t})$. If we let ${\bf u}'=(u_0,\ldots,u_M, t_{j+2},\ldots,t_N)$,
then ${\bf u}'$ is a partition of $[a,t']$ and ${\rm
Le}(\xi_{t'},{\bf u}')>\eps+ {\rm Le}(\xi_{t'},{\bf t}')\ge
s_\xi(t')$. The inequality ${\rm Le}(\xi_{t'},{\bf u}')>s_\xi(t')$ 
clearly contradicts the definition of $s_\xi(t')$, so
the first inequality of (\ref{isgu}) is proved. 

On the other hand, if ${\bf \tilde t}=(t_0,\ldots,t_{j-1})$, then 
\begin{equation}\label{isgu-a}
{\rm Le}(\xi_{t_{j-1}},{\bf \tilde t})\le s_{\xi}(t_{j-1})\le\lambda_-\,.
\end{equation}

It follows from (\ref{isgu}) and (\ref{isgu-a}) that
\begin{equation}\label{isgu-b}
{\rm Le}(\xi_{t_{j+1}},{\bf \hat t})-{\rm Le}(\xi_{t_{j-1}},{\bf \tilde t})
\ge \lambda_+-\eps-\lambda_-=\alpha-\eps\,.
\end{equation}
But
\begin{eqnarray*}
{\rm Le}(\xi_{t_{j+1}},{\bf \hat t})-{\rm Le}(\xi_{t_{j-1}},{\bf \tilde t})
&=&\|\xi(t_{j+1}-\xi(t_j)\|+\|\xi(t_{j}-\xi(t_{j-1})\|\\
&=&\|\xi(t_{j+1}-\xi(t)\|+\|\xi(t)-\xi(t_{j-1})\|\\
&\le& 2\eps\,,
\end{eqnarray*}
and then (\ref{isgu-b}) implies that $\alpha\le 3\eps$.

The inequality $\alpha\le 3\eps$ also follows easily if $j=0$ or $j=N$. 
Since $\eps$ was arbitrary, $\alpha$ must vanish, and the continuity 
of $s_\xi$ at $t$ is proved. \proofend

\medbreak
To any rectifiable curve $\xi:[a,b]\mapsto\rr^n$ of
length $L$ 
we can \mbox{associate} the curve $\gamma_\xi:[0,L]\mapsto\rr^n$ 
obtained from $\xi$ by 
{\em reparametrization by arc-length}. The precise definition of 
$\gamma_\xi$ is as follows. We know that
the arc length
$s_\xi$ is a monotonically nondecreasing continuous
function from $[a,b]$ onto $[0,L]$. Furthermore, $s_\xi$ satisfies
the inequality
\begin{equation}\label{inequ-g}
s_\xi(t)+\|\xi(t')-\xi(t)\|\le s_\xi(t')\quad{\rm whenever}
\quad a\le t<t'\le b\,.
\end{equation}
(Proof: the inequality is trivial if $t=t'$;
assume that $t<t'$; if ${\bf t}=(t_0,\ldots,t_N)$ is a partition
of $[a,t]$, then ${\bf t}'=(t_0,\ldots,t_N,t')$ is a partition
of $[a,t']$, and ${\rm Le}(\xi_{t},{\bf t})+\|\xi(t')-\xi(t)\|=
{\rm Le}(\xi_{t'},{\bf t}')\le s_\xi(t')$. Taking the supremum 
over ${\bf t}$, we find that (\ref{inequ-g}) holds.)
It follows from (\ref{inequ-g}) that if
$a\le t<t'\le b$ and $s_\xi(t)=s_\xi(t')$ then
$\xi(t)=\xi(t')$. Therefore there exists a unique map 
$\gamma_\xi:[0,L]\mapsto\rr^n$  having the property that
$\gamma_\xi(s)=\xi(t)$ whenever $s=s_\xi(t)$. If $0\le s<s'\le L$, 
then we can pick $t,t'\in[a,b]$ such that 
$s=s_\xi(t)$ and $s'=s_\xi(t')$. Then $t<t'$, and
(\ref{inequ-g}) implies that 
$\|\gamma_\xi(s')-\gamma_\xi(s)\|\le s'-s$.

Therefore 
\begin{equation}\label{inequ-gg}
\|\gamma_\xi(s')-\gamma_\xi(s)\|\le|s'-s|\quad{\rm 
whenever}\quad s,s'\in[0,L]\,.
\end{equation}
So {\em $\gamma_\xi$ is Lipschitz with constant $1$.}  
It follows that $\gamma_\xi$ is 
absolutely continuous and 
$\Big\|\frac{d\,\,}{ds}(\gamma_\xi(s))\Big\|\le 1$ 
for almost all $s\in[0,L]$. \label{pagelip}

Formally, the identity $\xi_\gamma\circ s_\gamma=\gamma$ implies
that 
$$\frac{d\,\,}{dt}(\gamma(t))=
\frac{d\,\,}{ds}(\xi_\gamma(s))\Big|_{s=s_\gamma(t)}
\cdot\frac{d\,\,}{dt}(s_\gamma(t))\,,
$$
and then
$$\Big\|\frac{d\,\,}{dt}(\gamma(t))\Big\|=
\Big\|\frac{d\,\,}{ds}(\xi_\gamma(s))\Big|_{s=s_\gamma(t)}
\Big\|\cdot\frac{d\,\,}{dt}(s_\gamma(t))\,,
$$
so that, making the change of variables $s=s_\gamma(t)$, we find
\begin{eqnarray*}
\int_0^L\Big\|\frac{d\,\,}{ds}(\xi_\gamma(s))\Big\|\,ds
&=&\int_0^1\Big\|\frac{d\,\,}{ds}(\xi_\gamma(s))\Big|_{s=s_\gamma(t)}
\Big\|\cdot\frac{ds_\gamma}{dt\,}(t)\,dt\\
&=&\int_0^1\Big\|\frac{d\,\,}{dt}(\gamma(t))\Big\|\,dt\\
&=&L\,.
\end{eqnarray*}
Therefore 
\begin{equation}\label{nonrig-uup}
\Big\|\frac{d\,\,}{ds}(\xi_\gamma(s))\Big\|=1\,\,\,{\rm
for\,\,\,almost\,\,\,all}\,\,\,s\in[0,L]\,.
\end{equation}
The above argument is of course nonrigorous. For a rigorous proof
of (\ref{nonrig-uup}), see Proposition \ref{rigochangev} below. 

\begin{proposition}\label{rigochangev}
Let $\xi:[a,b]\mapsto\rr^n$ be a rectifiable map of length $L$, and let 
$\gamma_\xi$ be the reparametrization
of $\xi$ by arc-length.  Then 
$\gamma_\xi$ is Lipschitz and
$\|\dot\gamma_\xi(s)\|=1$ for almost all $s\in[0,L]$.
\end{proposition}
\medbreak\noindent PROOF.\ Inequality (\ref{inequ-gg}), which 
was rigorously proved in page \pageref{inequ-gg},
shows that $\gamma_\xi$ is Lipschitz with constant
$1$. Hence $\|\dot\gamma_\xi(s)\|\le 1$
for almost all $s\in[0,L]$. 

Let us show that ${\rm Le}(\gamma_\xi)=L$. Let
${\bf t}=(t_0,\ldots,t_N)$ be a  partition 
of $[a,b]$. Let ${\bf s}=(s_0,\ldots,s_N)$, where
$s_j=s_\xi(t_j)$. Then ${\bf s}$ is a partition
of $[0,L]$, except for the fact that the $s_j$ need not be all
different. Let ${\bf \hat s}$ be the partition 
obtained by eliminating from ${\bf s}$ every $s_j$ that
is equal to some previous $s_k$. It is then clear that
$$
{\rm Le}(\gamma_\xi)\ge{\rm Le}(\gamma_\xi,{\bf \hat s})
=\sum_{j=1}^N\|\gamma_\xi(s_j)-\gamma_\xi(s_{j-1})\|
=\sum_{j=1}^N\|\xi(t_j)-\xi(s_{j-1})\|\,.
$$
Hence ${\rm Le}(\gamma_\xi)\ge{\rm Le}(\xi,{\bf t})$ for an arbitrary
partition ${\bf t}$ of $[a,b]$. Therefore 
${\rm Le}(\gamma_\xi)\ge{\rm Le}(\xi)=L$. On the other hand, 
Proposition (\ref{rigochangev-a}) tells us that 
${\rm Le}(\gamma_\xi)=\int_0^L\|\dot\gamma_\xi(s)\|\,ds$,
so ${\rm Le}(\gamma_\xi)\le L$, because
$\|\dot\gamma_\xi(s)\|\le 1$ for almost all $s\in[0,L]$.
Therefore   ${\rm Le}(\gamma_\xi)=L$. Since
$\int_0^L\|\dot\gamma_\xi(s)\|\,ds=L$, and 
$\|\dot\gamma_\xi(s)\|\le 1$ for almost all $s\in[0,L]$,
it must be the case that $\|\dot\gamma_\xi(s)\|=1$ for almost all $s\in[0,L]$,
and our proof is complete. \proofend


\ssection{Changing variables}

Suppose $a<b$, $c<d$, $f:[c,d]\mapsto\rr^n$ is an integrable
vector-valued function, and $\sigma:[a,b]\mapsto[c,d]$ 
is an absolutely continuous, \mbox{monotonically} nondecreasing
function such that $\sigma(a)=c$ and $\sigma(b)=d$.
Can we ``change variables'' in the integral 
$\int_c^df(s)\,ds$ by writing $s=\sigma(t)$ and
$ds=\dot\sigma(t)\,dt$, and conclude that the formula 
\begin{equation}\label{chovar}
\int_c^df(s)\,ds=\int_a^b\dot\sigma(t)\cdot f(\sigma(t))
\,dt\,
\end{equation}
holds? 

Notice, first of all, that if $f$ is an ``integrable function''
in the usual sense (that is, an equivalence class 
of ``true functions'' from $[c,d]$ to $\rr^n$ modulo
equality almost everywhere, such that every member of the class
is Lebesgue-integrable), then $f\circ\sigma$ is not a well defined
``integrable function'' on $[a,b]$, because if two functions
$f_1$, $f_2$ on $[c,d]$ coincide on the complement of a null set $E$,
and $g_1=f_1\circ\sigma$, $g_2=f_2\circ\sigma$, then all we can
conclude is that $g_1$ and $g_2$ coincide on the complement of
$\sigma^{-1}(E)$, and $\sigma^{-1}(E)$ need not be a null set.  
It turns out, however that this is not a serious difficulty because, 
as we will show, if $E$ is a null set then $\dot\sigma(t)=0$ for 
almost all $t\in\sigma^{-1}(E)$, so the function 
$\dot\sigma\cdot(f\circ\sigma)$ is well defined modulo a.e. equality.
Furthermore, $\dot\sigma\cdot(f\circ\sigma)$ is integrable, 
because $\dot\sigma$ is integrable and $f\circ\sigma$ is bounded.
So it is at least plausible that (\ref{chovar}) might be true.
The following proposition says that this is indeed so. For the 
reasons explained above, we state it first for ``true functions,''
where a {\em true function} on $[a,b]$ is a function 
on $[a,b]$, rather than an equivalence class modulo a.e.
equality.

\begin{proposition}\label{chingo}
Let $n\in\nn$, $a,b,c,d\in\rr$ be such that $a<b$ and $c<d$.
Let $f:[c,d]\mapsto\rr^n$ be a Lebesgue integrable true function.
Assume that \mbox{$\sigma:[a,b]\mapsto\rr$} is an absolutely continuous, 
monotonically nondecreasing map such that $\sigma(a)=c$ and $\sigma(b)=d$. 
Then the true function 
$$
[a,b]\ni t\mapsto\dot\sigma(t)\cdot f(\sigma(t))
$$
is integrable, and (\ref{chovar}) holds.
\end{proposition}

\medbreak\noindent PROOF.\ If $I$ is a closed interval
$[\gamma,\delta]$ (where $c\le\gamma\le\delta\le b$), 
then $\sigma^{-1}(I)$ is the interval
$J=[\alpha,\beta]$, where 
\mbox{$\alpha=\min\{s\in[a,b]:\sigma(s)=\gamma\}$}
and \mbox{$\beta=\max\{s\in[a,b]:\sigma(s)=\delta\}$}. Therefore
$$
{\rm meas}(I)=\delta-\gamma
=\sigma(\beta)-\sigma(\alpha)
=\int_\alpha^\beta\dot\sigma(t)\,dt
=\int_{\sigma^{-1}(I)}\dot\sigma(t)\,dt\,.
$$
A similar argument applies if $I$ is half-open half-closed, or
if $I$ is open. (For example,
if $I=]\gamma,\delta]$, then $\beta$ is defined as above, but $\alpha$ 
and $J$ must be taken to be $\max\{s\in[a,b]:\sigma(s)=\gamma\}$, and
$]\alpha,\beta]$, respectively.) So 
${\rm meas}(I)=\int_{\sigma^{-1}(I)}\dot\sigma(t)\,dt$
for all intervals $I$ such that $I\subseteq[c,d]$.
Using the formula $\sigma^{-1}(E\cup F)=\sigma^{-1}(E)\cup
\sigma^{-1}(F)$, it follows that 
${\rm meas}(E)=\int_{\sigma^{-1}(E)}\dot\sigma(t)\,dt$
for every subset of $[c,d]$ which is a finite
union of intervals, since every such subset is in fact 
a finite union of pairwise disjoint intervals.
  
Let $\mathcal{E}$ be the class of all Borel measurable subsets 
$E$ of $[c,d]$ such that 
${\rm meas}(E)=\int_{\sigma^{-1}(E)}\dot\sigma(t)\,dt$, and let 
$\mathcal{I}$ be the class of subsets of $[c,d]$ that 
are finite unions of intervals.  Then 
$\mathcal{I}\subseteq\mathcal{E}$. Furthermore, it is easy to see
that $\mathcal{E}$ is a monotone class of subsets of $[c,d]$,
that is, (i)~if $\{E_j\}_{j=1}^\infty$ 
is a sequence of members of $\mathcal{E}$ such that 
$E_j\subseteq E_{j+1}$ for every $j$, and $E=\bigcup_{j=1}^\infty E_j$, 
then $E\in\mathcal{E}$, and (ii)~if $\{E_j\}_{j=1}^\infty$ 
is a sequence of members of $\mathcal{E}$ such that 
$E_{j+1}\subseteq E_{j}$ for every $j$, and $E=\bigcap_{j=1}^\infty E_j$, 
then $E\in\mathcal{E}$.  (Indeed, if $E\in\mathcal{E}$ and
$F=[c,d]\backslash E$, then 
\begin{eqnarray*} 
\int_{\sigma^{-1}(F)}\dot\sigma&=&\int_a^b\dot\sigma-
\int_{\sigma^{-1}(E)}\dot\sigma=\sigma(b)-\sigma(a)-{\rm meas}(E)\\
&=&d-c-{\rm meas}(E)={\rm meas}([c,d])-{\rm meas}(E)={\rm meas}(F)\,,
\end{eqnarray*}
so $F\in\mathcal{E}$. This shows that $\mathcal{E}$ is closed
under complementation, from which it follows that if we prove
(i) then (ii) is true as well. To prove (i), we assume that
$E=\bigcup_{j=1}^\infty E_j$, $E_j\in\mathcal{E}$, and
$\{E_j\}_{j=1}^\infty$  is increasing. Then
\begin{eqnarray*} 
{\rm meas}(E)&=&\lim_{j\to\infty}{\rm meas}(E_j)=
\lim_{j\to\infty}\int_{\sigma^{-1}(E_j)}\dot\sigma=
\lim_{j\to\infty}\int_a^b\dot\sigma\cdot\chi_{_{\sigma^{-1}(E_j)}}\\
&=&\int_a^b\dot\sigma\cdot\chi_{_{\sigma^{-1}(E)}}=\int_{\sigma^{-1}(E)}
\dot\sigma\,,
\end{eqnarray*}
by the Lebesgue monotone convergence theorem. So 
$E\in\mathcal{E}$.)

It follows from the monotone convergence theorem (cf. Halmos, vaca, Theorem
vaca) that $\mathcal{E}$ is the set of all Borel subsets of $[c,d]$. So we 
have proved that 
${\rm meas}(E)=\int_{\sigma^{-1}(E)}\dot\sigma(t)\,dt$ for all
Borel subsets of $[c,d]$.

Now\footnote{If $E$ is Borel measurable then $\sigma^{-1}(E)$
is also Borel measurable, because $\sigma$ is continuous. But if
$E$ is Lebesgue measurable then it does not follow that $\sigma^{-1}(E)$
is Lebesgue measurable as well.}
suppose that $E$ is a Lebesgue measurable subset of $[c,d]$. Then 
there exist Borel measurable subsets $E_1$, $E_2$ of $[c,d]$ such
that $E_1\subseteq E\subseteq E_2$ and ${\rm meas}(E_2\backslash E_1)=0$.
Let $F_1=\sigma^{-1}(E_1)$, $F=\sigma^{-1}(E)$, $F_2=\sigma^{-1}(E_2)$.
Then $F_1$ and $F_2$ are Borel measurable, $F_1\subseteq F\subseteq F_2$,
and $\int_{F_2\backslash F_1}\dot\sigma=0$. Since $\dot\sigma$ is nonnegative
and measurable, there exists a Lebesgue null subset $N$ of
$F_2\backslash F_1$ such that 
$\dot\sigma=0$ on $(F_2\backslash F_1)\backslash N$. It follows
that the functions $\dot\sigma\cdot\chi_{_{F}}$ and 
$\dot\sigma\cdot\chi_{_{F_2}}$ coincide on the set 
$[a,b]\backslash N$. Since $N$ is a null set, and 
$\dot\sigma\cdot\chi_{_{F_2}}$ is integrable, we conclude that 
$\dot\sigma\cdot\chi_{_{F}}$ is integrable as well, and
$$
\int_a^b\dot\sigma\cdot\chi_{_{F}}=\int_a^b\dot\sigma\cdot\chi_{_{F_2}}
=\int_{F_2}\dot\sigma={\rm meas}(E_2)={\rm meas}(E)\,.
$$
Since $\chi_{_F}=\chi_{_E}\circ\sigma$, we have shown that 
$\dot\sigma\cdot(\chi_{_{E}}\circ\sigma)$ is integrable and 
$$
\int_a^b\dot\sigma\cdot(\chi_{_{E}}\circ\sigma)=\int_c^d\chi_{_{E}}
$$
whenever $E$ is a Lebesgue measurable subset of $[c,d]$.
In other words, we have proved that 
$\dot\sigma\cdot(f\circ\sigma)$ is integrable and 
(\ref{chovar}) holds 
whenever $f$ is the indicator function of a Lebesgue measurable
subset of $[c,d]$. 

It then follows easily that $\dot\sigma\cdot(f\circ\sigma)$ is integrable 
and  (\ref{chovar}) holds \mbox{whenever} \mbox{$f$ is a simple function}
(i.e., a finite linear combination 
of indicator \mbox{functions} of a Lebesgue measurable
subsets of $[c,d]$).    Then 
$\dot\sigma\cdot(f\circ\sigma)$ is integrable and 
(\ref{chovar}) holds whenever $f$ is a bounded Lebesgue measurable
real-valued \mbox{function} on $[c,d]$, since every such function
is a uniform limit of simple \mbox{functions}. If $f$ is nonnegative
and measurable, then $f=\lim f_j$, where 
$f_j(s)=f(s)$ when $f(s)\le j$ and $f_j(s)=j$ when $f(s)>j$.
Therefore \mbox{$f\circ\sigma=\lim f_j\circ\sigma$}, so 
$\dot\sigma\cdot f\circ\sigma$ is measurable and 
(\ref{chovar}) holds. In particular, if $f$ is integrable
(i.e., if $\int_a^bf<\infty$) then 
$\dot\sigma\cdot f\circ\sigma$ is integrable as well.
Now, if $f:[c,d]\mapsto\rr$ is integrable, then we can write
$f=f_+-f_-$, where both $f_+$ and $f_-$ are integrable and nonnegative,
and conclude that 
$\dot\sigma\cdot(f\circ\sigma)$ is integrable and 
(\ref{chovar}) holds. Finally, if $f$ is vector-valued, 
then the desired conclusion follows by applying what we have shown
to each of the components of $f$.\proofend

Now suppose that 
\begin{itemize}
\item[(A1)] $Q$ is an open subset of $\rr^n$, 
\item[(A2)]  $Q\times\rr^n\ni(q,u)\mapsto L(q,u)\in\rr$ is a 
continuous function, thought of as a time-independent Lagrangian,
\item[(A3)]  $L$ is positively homogeneous of degree $1$ with respect 
to the velocity, that is, $L(q,ru)=rL(q,u)$ whenever
$q\in Q$, $u\in\rr^n$ and $r\ge 0$,
\item[(A4)]  $\xi:[c,d]\mapsto Q$ is an absolutely
continuous curve in $Q$,
\item[(A5)] $\sigma:[a,b]\mapsto\rr$ is absolutely continuous,
monotonically nondecreasing, and such that
$\sigma(a)=c$ and $\sigma(b)=d$,
\item[(A6)]  $\gamma=\xi\compo\sigma$.
\end{itemize} 

\begin{proposition}\label{changev-lag}
Assume that conditions (A1)$\cdots$(A6) above hold. Then
$\gamma$ is \mbox{absolutely} continuous,
\begin{equation}\label{forei}
\frac{d\gamma}{dt}(t)=\dot\sigma(t)\cdot
\frac{d\xi}{ds}(\sigma(t))\quad{\rm for\,\,\,almost\,\,\, all}
\quad t\in[a,b]\,,
\end{equation}
and
\begin{equation}\label{forei-a}
\int_c^dL\Big(\xi(s),\frac{d\xi}{ds}(s)\Big)\,ds=
\int_a^bL\Big(\gamma(t),\frac{d\gamma}{dt}(t)\Big)\,dt\,.
\end{equation}
\end{proposition}

\medbreak\noindent PROOF.\ Let 
$f(s)=\frac{d\xi}{ds}(s)$, $g(t)=\dot\sigma(t)\cdot f(\sigma(t))$.
Then Proposition \ref{chingo} implies that $g$ is integrable.
Furthermore, we can pick an arbitrary $t\in[a,b]$ and apply 
Proposition \ref{chingo} to the interval $[a,t]$, with 
$\sigma$ replaced by its restriction to $[a,t]$, and  conclude
that 
$$
\gamma(t)-\gamma(a)=\xi(\sigma(t))-\xi(c)=\int_c^{\sigma(t)}
f(s)\,ds=\int_a^t g(\tau)\,d\tau\,.
$$
Since $g$ is integrable and
$t$ is arbitrary, $\gamma$ is absolutely continuous and
(\ref{forei}) holds. 

Now let $h(s)=L\Big(\xi(s),\frac{d\xi}{ds}(s)\Big)$. 
Then Proposition \ref{chingo} implies that 
\begin{eqnarray*}
\int_c^dh(s)\,ds&=&\int_a^b\dot\sigma(t)\cdot h(\sigma(t))\\
&=&\int_a^b\dot\sigma(t)\cdot L\Big(\xi(\sigma(t)),\frac{d\xi}{ds}
(\sigma(t))\Big)\,dt\\
&=&\int_a^bL\Big(\gamma(t),\dot\sigma(t)\cdot
\frac{d\xi}{ds}(\sigma(t))\Big)\,dt\\
&=&\int_a^bL\Big(\gamma(t),\frac{d\gamma}{dt}(t)\Big)\,dt\,,
\end{eqnarray*}
where in the last two steps we have used the homogeneity of $L$ and
(\ref{forei}).

Therefore (\ref{forei-a}) holds, and our proof is complete.\proofend

\ssection{The complex exponential}

We will need to use the {\em exponential function}
$\mathbb{C}\ni z\mapsto e^z\in\mathbb{C}$. By definition,
\begin{equation}\label{defofexp}
e^{x+iy}=e^x\cdot e^{iy}=e^x(\cos{y}+i\sin{y})\,.
\end{equation}
The following well-known properties of this function (whose 
proofs can be found in any elementary texbook of 
complex function theory)  will be used:
\begin{itemize}
\item[(E1)] $e^{z_1+z_2}= e^{z_1}\cdot e^{z_2}$ whenever 
$z_1,z_2\in\mathbb{C}$,
\item[(E2)] $e^{0}=1$,
\item[(E3)] $e^{z}\neq 0$ whenever $z\in\mathbb{C}$,
\item[(E4)] if $z\in\mathbb{C}$, then $e^{z}=1$ if and only if 
$z=2\pi ik$ for some integer $k$,
\item[(E5)] the series $z-1-\frac{(z-1)^2}{2}+\frac{(z-1)^3}{3}- 
\frac{(z-1)^4}{4}+\ldots$
is convergent for $z\in\mathbb{C}$ such that $|z-1|< 1$, and 
defines a continuous function 
$$
\{u\in\mathbb{C}:|u-1|<1\}\ni z\mapsto{\rm Ln}(z)\in\mathbb{C}
$$
such that $e^{{\rm Ln}(z)}=z$ whenever $|z-1|<1$.
\end{itemize}
Since $e^{{\rm Ln}(z)}=z$ whenever $|z-1|<1$, the function 
$\,{\rm Ln}\,$ is called ``the standard branch of the natural
logarithm.'' 

If follows from the above properties that
\begin{itemize}
\item[(E6)] if $z_0\in\mathbb{C}$ and $z_0\neq 0$, then there exists
continuous function 
$$
\{u\in\mathbb{C}:|u-z_0|<|z_0|\}\ni z\mapsto\Phi(z)\in\mathbb{C}
$$
such that $e^{\Phi(z)}=z$ whenever $|z-z_0|<|z_0|$.
\item[(E7)] if $[a,b]\ni t\mapsto\gamma(t)\in\mathbb{C}$ is 
an arc in the complex plane (i.e., a continuous
map with values in $\mathbb{C}$) such that $\gamma(t)\neq 0$ for all
$t\in[a,b]$, then there exists a continuous function
$[a,b]\ni t\mapsto f(t)\in\mathbb{C}$ such that
$e^{f(t)}=\gamma(t)$ for all $t\in[a,b]$. 
\end{itemize}
Indeed, to prove (E6), we write $z_0=x_0+iy_0$, $r_0=\sqrt{x_0^2+y_0^2}$,
$x_0=r_0\cdot\cos{\theta_0}$, $y_0=r_0\cdot\sin{\theta_0}$, 
and observe that $r_0>0$, so $r_0$ has a natural logarithm 
$\lambda_0$, and then $e^{\lambda_0}=r_0$. Therefore, if we write
$s_0=\lambda_0+i\theta_0$, (\ref{defofexp}) tells us
that $e^{s_0}=z_0$. Then, if $|z-z_0|<|z_0|$, we have 
$\Big|\frac{z}{z_0}-1\Big|<1$, so 
${\rm Ln}\Big(\frac{z}{z_0}\Big)$ is well defined. We then define
$\Phi(z)=s_0+{\rm Ln}\Big(\frac{z}{z_0}\Big)$, so that
$$
e^{\Phi(z)}=e^{s_0}\cdot e^{{\rm Ln}\Big(\frac{z}{z_0}\Big)}
=z_0\cdot\frac{z}{z_0}=z\,.
$$
To prove (E7), we let $\eps=\min\{\,|\gamma(t)|:t\in[a,b]\,\}$, so that
$\eps>0$, and use the fact that the map $\gamma$ is uniformly 
continuous to obtain a positive $\delta$ such that
$|\gamma(t_1)-\gamma(t_2)|<\eps$ whenever $t_1,t_2\in[a,b]$
and $|t_1-t_2|<\delta$. We then find $t_0,t_1,\ldots,t_N$ 
such that 
\mbox{$a\!=\!t_0\!<\!t_1\!<\!t_2\!<\!\ldots\!<\!t_{N-1}\!<\!t_N\!=\!b$} 
\mbox{and $t_j-t_{j-1}<\delta$} if $j=1,2,\ldots,N$. We write $z_j=\gamma(t_j)$
and \mbox{$D_j=\{z\in\mathbb{C}:|z-z_j|<|z_j|\}$} for $j=1,\ldots,N$,
and use (E6) to pick, for each $j\in\{1,\ldots,N\}$, a 
\mbox{continuous} function  $\Phi_j:D_j\mapsto\mathbb{C}$ such that
$e^{\Phi_j(z)}=z$ whenever $z\in D_j$. We then \mbox{modify}
the $\Phi_j$ recursively, and define new functions 
$\Psi_j:D_j\mapsto\mathbb{C}$, as 
\mbox{follows}: we take $\Psi_1=\Phi_1$;
if $1<j\le N$, and $\Psi_1,\ldots,\Psi_{j-1}$ have been defined
and are such that  $e^{\Psi_\ell(z)}=z$ whenever
$z\in D_\ell$, $0\le\ell<j$, then 
we observe that \mbox{$z_j\in D_{j-1}$}, because 
$|z_j-z_{j-1}|=|\gamma(t_j)-\gamma(t_{j-1})|<\eps\le
|z_{j-1}|$, since $t_j-t_{j-1}<\delta$; therefore
$e^{\Psi_{j-1}(z_j)}=z_j=e^{\Phi_{j}(z_j)}$, from which 
it \mbox{follows} that $\Phi_{j}(z_j)=\Psi_{j-1}(z_j)+2\pi i k_j$ for 
some integer $k_j$; we then \mbox{define}
\mbox{$\Psi_j(z)=\Phi_{j}(z_j)-2\pi i k_j$} for $z\in D_j$, so  that 
$e^{\Psi_j(z)}=z$ whenever $z\in D_j$.  \mbox{Using} the 
$\Psi_j$, we define $f(t)$, for $t\in[a,b]$, by letting
$f(t)=\Psi_j(\gamma(t))$ \mbox{whenever} $t_{j-1}\le t\le t_{j}$,
$j\in\{1,\ldots,N\}$. Then $f$ is well defined.
(Indeed, the only values of $t$ for which the condition
$t_{j-1}\le t\le t_{j}$ does not determine 
$j$ uniquely are $t_1,\ldots,t_{N-1}$. For each of these
$t$'s, if $t=t_j$, $1\le j<N$, then $t\in[t_{j-1},t_{j}]$ and 
$t\in[t_{j},t_{j+1}]$, so the definition would yield
both $f(t)=\Psi_j(\gamma(t))=\Psi_{j}(z_j)$
and $f(t)=\Psi_{j+1}(\gamma(t))=\Psi_{j+1}(z_j)$, but 
$$
\Psi_{j+1}(z_j)=\Phi_{j+1}(z_j)-2\pi i k_{j+1}
=\Big(\,\Psi_{j}(z_j)+2\pi i k_{j}\,\Big)
-2\pi i k_{j+1}=\Psi_{j}(z_j)\,,
$$
so the two values of $f(t)$ coincide.)

\ssection{Loops}

We think of $\rr^2$ as the complex plane $\mathbb{C}$. 
We use $\mathbb{S}^1$ to denote the unit circle 
$\{(x,y)\in\rr^2:x^2+y^2=1\}=\{z\in\mathbb{C}:|z|=1\}$.  

For each positive real number $L$, we let $\kappa_{_L}$ be the map 
$$
\rr\ni t\mapsto \kappa_{_L}(t)\equaldef
e^{\frac{2\pi i}{L}t}\in\mathbb{S}^1\,.
$$

A {\em loop} (or {\em closed curve}) in  $\rr^2$ is a continuous map
$\xi:\mathbb{S}^1\mapsto\rr^2$.  Then 
$C^0(\mathbb{S}^1,\rr^2)$---or $C^0(\mathbb{S}^1,\mathbb{C})$--- 
is the set of all loops in $\rr^2$.
The  {\em carrier}, or
{\em locus}, of the loop $\xi$ is the set $\{\xi(s):s\in\mathbb{C}\}$. 
Clearly, $|\xi|$ is compact.

If $L>1$, then $C^0(\mathbb{S}^1,\rr^2)$ can be identified in an
obvious way with $C_{per,L}^0(\rr,\rr^2)$, the set of all continuous
maps $\zeta:\rr\mapsto\rr^2$ that are $L$-periodic, i.e., such that
$\zeta(t+L)=\zeta(t)$ for all $t\in\rr$. Naturally,
the identification is made by assigning to each $\xi\in
C^0(\mathbb{S}^1,\rr^2)$ the map $\zeta=\xi\circ\kappa_{_L}\in
C_{per,L}^0(\rr,\rr^2)$.

A loop $\xi\in C^0(\mathbb{S}^1,\rr^2)$ is said to be {\em simple} if
it is a one-to-one map, i.e., if 
$\xi(z_1)\neq\xi(z_2)$ whenever $z_1,z_2\in\mathbb{S}^1$ 
and $z_1\neq z_2$.  

A loop $\xi\in C^0(\mathbb{S}^1,\rr^2)$ is said to be {\em
rectifiable} if it is absolutely continuous, in the sense that for
some positive $L$ the function $\zeta=\xi\circ\kappa_{_L}$ is
locally absolutely continuous (in which case it follows 
easily that $\xi\circ\kappa_L$ is locally absolutely
continuous for every positive $L$).

\ssection{Reparametrization of loops}

A {\em reparametrization} of a loop $\xi:\mathbb{S}^1\mapsto\mathbb{C}$ 
is a loop of the form $\xi\circ\psi$, where 
$\psi:\mathbb{S}^1\mapsto\mathbb{S}^1$ is an orientation-preserving 
homeomorphism.
(``Orientation-preserving'' means, for example, that if $\bar t$ is such
that $\psi(e^{2\pi i\bar t})=1$, then the imaginary part of
$\psi(e^{2\pi i(\bar t+\eps)})$ is positive for all sufficiently small
positive $\eps$.) 

A rectifiable loop $\xi\in C^0(\mathbb{S}^1,\mathbb{C})$ is said to be
{\em parametrized by constant times arc-length} (abbr. PCAL) if 
there exists a positive constant $C$ such that 
\mbox{$\|\dot\zeta(t)\|=C$} for all $t\in\rr$, where 
$\zeta(t)\equaldef\xi(e^{2\pi it})$. 

To any rectifiable loop $\xi\in C^0(\mathbb{S}^1,\mathbb{C})$ 
we can associate another loop 
\mbox{$\gamma_\xi:\mathbb{S}^1\mapsto\mathbb{C}$} 
obtained from $\xi$ by {\em reparametrization by constant
times arc-length}. Precisely, $\gamma_\xi$ is defined
as follows. Let $\zeta(t)=\xi(e^{2\pi it})$, for $t\in\rr$.
Write $\zeta(t)=x(t)+iy(t)$ for $t\in\rr$, and
define a function $s_\xi:\rr\mapsto\rr$ 
(the ``arc-length along $\zeta$'') by letting
$$
s_\xi(t)=\int_0^t\sqrt{\dot x(u)^2+\dot y(u)^2}\,du\,.
$$
Then $s_\xi$ is a monotonically nondecreasing continuous
function from $\rr$ to $\rr$. If $L=s_\xi(1)$ (so that
$L$ is the length of $\xi$) then $s_\xi$ maps
$[0,1]$ \mbox{onto} $[0,L]$. \mbox{Furthermore}, if
$0\le t_1<t_2\le 1$ and $s_\xi(t_1)=s_\xi(t_2)$, then
$$
\int_{t_1}^{t_2}\sqrt{\dot x(u)^2+\dot y(u)^2}\,du
=s_\xi(t_2)-s_\xi(t_1)=0\,,
$$
so $\dot x(u)\equiv \dot y(u)\equiv 0$
for almost all $t_1\le u\le t_2$, and then 
$\xi(t_2)=\xi(t_1)$. Therefore there
exists a unique map $\theta_\xi:[0,L]\mapsto\mathbb{C}$ such that
$\theta_\xi(s)=\zeta(t)$ whenever $s=s_\xi(t)$, $t\in[0,1]$. 
In particular, this map satisfies the identities
\mbox{$\theta_\xi(L)=\zeta(1)=\zeta(0)=\theta_\xi(0)$}, 
since $L=s_\xi(1)$. So we can define
a continuous map $\gamma_\xi:\mathbb{S}^1\mapsto\mathbb{C}$ 
by letting $\gamma_\xi(e^{2\pi it})=\theta_\xi(Lt)$ for $0\le t\le 1$.

If $0\le s_1<s_2\le L$, 
then we can pick $t_1,t_2\in[0,1]$ such that 
$s_1=s_\xi(t_1)$ and $s_2=s_\xi(t_2)$. Then $t_1<t_2$, and
\begin{eqnarray*}
\|\theta_\xi(s_2)-\theta_\xi(s_1)\|&=&\|\zeta(t_2)-\zeta(t_1)\|\\
&=&\|\int_{t_1}^{t_2}\dot\zeta(t)\,dt\|\\
&\le&\int_{t_1}^{t_2}\|\dot\zeta(t)\|\,dt\\
&=&s_\xi(t_2)-s_\xi(t_1)=s_2-s_1\,.
\end{eqnarray*}
Therefore $\|\theta_\xi(s_2)-\theta_\xi(s_1)\|\le|s_2-s_1|$
whenever $s_1,s_2\in[0,L]$. So {\em $\theta_\xi$ is Lipschitz
with constant $1$.}  It follows that $\theta_\xi$ is 
absolutely continuous and 
$\Big\|\frac{d\,\,}{ds}(\theta_\xi(s))\Big\|\le 1$ 
for almost all $s\in[0,L]$. \label{vaca}

Formally, the identity $\theta_\xi\circ s_\xi=\zeta$ implies
that 
$$\frac{d\,\,}{dt}(\zeta(t))=
\frac{d\,\,}{ds}(\theta_\xi(s))\Big|_{s=s_\xi(t)}
\cdot\frac{d\,\,}{dt}(s_\xi(t))\,,
$$
and then
$$\|\dot\zeta(t)\|=
\Big\|\frac{d\,\,}{ds}(\theta_\xi(s))\Big|_{s=s_\xi(t)}
\Big\|\cdot\|\dot\zeta(t)\|\,,
$$
so that
\begin{equation}\label{nonrig}
\Big\|\frac{d\,\,}{ds}(\theta_\xi(s))\Big\|=1\,\,\,{\rm
for\,\,\,almost\,\,\,all}\,\,\,s\in[0,L]\,.
\end{equation}
The above argument is of course nonrigorous. For a rigorous proof
of (\ref{nonrig}), see Proposition \ref{rigochangev} below. 

It then follows that
\begin{equation}\label{nonrig-uu}
\Big\|\frac{d\,\,}{dt}(\gamma_\xi(e^{2\pi it})\Big\|=L\,\,\,{\rm
for\,\,\,almost\,\,\,all}\,\,\,t\in[0,1]\,.
\end{equation}
Therefore
\begin{proposition}
If $\xi\in C^0(\mathbb{S}^1,\mathbb{C})$ is a rectifiable loop
of length $L$, and the loop $\gamma_\xi$ is defined as above,
then $\gamma_\xi$ is parametrized by constant times arc-length
(the constant being $L$), and $\xi=\gamma\circ\psi$, where 
$\psi$ is the map 
$$
\mathbb{S}^1\ni e^{2\pi it}\mapsto
\psi(e^{2\pi it})\,\,\equaldef\,\,
e^{2\pi i \lambda s_\xi(t)}\in\mathbb{S}^1\,,
$$
and $\lambda=L^{-1}$.
\end{proposition}

\ssection{Connected components}\label{secconnected}

Vaca.

\ssection{The winding number}\label{secwinding}

If $\xi:\mathbb{S}^1\mapsto\mathbb{C}$ is a loop, and
$p\in\mathbb{C}\backslash|\xi|$, then the {\em winding number} of
$\xi$ about $p$ in the integer $\nu(\xi,p)$ defined as follows: pick
any positive $L$, and use (E7) to choose a continuous function
$f:\rr\mapsto\mathbb{C}$ such that
\mbox{$e^{f(t)}=(\xi\circ\kappa_{_L})(t)-p$} for all $t\in\rr$.  Then
(E4) implies that the number $\frac{f(t+L)-f(t)}{2\pi i}$ necessarily
has to be an integer, because
$$
e^{f(t+L)}=(\xi\circ\kappa_{_L})
(t+L)-p=(\xi\circ\kappa_{_L})(t)-p=e^{f(t)}\,.
$$
It is clear that this integer depends continuously
on $t$. Furthermore, if \mbox{$f_1,f_2:\rr\mapsto\mathbb{C}$} are
any two continuous maps such that
$e^{f_1(t)}=(\xi\circ\kappa_{_L})(t)-p$ and
$e^{f_2(t)}=(\xi\circ\kappa_{_L})(t)-p$ for all $t\in\rr$, 
then $f_1$ and $f_2$ must differ by a constant, because
$e^{f_1(t)-f_2(t)}=1$, so that $f_1(t)-f_2(t)=2\pi ik(t)$, where
$k(t)$ is an integer that depends continuously on $t$. 
So $\frac{f(t+L)-f(t)}{2\pi i}$ is in fact independent
of $t$ and $f$, and it is easy to see that it is also 
independent of $L$. We define 
$\nu(\xi,p)\equaldef\frac{f(t+L)-f(t)}{2\pi i}$ for any $t\in\rr$,
where $L$ and $f$ are as above. 

It is clear that reparametrization preserves
carriers and winding numbers. Precisely, 
\begin{proposition}
If $\eta$ is a reparametrization of a loop 
$\xi\in C^0(\mathbb{S}^1,\mathbb{C})$, then 
$|\eta|=|\xi|$ and $\nu(\eta,p)=\nu(\xi,p)$ for all 
$p\in\mathbb{C}\backslash|\xi|$.
\end{proposition}

\medbreak\noindent PROOF.\ The fact that 
$|\eta|=|\xi|$ is trivial. Let $f:\rr\mapsto\mathbb{C}$ be continuous
and such that $e^{f(t)}=\eta(e^{2\pi i t})$ for all $t$. Let 
$\psi:\mathbb{S}^1\mapsto\mathbb{S}^1$ be an orientation-preserving 
homeomorphism such that $\eta=\xi\circ\psi$. Then there exist
$a\in\rr$ and a homeomorphism
$\tilde\psi:[0,1]\mapsto[a,a+1]$ such that 
\begin{equation}\label{newway}
e^{2\pi i\tilde\psi(t)}=\psi(e^{2\pi it})
\end{equation}
for all $t\in[0,1]$, $\tilde\psi(0)=a$ and $\tilde\psi(1)=a+1$. Extend
$\tilde\psi$ to the whole real line by defining
$\tilde\psi(k+t)=k+\psi(t)$ for $t\in[0,1]$ and $k\in\zz$. Let
$g=f\circ\tilde\psi$. Then (\ref{newway}) holds for all $t\in\rr$. 
Moreover,
$$
e^{g(t)}=e^{f(\tilde\psi(t))}=\eta(e^{2\pi i\tilde\psi(t)})
=\eta\Big(\psi(e^{2\pi it})\Big)=(\eta\circ\psi)(e^{2\pi it})=
\xi(e^{2\pi it})\,.
$$
Therefore $\nu(\xi,p)=\frac{g(1)-g(0)}{2\pi i}=
\frac{f(\tilde\psi(1))-f(\tilde\psi(0))}{2\pi i}=
\frac{f(a+1)-f(a)}{2\pi i}=\nu(\eta,p)$. \proofend

\medbreak
If a loop $\xi$ is rectifiable, then we can express the winding 
number as an integral.
\begin{proposition}
Let $\xi$ be a rectifiable loop in $\mathbb{C}$. Then 
\begin{equation}\label{flawind}
\nu(\xi,p)=\frac{1}{2\pi i}
\int_0^L\frac{\dot\zeta(s)}{\zeta(s)-p}\,ds\,\,\,{\rm if}
\,\,p\in\mathbb{C}\backslash|\xi|\,,\,\,\,L>0\,,\,\,\,{\rm and}\,\,\,
\zeta=\xi\circ\kappa_L.
\end{equation}
\end{proposition}

\medbreak\noindent PROOF.\ We can choose $f$ by letting
\begin{equation}\label{flawind-oo}
f(t)=f_0+\int_0^t\frac{\dot\zeta(s)}{\zeta(s)-p}\,ds\,,
\end{equation}
where $f_0\in\mathbb{C}$ is such that
$e^{f_0}=\zeta(0)-p$. Indeed, if we define $g:\rr\mapsto\mathbb{C}$
by letting $g(t)=(\zeta(t)-p).e^{-f(t)}$, we have 
\begin{eqnarray*}
\dot g(t)&=&-(\zeta(t)-p).\dot f(t).e^{-f(t)}+\dot\zeta(t).e^{-f(t)}\\
&=&-(\zeta(t)-p).\frac{\dot\zeta(t)}{\gamma(t)-p}.e^{-f(t)}
+\dot\zeta(t).e^{-f(t)}\\
&=&-\dot\zeta(t).e^{-f(t)}+\dot\zeta(t).e^{-f(t)}\\
&=&0\,.
\end{eqnarray*}
Therefore $g(t)\equiv{\rm constant}$. Since $g(0)=1$, we have
$g(t)\equiv 1$ for all $t$,  
so $e^{f(t)}\equiv\gamma(t)-p$. Therefore
$\nu(\xi,p)=\frac{1}{2\pi i}(f(L)-f(0))$, and then 
(\ref{flawind-oo}) implies that (\ref{flawind}) holds.\proofend

\begin{proposition}\label{iscontin}
The winding number $\nu(\xi,p)$ 
depends continuously on the point $p$ and the curve $\xi$ 
as long as $p$ and $\xi$ vary in such a way that 
$p\in \mathbb{C}\backslash|\xi|$. More precisely,
if $\{p_j\}_{j=1}^\infty$ is a sequence of points of
$\mathbb{C}$ that converges to $p\in\mathbb{C}$, 
$\{\xi_j\}_{j=1}^\infty$ is a sequence of loops in
$\mathbb{C}$ that converges uniformly to a loop $\xi$, 
and $p\notin\mathbb{C}\backslash|\xi|$, then there exists 
a positive integer $j_*$ such that
$p_j\notin\mathbb{C}\backslash|\xi_j|$ and 
$\nu(\xi_j,p_j)=\nu(\xi,p)$ whenever $j\ge j_*$
\end{proposition}

\medbreak\noindent PROOF.\  Let $f$ be a continuous map
from $\rr$ to $\mathbb{C}$ having with the property that
\mbox{$e^{f(t)}=\xi(e^{2\pi it})-p\equaldef h(t)$} for all $t\in\rr$. 
Let $h_j(t)=\xi_j(e^{2\pi it})-p_j$. Then $h_j\to h$ as $j\to\infty$, 
uniformly on compact sets, so  $h_j\to h$ 
uniformly, since the $h_j$ are $1$-periodic. Since 
\mbox{$h(t)\neq 0$} for 
all $t$, the functions $g_j\equaldef\frac{h_j}{h}$ converge uniformly
to the constant
$1$ as $j\to\infty$. Let $D$ be the open disc $\{z\in\mathbb{C}:
\|z-1\|<1\}$. Let ${\rm Ln}:D\mapsto\mathbb{C}$ be the 
function defined in (E5). Fix $j_*\in\nn$ such that
$g_j(t)\in D$ whenever $t\in\rr$ and $j\ge j_*$. 
Then, if $j\ge j_*$, $h_j(t)$ must be $\neq 0$ for 
every $t$, so $p_j\notin|\xi_j|$. Define
$$
f_j(t)=f(t)+{\rm Ln}\Big(g_j(t)\Big)\,\,\,{\rm for}
\,\,\,t\in\rr\,,\,j\ge j_*.
$$
Then 
$$e^{f_j(t)}=e^{f(t)}.e^{{\rm Ln}(g_j(t))}
=h(t).g_j(t)=h_j(t)=\xi_j(e^{2\pi it})-p_j\,.
$$
Therefore 
\begin{eqnarray*}
\nu(\xi_j,p_j)&=&\frac{1}{2\pi i}(f_j(1)-f_j(0))\\
&=&\frac{1}{2\pi i}\Big(f(1)-f(0)+{\rm Ln}(g_j(1))-{\rm Ln}(g_j(0))\Big)\\
&=&\frac{1}{2\pi i}\Big(f(1)-f(0)\Big)\\
&=&\nu(\xi,p)
\end{eqnarray*}
whenever $j\ge j_*$, since ${\rm Ln}(g_j(1))={\rm Ln}(g_j(0))$, because
the function $g_j$ is 
\mbox{$1$-periodic}.) \proofend


\ssection{The no-retraction theorem}


\begin{theorem}
Let $R>0$, and write 
$$
D(R)=\{z\in\mathbb{C}:|z|\le 1\}\,,\qquad 
S(R)=\{z\in\mathbb{C}:|z|=1\}\,.
$$
Then there does not exist a continuous map $\rho:D(R)\mapsto S(R)$ such that
$\rho(z)=z$ whenever $z\in S(R)$. 
\end{theorem}

\medbreak\noindent PROOF.\  Suppose $\rho$ exists. For 
$0\le r\le 1$, define a loop $\xi_r:\mathbb{S}^1\mapsto\mathbb{C}$ 
by letting $\xi_r(e^{2\pi it})=\rho(re^{2\pi it})$. Then 
each $\xi_r$ satisfies $|\xi_r(e^{2\pi it})|=1$ for all $t$,
so $0\notin|\xi_r|$. Let $w(r)=\nu(\xi_r,0)$. Therefore
Proposition \ref{iscontin} implies that the integer-valued
function $[0,1]\ni r\mapsto w(r)\in\rr$ is continuous.  
Hence $w(r)\equiv{\rm constant}$, so $w(0)=w(1)$. Furthermore,
$\xi_0$ is a constant map, from which it follows easily
that $\nu(\xi_0,0)=0$. On the other hand, 
since $\xi_1(e^{2\pi it})=\rho(e^{2\pi it})=e^{2\pi it}$, 
the function $f:\rr\mapsto\mathbb{C}$ given by 
$f(t)=2\pi it$ satisfies $e^{f(t)}=\xi_1(e^{2\pi it})$, 
so $\nu(\xi_1,0)=\frac{f(1)-f(0)}{2\pi i}=1$. So
$w(1)\neq w(0)$, and we have reached a contradiction.\proofend


\ssection{The Jordan curve theorem}

If $K$ is a compact subset of $\mathbb{C}$, and $p\in\mathbb{C}\backslash K$,
the {\em Borsuk map} $\beta_{p,K}:K\mapsto\mathbb{S}^1$ is defined by
$$
\beta_{p,K}(z)=\frac{z-p}{|z-p|}\qquad{\rm for}\,\,z\in K\,.
$$

\begin{proposition}\label{rice}
Let $K$ be a compact subset of $\mathbb{C}$, let $p\in\mathbb{C}\backslash K$,
and let $U$ be the connected component of $\mathbb{C}\backslash K$  
that contains $p$. Assume that $U$ is bounded.
Then the  Borsuk map $\beta_{p,K}$ cannot be extended
to a continuous map $\beta:K\cup U\mapsto\mathbb{S}^1$. 
\end{proposition}
\medbreak\noindent PROOF.\ Assume $\beta$ exists. Let $R$ be positive and so 
large that $K\cup U$ is entirely contained in the  
disc $D(p,R)=\{z\in\mathbb{C}:|z-p|\le R\}$. Define a map $\tilde\beta:
D(p,R)\mapsto\partial D(p,R)$ by letting 
$$
\tilde\beta(z)=\left\{\begin{array}{lcl}
p+R\cdot\beta(z)&{\rm if}&z\in K\cup U\,,\\
p+R\cdot\frac{z-p}{|z-p|}&{\rm if}&z\in D(p,R)\backslash U\,.\end{array}\right.
$$
Then $\tilde\beta$ is well defined, because
$(D(p,R)\backslash U)\cap(K\cup U)\subseteq K$ and
$\beta(z)=\frac{z-p}{|z-p|}$ if $z\in K\cup U$. Furthermore, 
$\tilde\beta$ is continuous, because both sets 
$D(p,R)\backslash U$ and $K\cup U$ are closed.
So $\tilde\beta$ is a continuous map from 
$D(p,R)$ to $\partial D(p,R)$. Moreover, it is clear that 
$\tilde\beta(z)=z$ if $z\in\partial D(p,R)$. So 
$\tilde\beta$ is a continuous retraction from 
$D(p,R)$ to $\partial D(p,R)$. Since such a retraction does not exist,
we have reached a contradiction.\proofend

\begin{proposition}\label{feinst}
Let $K$ be a compact subset of $\mathbb{C}$, and
let $Q$ be a subset of $\mathbb{C}\backslash K$ that contains
at least one point of each bounded connected component
of $\mathbb{C}\backslash K$. Then every continuous
map $f:K\mapsto\mathbb{S}^1$ can be extended to a continuous
map $F:\mathbb{C}\backslash Q\mapsto\mathbb{S}^1$.
\end{proposition}
\medbreak\noindent PROOF.\ For each positive number $r$ such that
$r<\sqrt{2}$, we define 
$$
S_r=\{s\in\mathbb{S}^1:|s-1|\le r\}\,,
$$
and let $\rho(r)=\frac{r}{2}\sqrt{4-r^2}$. Then the set $S_r$
can be identified with the real interval $I_r\equaldef[-\rho(r),\rho(r)]$ 
by means of the map 
\begin{equation}\label{themap}
I_r\ni u\mapsto\mu_r(u)\equaldef\sqrt{1-u^2}+ui\in\mathbb{S}^1\,.
\end{equation}

Now fix $\alpha$ such that
$0<\alpha<\frac{\sqrt{2}}{3}$. Choose a positive number $\delta$ such that
$|f(z)-f(z')|<\alpha$ whenever $z,z'\in K$ and $|z-z'|\le 2\sqrt{2}\delta$. 
Let $G^\delta$ be the $\delta$-grid of $\mathbb{C}$, i.e., the set
of all rectangles 
$$
R_{n,m}^{\delta}=\{x+iy:\delta n\le x\le\delta(n+1),
\delta m\le x\le\delta(m+1)\}\,,
$$
for all $n,m\in\zz$. For each $R\in G^\delta$ we let $V(R)$, 
$E(R)$ be, respectively,  the set 
of vertices of $R$ and the set of edges of $R$.

Let $G_K^\delta$ be the set of those 
rectangles $R\in G^\delta$ such that $R$ 
intersects $K$. Define
$$
|G_K^\delta|\,\,\equaldef\,\,\bigcup\{R:R\in G^\delta\}\,, \qquad
\tilde K\,\,\equaldef\,\,K\cup|G_K^\delta|\,.
$$

We will first extend $f$ to a continuous map 
$\tilde f:\tilde K\mapsto\mathbb{S}^1$. For this purpose, we pick
a point $z_R\in K\cap R$ for each
$R\in G_K^\delta$. Let 
$$
V_K=\bigcup\{V(R):R\in G_K^\delta\}\,.
$$ 
Let $K_1=K\cup V_K$. We then 
extend $f$ to a continuous map $f_1:K_1\mapsto\mathbb{S}^1$ 
by defining $f_1(z)$ as follows, if $z\in K_1$: 
\begin{itemize}
\item[1.] If $z\in K$, we define $f_1(z)=f(z)$.
\item[2.] If $z\in V_K\backslash K$, we take $f_1(z)$ to be 
$f(z_R)$, where $R$ is such that 
$z\in R$ and $R\in G_K^\delta$. (For a given $z$, 
the rectangle $R$ exists, but need not be unique, since $z$ could 
belong to two, three or four rectangles in $G_K^\delta$.
If $R$ is not unique, we choose one for each $z$, 
in an arbitrary fashion.)
\end{itemize}
With this definition, $f_1$ satisfies the inequality
\begin{equation}\label{implpl}
|f_1(z)-f_1(z_R)|<\alpha\quad
{\rm whenever}\quad z\in R\cap K_1\,,\,\,R\in G_K^\delta\,.
\end{equation}
Indeed, if $z\in R\cap K$, then 
$|z-z_R|\le\sqrt{\delta}$, so 
$|f_1(z)-f_1(z_R)|<\alpha$. If $z\in V_K$ but 
$z\notin K$, then $f_1(z)=f(z_{R'})$ for some 
rectangle $R'\in G_K^\delta$ such that 
$R'\cap R\neq\emptyset$.  Then 
$|z_{R'}-z_{R}|\le 2\sqrt{2}\delta$, so 
$|f_1(z_{R'})-f_1(z_{R})|<\alpha$, and then
$|f_1(z)-f_1(z_{R})|<\alpha$. 

Next, we let 
$$
E_K=\bigcup\{E(R):R\in G_K^\delta\}\,,\qquad|E_K|=\bigcup\{e:e\in E_K\}\,.
$$
Let $K_2=K\cup|E_K|=K_1\cup|E_K|$. We extend $f_1$ to a continuous map 
$f_2:K_2\mapsto\mathbb{S}^1$ by defining $f_2$ as follows: 
\begin{itemize}
\item[1.] We let $f_2(z)=f_1(z)$ if $z\in K_1$.
\item[2.] For each edge $e\in E_K$, we pick a point 
$z_e\in e\cap K_1$. (This is possible because
$e\cap K_1\neq\emptyset$, since the vertices of
$e$ belong to $K_1$.) Then the inequality
$|f_1(z)-f_1(z_e)|<2\alpha$ holds for all 
$z\in e\cap K_1$, because there exists $R\in G_K^\delta$
such that $e\subseteq R$, and then 
$|f_1(z)-f_1(z_R)|<\alpha$ for all $z\in e\cap K_1$. Then
$$
e\cap K_1\ni z\mapsto\frac{f_1(z)}{f_1(z_e)}\in\mathbb{S}^1
$$
is a continuous map from $e\cap K_1$ to the set $S_{2\alpha}$ (thanks 
to the identity $\Big|\frac{f_1(z)}{f_1(z_{e})}-1\Big|=|f_1(z)-f_1(z_e)|$, 
which is true because $|f_1(z_e)|=1$). Since $2\alpha<\sqrt{2}$, 
we know that $S_{2\alpha}$ can be identified with the real interval 
$I_{2\alpha}$ by means of the map $\mu_{2\alpha}$  defined in
(\ref{themap}). Then the Tietze extension 
theorem implies that there exists a map $\varphi_e:e\mapsto S_{2\alpha}$ 
such that $\varphi_e(z)=\frac{f_1(z)}{f_1(z_e)}$ if 
$z\in e\cap K_1$. We can then define $f_2$ on $e$ by letting
$f_2(z)=\varphi_e(z)\cdot f_1(z_e)$ for $z\in e$. 
\end{itemize}
We now claim that $f_2$ satisfies the inequality
\begin{equation}\label{impl3}
|f_2(z)-f_2(z_{R})|<3\alpha\quad
{\rm whenever}\quad z\in R\cap K_2\,,\,\,R\in G_K^\delta\,.
\end{equation}
Indeed, if $z\in R\cap K_1$, then 
$f_2(z)=f_1(z)$ and $f_2(z_R)=f_1(z_R)$, 
so (\ref{implpl}) implies that $|f_2(z)-f_2(z_R)|<\alpha$. 
If $z\in R$ but $z\notin K_1$, then $z$ belongs to
and edge $e\in E(R)$. Then 
(\ref{implpl}) implies that $|f_2(z_e)-f_2(z_R)|=
|f_1(z_e)-f_1(z_R)|<\alpha$. On the other hand, 
$$
|f_2(z)-f_2(z_e)|=|f_2(z_e)|\cdot\Big|\frac{f_2(z)}{f_2(z_e)}-1\Big|
=|\varphi_e(z)-1|<2\alpha\,.
$$
Therefore $|f_2(z)-f_2(z_R)|<3\alpha$.

We now define the extension $\tilde f:\tilde K\mapsto\mathbb{S}^1$.
For each rectangle $R\in G_K^\delta$, the map $f_2$ is defined on the
compact set $K_2\cap R$, which contains the boundary of
$R$. Furthermore, (\ref{impl3}) implies that
$$
K_2\cap R\ni z\mapsto\frac{f_2(z)}{f_2(z_R)}\in\mathbb{S}^1
$$
is a continuous map to the set $S_{3\alpha}$.  Since
$3\alpha<\sqrt{2}$, the set $S_{3\alpha}$ can be identified with the
real interval $I_{3\alpha}$ by means of the map $\mu_{3\alpha}$ of
(\ref{themap}). Then the Tietze extension theorem implies that there
exists a map $\psi_R:R^\delta \mapsto S_{3\alpha}$ such that
$\psi_R(z)=\frac{f_2(z)}{f_2(z_R)}$ if $z\in K_2\cap R$. We then
define $\tilde f$ on $R$ by letting $\tilde f(z)=\psi_R(z)\cdot
f_2(z_R)$ for $z\in R$.

Let $W$ be the set of all vertices of $G^\delta$ that do not belong to
$\tilde K$. We let $\tilde K_1=\tilde K\cup W$, and
extend $\tilde f$ to a map $\tilde f_1:\tilde K_1\mapsto\mathbb{S}^1$
by choosing $\tilde f_1(z)\in\mathbb{S}^1$ in an arbitrary fashion
if $z\in W$. We then let $H$ be the set of all edges of $G^\delta$
that are not contained in $\tilde K$, and define
$|H|=\bigcup\{h:h\in H\}$. We define $\tilde K_2=\tilde K_1\cup|H|$,
and extend $\tilde f_1$ to a continuous map 
$\tilde f_2:\tilde K_2\mapsto\mathbb{S}^1$ as follows: for each 
$h\in H$, let $v$, $w$ be the vertices of $h$, pick real numbers 
$a$, $b$ such that $e^{2\pi ia}=v$ and $e^{2\pi ib}=w$, and
define $\tilde f_2(tv+(1-t)w)=e^{2\pi i(ta+(1-t)b)}$ 
if $0\le t\le 1$. 

Next, we let  $\mathcal{R}=G^\delta\backslash G_K^\delta$, so 
$\mathcal{R}$ is the set of all rectangles $R\in G^\delta$ that 
do not intersect $K$. For each $R\in\mathcal{R}$, use 
$c_R$ to denote the center of $R$. The map $\tilde f_2$ is defined
on the boundary $\partial R$. Pick, for each $R$,
a continuous map $p_R:R\backslash\{c_R\}\mapsto\partial R$
such that $p_R(z)=z$ whenever $z\in\partial R$.
We then let $\tilde K_3=\tilde K_2\cup\bigcup\{R\backslash\{c_R\}:
R\in\mathcal{R}\}$ and extend $\tilde f_2$ to 
$\tilde K_3$ by letting 
$$
\tilde f_3(z)=\left\{\begin{array}{lcl}
\tilde f_2(z)&{\rm if}&z\in\tilde K_2\,,\\
\tilde f_2(p_R(z))&{\rm if}&z\in R\backslash\{c_R\}\,\,{\rm and}\,\,
R\in\mathcal{R}\,.\end{array}\right.
$$

We have now extended $f$ to a continuous map
$\tilde f_3:\tilde K_3\mapsto\mathbb{S}^1$, where  
$\tilde K_3$ is the whole complex plane $\mathbb{C}$, minus
the discrete set $C=\{c_R:R\in\mathcal{R}\}$.  Then
$C\subseteq\mathbb{C}\backslash K$. If $U$ is any bounded
connected component of $\mathbb{C}\backslash K$, then 
$C\cap U$ is finite. Pick a point $q_U\in Q\cap U$,
and define $C_U^0=(C\cap U)\cup\{q_U\}$.

The map $\tilde f_3$ is then continuous on $U\backslash C_U^0$. 
We show how to replace the restriction 
$\tilde f_{3,U}$ of $\tilde f_3$ to $U\backslash C_U^0$
by a map $\tilde f_{4,U}$ into $\mathbb{S}^1$ which is 
continuous on $U\backslash\{q_U\}$ and coincides with 
$\tilde f_{3,U}$ outside a compact subset of $U$ that contains
$C_U^0$. If $C_U^0=\{q_U\}$, then of course 
we can take $\tilde f_{4,U}=\tilde f_{3,U}$. Otherwise,
we pick a point $P$ of $C_U^0\backslash\{q_U\}$, and
construct a polygonal curve $\xi:[a,b]\mapsto U$ 
such that $\xi(a)=P$, $\xi(b)=q_U$, the map $\xi$ is 
one-to-one, and $\xi(t)\notin C_U^0$ whenever 
$a<t<b$. Let $(P_0,\ldots,P_k)$ be the ordered sequence 
of vertices of $\xi$, so that $P_0=P$, $P_k=q_U$, 
and there exist $t_0,\ldots,t_k$ with the property that
$a=t_0<t_1<\ldots<t_{k-1}<t_k=b$, 
$\xi(t_j)=P_j$ for $j=0,\ldots,k$, and 
$\xi$ is linear on each interval $[t_{j-1},t_j]$. 

Let $\sigma_j=\{\xi(t):t_{j-1}\le t\le t_j\}$, so 
$\sigma_j$ is the segment  joining $P_{j-1}$ to $P_j$.
Then $\sigma_0$ does not contain any points of 
$C_U^0$ other than $P$ and $q_U$. Therefore the map $\tilde f_{3,U}$ 
is continuous on a neighborhood of $\sigma_0$, except possibly
at the endpoints of $\sigma_0$. Let $\tau_0$ be the compact neighborhood  
$\{z:{\rm dist}(z,\sigma_0)\le\theta\}$ of $\sigma_0$, where 
$\theta$ is positive and small enough, so that 
$\tau_0\subseteq U$ and $\tau_0$ does not contain any points of 
$C_U^0$ other than $P$ and $q_U$. Then $\tilde f_{3,U}$ 
is continuous on $\tau_0$, except possibly
at $P_0$ and $P_1$. The set $\tau_0\backslash\{P_1\}$ obviously admits 
a continuous retraction
$\kappa_0:\tau_0\backslash\{P_1\}\mapsto\partial\tau_0$. Replace
$\tilde f_{3,U}$ by $\tilde f_{3,U}\circ\kappa_0$ on 
$\tau_0\backslash\{P_1\}$. Then the modified map 
$\tilde f_{3,U}^0$ is continuous wherever 
$\tilde f_{3,U}$ was, except possibly at $P_1$, and
is also continuous at $P_0$. So the discontinuity at $P_0$ has 
been eliminated, and a possible discontinuity at $P_1$ has been 
introduced. Repeating this procedure, we replace the discontinuity 
at $P_1$ by a possible discontinuity at $P_2$, and continue 
in this way until, of all the points of $\xi$, only the terminal
point $q_U$ remains as a possible discontinuity of the new map.
This means that the set $C_U^0$ of points of possible discontinuity 
of $\tilde f_{3,U}$ has been reduced by eliminating one point 
other than $q_U$.  We then repeat this procedure, until
all points of discontinuity other than $q_U$ have been eliminated.

By carrying out this elimination of discontinuities separately
on each bounded
connected component of $\mathbb{C}\backslash K$, we obtain a
continuous extension $\hat f:\mathbb{C}\backslash(Q\cup
D)\mapsto\mathbb{S}^1$ of $f$ to the whole complex plane, minus  
$Q\cup D$, where $D$ is a discrete subset which is entirely contained
in the unbounded component $\Omega$ of $\mathbb{C}\backslash K$. 
The set $D$ can then be completely eliminated, by 
first picking a positive $L$ such that $K\subseteq\{z:|z|\le L\}$,
and then eliminating all the discontinuities in 
$\{z:|z|\le L\}$ as was done for the bounded components. We then 
let $\kappa$ be the retraction $z\mapsto L\frac{z}{|z|}$ from 
$\{z:|z|\ge L\}$ to $\{z:|z|=L\}$, and replace $\hat f$ 
by $\hat f\circ\kappa$ on $\{z:|z|\ge L\}$. The resulting
map $F$ now has no discontinuities whatsoever on $\Omega$, so 
$F$ is the desired extension.\proofend

\medbreak
If $K$ is a subset of $\mathbb{C}$, and $f_0$, $f_1$ are
continuous maps from $K$ to $\mathbb{S}^1$, 
a {\em homotopy} from $f_0$ to $f_1$ is a continuous map 
$h:K\times[0,1]\mapsto\mathbb{S}^1$ 
such that $h(z,0)=f_0(z)$ and 
$h(z,1)=f_1(z)$ whenever $z\in K$. We say that $f_0$ and $f_1$ are
{\em homotopic} if there exists a 
homotopy from $f_0$ to $f_1$.

\begin{proposition}\label{peres}
Let $K$ be a compact subset of $\mathbb{C}$, and let $C$ 
be a closed subset of $\mathbb{C}$ such that 
$K\subseteq C$. Let $f_0$, $f_1$ be continuous maps
from $K$ to $\mathbb{S}^1$, such  that $f_0$ and $f_1$ are homotopic, 
and $f_0$ admits a continuous extension to a map 
$F_0:C\mapsto\mathbb{S}^1$. Then 
$f_1$ admits a continuous extension to a map 
$F_1:C\mapsto \mathbb{S}^1$. 
\end{proposition}
\medbreak\noindent PROOF.\ Let $h:K\times[0,1]\mapsto\mathbb{S}^1$ 
be a continuous map that satisfies the conditions $h(z,0)=f_0(z)$ and 
$h(z,1)=f_1(z)$ whenever $z\in K$. The map $h$ is uniformly continuous
on $K\times[0,1]\mapsto\mathbb{S}^1$, so 
there exists a positive $\delta$ such that 
$|h(z,t)-h(z,s)|<1$ whenever $z\in K$, $t,s\in[0,1]$, and 
$|t-s|\le\delta$. Let $N$ be a positive integer such that
$N\delta>1$. For $k=0,\ldots,N$, let $g_k$ be the map
$$
K\ni z\mapsto g_k(z)\equaldef h\Big(z,\frac{k}{N}\Big)\in\mathbb{S}^1\,.
$$
Then $g_0=f_0$ and $g_N=f_1$.  We prove by induction on $k$ that
$g_k$ has a continuous extension to $C$. The fact that 
$g_0$ has a continuous extension follows because $g_0=f_0$. 
Assuming that $g_k$ has a continuous extension $G_k$, and $k<N$,
use the inequality $|g_{k}(z)-g_{k+1}(z)|<1$, which 
is valid for all $z\in K$, to conclude that 
$|\theta_k(z)-1|<1$ for all $z\in K$, where 
$\theta_k(z)\equaldef\frac{g_{k+1}(z)}{g_{k}(z)}$.

So $\theta_k$ is a continuous map from $K$ to the set 
\mbox{$S_1=\{z\in\mathbb{S}^1:|z-1|\le 1\}$}. This set can be identified
with the real interval $I_1=[-\frac{\sqrt{3}}{2},\frac{\sqrt{3}}{2}]$, 
as explained in the proof of Proposition \ref{feinst}.
Therefore the Tietze extension theorem yields an extension 
of $\theta_k$ to a continuous map $\Theta_k:C\mapsto\mathbb{S}^1$. 
Define $G_{k+1}(z)=\Theta_k(z)\cdot G_k(z)$ for $z\in C$. 
Then $G_{k+1}(z)$ is a continuous map from $C$ to $\mathbb{S}^1$,
and  $G_{k+1}(z)=g_{k+1}(z)$ whenever $z\in K$.

In particular, $g_N$ has a continuous extension. Since $g_N=f_1$,\
our proof is complete. \proofend

\begin{proposition}\label{engag}
Let $K$ be a compact subset of $\mathbb{C}$. Then 
$\mathbb{C}\backslash K$ is connected if and only if 
every continuous map $f:K\mapsto\mathbb{S}^1$ is homotopic to a constant.
\end{proposition}
\medbreak\noindent PROOF.\ If $\mathbb{C}\backslash K$ is connected, 
and $f:K\mapsto\mathbb{S}^1$ is continuous, then 
Proposition \ref{feinst} implies that $f$ has
a  continuous extension $F:\mathbb{C}\mapsto\mathbb{S}^1$.
If we define $h(z,t)=F((1-t)z)$ for $z\in K$ and $t\in[0,1]$, then
$h$ is clearly a homotopy from $f$ to the constant 
map $K\ni z\mapsto F(0)\in\mathbb{S}^1$.

To prove the converse, suppose every continuous map
$f:K\mapsto\mathbb{S}^1$ is homotopic to a constant. Since a constant
map obviously admits a continuous extension to $\mathbb{C}$,
Proposition \ref{peres} implies that every continuous map
$f:K\mapsto\mathbb{S}^1$ admits a continuous extension to a map
$F:\mathbb{C}\mapsto\mathbb{S}^1$.  In particular, if $U$ is a bounded
connected component of $\mathbb{C}\backslash K$, and $p\in U$, then
the Borsuk map $\beta_{p,K}$ would admit a continuous extension.  But
Proposition \ref{rice} says that $\beta_{p,K}$ does not have a
continuous extension. Therefore $\mathbb{C}\backslash K$ has no
bounded connected components.  Since  $\mathbb{C}\backslash K$ has 
only one unbounded connected component, we have established 
that  $\mathbb{C}\backslash K$ is connected, as desired.\proofend

\begin{proposition}\label{iscon}
Let $K$ be a compact subset of $\mathbb{C}$. Assume that  
$K$ is homeomorphic to the interval $[0,1]$. Then 
$\mathbb{C}\backslash K$ is connected.
\end{proposition}
\medbreak\noindent PROOF.\ It is clear that every 
continuous map from $[0,1]$ to $\mathbb{S}^1$ is homotopic
to a constant. The conclusion then follows from 
Proposition \ref{engag}.\proofend

\begin{proposition}\label{weknow}
Let $K$ be a compact subset of $\mathbb{C}$. Assume that  
$K$ is homeomorphic to $\mathbb{S}^1$. Then 
\begin{itemize}
\item[1.] $\mathbb{C}\backslash K$ is not connected;
\item[2.] if $U$ is any connected component of
$\mathbb{C}\backslash K$, then the closure of $U$ is the set
$U\cup K$.
\end{itemize}
\end{proposition}
\medbreak\noindent PROOF.\ The identity map $z\mapsto z$ from 
$\mathbb{S}^1$ to $\mathbb{S}^1$ is not  homotopic to a constant. 
(For example,
if it was homotopic to a constant then Proposition \ref{peres}
would yield a continuous extension to the disc 
$D(0,1)=\{z\in\mathbb{C}:|z|\le 1\}$. This extension
would be a continuous retraction from the disc to its boundary,
contradicting the no-retraction theorem.)

Therefore not every continuous map from 
$\mathbb{S}^1$ to $\mathbb{S}^1$ is homotopic to a constant. 
So not every continuous map from 
$K$ to $\mathbb{S}^1$ is homotopic to a constant. Therefore 
Proposition \ref{engag} implies that 
$\mathbb{C}\backslash K$ is not connected.

Now let $U$ be a connected component of 
$\mathbb{C}\backslash K$,  and let $\bar U$ denote its closure.
Then $U\subseteq\bar U\subseteq U\cup K$. To prove our second conclusion,
we have to pick $z\in K$ and show that $z\in\bar U$. Let 
$B$ be an open disc with center $z$, and let 
$B'$ be a compact ball with center $z$ such that
$B'\subseteq B$. Let $J$ be a nonempty subset of 
$K$ which is contained in $B$ and is such that 
$K'=K\backslash J$ is homeomorphic to $[0,1]$. Let $V$ be another connected
component of $\mathbb{C}\backslash K$. Pick points 
$p\in U$, $q\in V$. Then $p$ and $q$ belong to 
$\mathbb{C}\backslash K'$, which is connected by 
Proposition \ref{iscon}. So there exists a continuous
path $\xi:[0,1]\mapsto\mathbb{C}\backslash K'$
such that $\xi(0)=p$ and $\xi(1)=q$. Then 
$\xi$ cannot be contained in $\mathbb{C}\backslash K$, because
it joins points in two different components of
$\mathbb{C}\backslash K$. Therefore 
$\xi(t)\in J$ for some $t$, and then 
$\xi(t)\in B'$ for some $t$. Let $\bar t$ be the smallest 
$t\in[0,1]$ such that $\xi(t)\in B'$. If $\bar t=0$, then 
$p\in B'$, so $B'\cap U\neq\emptyset$ and then
$B\cap U\neq\emptyset$. If $\bar t>0$, then 
the set $\Xi=\{\xi(t):0\le t<\bar t\}$ is connected and
entirely contained in $U$ (because $\Xi$ does not intersect
$K$). Since $\xi(\bar t)\in B'$, $\xi(t)$ must belong to
$B$ for $t<\bar t$, $t$ sufficiently close to $\bar t$. Therefore
$B\cap U\neq\emptyset$.

We have shown that $B\cap U\neq\emptyset$
if $B$ is an arbitrary open disc with center $z$. Therefore
$z\in\bar U$, and our proof is complete. \proofend

\begin{proposition}\label{islift}
Let $K$ be a compact subset of $\mathbb{C}$, and let 
$f:K\mapsto\mathbb{C}$ be a continuous map. Then 
$f$ is homotopic to a constant map if and only if
there exists a continuous function $\varphi:K\mapsto\rr$
such that $f(z)=e^{2\pi i\varphi(z)}$ for all
$z\in K$.
\end{proposition}
\medbreak\noindent PROOF.\ If $\varphi$ exists, then 
we can define $h(z,t)=e^{2\pi i(1-t)\varphi(z)}$ for $z\in K$ and 
$t\in[0,1]$, and it is clear that $h$ is a homotopy from
$f$ to a constant map. 

Conversely, suppose $q\in\mathbb{S}^1$ and
$h:K\times[0,1]\mapsto\mathbb{S}^1$ 
be a continuous map that satisfies the conditions $h(z,0)=f(z)$ and 
$h(z,1)=q$ whenever $z\in K$. Fix $\alpha$ such that 
$0<\alpha<\frac{1}{2}$. Then 
the map $[-\alpha,\alpha]\ni u e^{2\pi iu}$ is a bijection onto
the compact subset $S=\{e^{2\pi iu}:u\in[-\alpha,\alpha]\}$ 
of $\mathbb{S}^1$. Fix $\beta$ such that $0<\beta$ and
the set $V=\{z\in\mathbb{S}^1:|z-1|\le\beta\}$ is contained
in $S$. 

The map $h$ is uniformly continuous
on $K\times[0,1]\mapsto\mathbb{S}^1$, so 
there exists a positive $\delta$ such that 
$|h(z,t)-h(z,s)|<\beta$ whenever $z\in K$, $t,s\in[0,1]$, and 
$|t-s|\le\delta$. Let $N$ be a positive integer such that
$N\delta>1$. For $j=1,\ldots,N$, let $g_j$ be the map
$$
K\ni z\mapsto g_j(z)\,\,\equaldef\,\,\frac{h\Big(z,\frac{j-1}{N}\Big)}
{h\Big(z,\frac{j}{N}\Big)}\in\mathbb{S}^1\,.
$$
Let $G(z)\equiv q$. 
Then $f=g_1\cdot g_2\cdot\,\ldots\,\cdot g_N\cdot G$.
Clearly, each $g_k$ takes values in $S$, so there exists
a continuous $\psi_j:K\mapsto\rr$ such that
$g_j(z)=e^{2\pi i\psi_j(z)}$ for all $z\in K$. Let $c\in\rr$ be such that
$q=e^{2\pi i c}$. Define $\varphi=\psi_1+\cdots+\psi_N+c$. Then
$f(z)=e^{2\pi i\varphi(z)}$ for all $z\in K$. \proofend

\begin{proposition}\label{isconcroc}
Let $K$ be a compact subset of $\mathbb{C}$.  Assume that \mbox{$K$}
is the union of two compact sets $K_1$, $K_2$, such that $K_1\cap K_2$
is connected. Let $p_1$, $p_2$ be points of $\mathbb{C}\backslash K$
that belong to the same connected component of
$\mathbb{C}\backslash K_1$ and to the same connected component of 
$\mathbb{C}\backslash K_2$.  Then $p_1$, $p_2$ belong to the same 
connected component of $\mathbb{C}\backslash K$.
\end{proposition}
\medbreak\noindent PROOF.\ Since $p_1$ and $p_2$ belong to the same
connected component of $\mathbb{C}\backslash K_1$, the Borsuk maps
$\beta_{p_1,K_1},\beta_{p_2,K_1}:K_1\mapsto\mathbb{S}^1$ are 
homotopic. Therefore, if we let 
$$ 
f_1(z)\equaldef\frac{\beta_{p_2,K_1}(z)}{\beta_{p_1,K_1}(z)}\qquad
{\rm for}\quad z\in K_1\,,
$$
it follows that
the map $f_1$ is homotopic to a constant. By Proposition \ref{islift},
this implies that there exists a continuous function
$\varphi_1:K_1\mapsto\rr$ such that $f_1(z)=e^{2\pi i\varphi_1(z)}$ 
whenever $z\in K_1$. A similar argument shows that
there exists a continuous function
$\varphi_2:K_2\mapsto\rr$ such that $f_2(z)=e^{2\pi i\varphi_2(z)}$ 
whenever $z\in K_2$, where 
$$ 
f_2(z)\equaldef\frac{\beta_{p_2,K_2}(z)}{\beta_{p_1,K_2}(z)}\qquad
{\rm for}\quad z\in K_2\,.
$$
The function $\psi=\varphi_2-\varphi_1:K_1\cap K_2\mapsto\rr$ 
is continuous and integer-valued. Since $K_1\cap K_2$ is connected,
$\psi$ is in fact constant. After subtracting this constant 
from $\varphi_2$, we may assume that $\varphi_2=\varphi_1$ 
on $K_1\cap K_2$. We can the define $\varphi:K\mapsto\rr$ by letting
$\varphi(z)=\varphi_j(z)$ whenever $z\in K_j$, $j=1,2$. 
Then, if we let 
$$ 
f(z)\equaldef\frac{\beta_{p_2,K}(z)}{\beta_{p_1,K}(z)}\qquad
{\rm for}\quad z\in K\,,
$$
we have $f(z)=e^{2\pi i\varphi(z)}$. Therefore $f$ is homotopic to a 
constant, and then the maps $\beta_{p_1,K}$ and $\beta_{p_2,K}$ are 
homotopic. 

Clearly, $\beta_{p_2,K}$ has a continuous extension to a map
$\mu:\mathbb{C}\backslash\{p_2\}$. Since $\beta_{p_1,K}$ is
homotopic to $\beta_{p_2,K}$, Proposition \ref{peres}
tells us that $\beta_{p_1,K}$ also has a continuous extension
to a map $\nu:\mathbb{C}\backslash\{p_2\}$. 

If the connected components $U_1$, $U_2$ of $p_1$, $p_2$ 
in $\mathbb{C}\backslash K$ were different, $p_2$ would not 
belong to $U_1$, and then $\nu$ could be restricted to 
$K\cup U_1$, yielding a continuous extension of $\beta_{p_1,K}$ to
$K\cup U_1$. On the other hand, Proposition \ref{rice}
tells us that such an extension does not exist. Therefore
$U_1=U_2$, completing our proof. \proofend



\begin{theorem}\label{jordantheo}
(The Jordan curve theorem.) 
Let $K$ be a compact subset of $\mathbb{C}$. Assume that  
$K$ is homeomorphic to $\mathbb{S}^1$. Then 
\begin{itemize}
\item[1.] $\mathbb{C}\backslash K$ has exactly two connected
components;
\item[2.] if $U$ is any connected component of
$\mathbb{C}\backslash K$, then ${\rm Closure}(U)=U\cup K$.
\end{itemize}
\end{theorem}
\medbreak\noindent PROOF.\ The second conclusion 
was already proved in Proposition \ref{weknow}. Furthermore,
we also know from Proposition \ref{weknow}  that 
$\mathbb{C}\backslash K$ has at least two connected components.
Therefore, it suffices to show that 
$\mathbb{C}\backslash K$ cannot have two different bounded 
connected components.

Let $\xi:\mathbb{S}^1\mapsto K$ be a homeomorphism, and let
$\gamma(t)=\xi(e^{2\pi it})$ for \mbox{$0\le t\le 1$}, so 
$\gamma$ is a continuous map from $[0,1]$ onto $K$ such that
$\gamma(0)=\gamma(1)$ and $\gamma$ is one-to-one on $[0,1[$.

Suppose $U_1$, $U_2$ are different bounded connected components of
$\mathbb{C}\backslash K$. Let $U_0$ be the unbounded component of
$\mathbb{C}\backslash K$. Fix points $p_0$, $p_1$, $p_2$ in $U_0$, 
$U_1$, $U_2$, respectively. Fix eight times $t_1$,
$t_2$, $\ldots$, $t_8$ having the property  that 
\mbox{$0<t_1<t_2<t_3<t_4<t_5<t_6<t_7<t_8<1$}. Write 
$J_\ell=[t_{2\ell-1},t_{2\ell}]$ and $I_\ell=]t_{2\ell-1},t_{2\ell}[$ 
for $\ell=1,2,3,4$. For each $\ell\in\{1,2,3,4\}$, define
a compact set $K_\ell$ by letting 
$K_\ell=K\backslash\gamma(I_\ell)$. Then each $K_\ell$ is homeomorphic
to a compact interval, so the open 
sets $\Omega_\ell=\mathbb{C}\backslash K_\ell$ are connected
by Proposition \ref{iscon}. 

Since $p_0$ and $p_1$ belong to $\Omega_\ell$ for 
each $\ell$, there exist
continuous one-to-one maps $\eta_\ell:[0,1]\mapsto\Omega_\ell$ such that 
$\eta_\ell(0)=p_0$ and $\eta_\ell(1)=p_1$. (We are only interested
in the curves $\eta_1$ and $\eta_3$.) Since $p_0$ and
$p_1$ belong to different connected components of
$\mathbb{C}\backslash K$, the curves $\eta_\ell$ must intersect 
$K$. Since $\eta_\ell$ does not intersect $K_\ell$, it must
intersect $\gamma(J_\ell)$. If we let $s_\ell$ be the smallest
$s\in[0,1]$ such that $\eta_\ell(s)\in\gamma(J_\ell)$, then 
$\eta_\ell(s_\ell)\in\gamma(J_\ell)$, and $\eta_\ell(s)\in U_0$
for $0\le s<s_\ell$. Define $\zeta_\ell:[0,1]\mapsto\mathbb{C}$ 
by restricting $\eta_\ell$ to $[0,s_\ell]$ and reparametrizing  
time so that  the interval $[0,s_\ell]$ becomes $[0,1]$. Then
$\zeta_\ell$ is a continuous one-to-one map from
$[0,1]$ to $\mathbb{C}$, such that $\zeta_\ell(0)=p_0$,
$\zeta_\ell(1)\in\gamma(J_\ell)$, and $\zeta_\ell(t)\in U_0$ for
$0\le t<1$. 

The sets $\zeta_3([0,1])$ and $\gamma(J_1)$ are compact and do not intersect,
because $\zeta_3([0,1])\cap K=\{\zeta_3(1)\}$, and 
$\zeta_3(1)\notin\gamma(J_1)$.  Therefore $\zeta_1(t)\notin
\zeta_3([0,1])$ if $t$ is close enough to $1$. Let $\bar t_1$
be the largest $t\in[0,1]$ such that 
$\zeta_1(t)\in\zeta_3([0,1])$. Let $\bar t_3$ be the unique 
$t\in[0,1]$ such that $\zeta_3(t)=\zeta_1(\bar t_1)$.
If $\bar t_1>0$, replace $p_0$ by $\zeta_1(\bar t_1)$,
and then replace $\zeta_1$ and $\zeta_3$ by their restrictions
to $[t_1,1]$ and $[t_3,1]$, reparametrized so that 
the intervals $[t_1,1]$,  $[t_3,1]$ become $[0,1]$. 

We then have two continuous maps $\zeta_1,\zeta_3:[0,1]\mapsto\mathbb{C}$
such that 
\begin{itemize}
\item[1.] $\zeta_1$ and $\zeta_3$ are one-to-one,
\item[2.] $\zeta_1(0)=\zeta_3(0)=p_0$,
\item[3.] $\zeta_1(t)\in U_0$ and $\zeta_3(t)\in U_0$ whenever 
$0\le t<1$, 
\item[4.] $\zeta_1(1)\in\gamma(J_1)$ and $\zeta_3(1)\in\gamma(J_3)$, 
\item[5.] $\zeta_1(t)\neq\zeta_3(s)$ whenever $0\le t\le 1$
$0\le s\le 1$, and $t+s>0$. 
\end{itemize}

Let $\tau_1$, $\tau_3$ be the unique points in $J_1$, $J_3$
such that $\gamma(\tau_1)=\zeta_1(1)$ and $\gamma(\tau_3)=\zeta_3(1)$.
We construct two simple closed curves $\Lambda_1$, 
$\Lambda_2$, as follows. The curve $\Lambda_1$ is the concatenation
of $\zeta_1$, followed by the restriction of
$\gamma$ to the interval $[\tau_1,\tau_3]$, followed 
by $\zeta_3$ run backwards. 
The curve $\Lambda_3$ is defined similarly, except 
for the fact that, to go from $\gamma(\tau_1)$ to 
$\gamma(\tau_3)$ we move along $K$ in the other direction (that
is, we follow the map $s\mapsto \xi(e^{2\pi i(\tau_1-s)})$ 
from $s=0$ until $s=1+\tau_1-\tau_3$, at which 
point $\xi(e^{2\pi i(\tau_1-s)})$ becomes $\gamma(\tau_3)$). 

We now show that $p_1$ and $p_2$ can be connected by a continuous path
$\omega_1$ in $\mathbb{C}\backslash|\Lambda_1|$, where $\Lambda_1$ is,
of course, the carrier of $\Lambda_1$. To see this, it suffices to
construct continuous paths $\theta_j:[0,1]\mapsto\mathbb{C}$, for
$j=1,2$, such that $\theta_1(0)=p_1$, $\theta_2(0)=p_2$,
$\theta_1(t)\in U_1$ for $0\le t<1$, $\theta_2(t)\in U_2$ for
$0\le t<1$, $\theta_1(1)\in\gamma(J_4)$, and $\theta_2(1)\in\gamma(J_4)$.
We then define $\omega_1$ by concatenating 
$\theta_1$, a portion of $\gamma$ going from 
$\theta_1(1)$ to $\theta_2(1)$ along $\gamma(J_4)$, and
$\theta_2$ run backwards. The resulting 
path $\omega_1$ is entirely contained in $U_1\cup U_2\cup\gamma(J_4)$, 
so $\omega_1$ does not intersect $|\Lambda_1|$. 

A similar argument shows that there exists a continuous path
$\omega_3$ connecting  $p_1$ and $p_2$ in 
$\mathbb{C}\backslash|\Lambda_3|$.  Since 
$|\Lambda_1|\cap|\Lambda_3|=|\zeta_1|\cup|\zeta_2|$, which 
is homeomorphic to a compact interval, Proposition 
\ref{isconcroc} tells us that there exists a continuous path
$\omega$ connecting  $p_1$ and $p_2$ in 
$\mathbb{C}\backslash(|\Lambda_1|\cup|\Lambda_3|)$.  Since
$K\subseteq|\Lambda_1|\cup|\Lambda_3|)$, 
$\omega$ connects  $p_1$ and $p_2$ in 
$\mathbb{C}\backslash K$, contradicting the assumption
that $U_1$ and $U_2$ are different connected components of 
$\mathbb{C}\backslash K$.\proofend


\ssection{The winding numbers of a simple loop}

The {\em Jordan curve theorem} says that if $\xi$ is a simple 
loop then $\rr^2\backslash|\xi|$ has exactly two connected
components. In other words, 
$\rr^2\backslash|\xi|$ is the union of two 
disjoint connected open subsets $\Omega_1$, $\Omega_2$ of $\rr^2$.
If $R\in\rr$ is such that $R>0$ and $\|\xi(s)\|\le R$ for all 
$s\in\mathbb{S}^1$, then the set
$S=\{q\in\rr^2:\|q\|>R\}$ is open, connected, and disjoint from
$|\xi|$. Therefore $S$ is contained in one of the components
$\Omega_1$, $\Omega_2$. The other component
must then be contained in 
$\{q\in\rr^2:\|q\|\le R\}$. So one of the components is unbounded
and the other one is bounded.  We use 
${\rm Ins}(\xi)$, ${\rm Out}(\xi)$ (the ``inside'' and the
``outside'' of $\xi$) to denote, respectively,
the bounded and the unbounded connected component
of $\rr^2\backslash|\xi|$. We refer to 
${\rm Ins}(\xi)$ as {\em the region enclosed by the 
simple loop $\xi$}.  We use $A(\xi)$ to denote the area 
of ${\rm Ins}(\xi)$.  

\begin{theorem}\label{theowinnum}
Let $\xi:\mathbb{S}^1\mapsto\rr^2=\mathbb{C}$ be a simple 
loop. Then the funcion $\mathbb{C}\backslash|\xi|\ni p\mapsto
\nu(\xi,p)\in\zz$ is constant on ${\rm Out}(\xi)$ and  
on ${\rm Ins}(\xi)$, and has the value $0$
on ${\rm Out}(\xi)$ and the value $1$ or $-1$ on 
${\rm Ins}(\xi)$.
\end{theorem}
\medbreak\noindent PROOF.\ Proposition  \ref{iscontin}
tells us that the winding number $\nu(\xi,p)$ 
depends \mbox{continuously} on $p$ as long as $p$ varies on
$\mathbb{C}\backslash|\xi|$. Since this number is always an integer,
the function $\mathbb{C}\backslash|\xi|\ni p\mapsto
\nu(\xi,p)\in\zz$ must be constant on ${\rm Out}(\xi)$ and  
on ${\rm Ins}(\xi)$.

Fix a positive $R\in\rr$ such that $|\xi(e^{2\pi it})|\le R$ for all 
$t\in R$, and a $p_*\in\mathbb{C}$ such that $|p|>R$. Then 
$\Big|\frac{\xi(e^{2\pi it})}{p_*}\Big|<1$ for every
$t\in\rr$, so we can define a continuous function 
$\sigma:\rr\mapsto\mathbb{C}$ by letting 
$$
\sigma(t)={\rm Ln}\Big(1-\frac{\xi(e^{2\pi it})}{p_*}\Big)\,\,\,\,{\rm for}
\,\,\,\,t\in\rr\,.
$$
Let $f_0\in\mathbb{C}$ be such that $e^{f_0}=-p_*$. Define
$f(t)=f_0+\sigma(t)$. Then $f$ is continuous and
$$
e^{f(t)}=e^{\bar f}.e^{\sigma(t)}=(-p_*).\Big(1-\frac{\xi(e^{2\pi it})}
{p_*}\Big)=\xi(e^{2\pi it})-p_*\,.
$$
Therefore
$\nu(\xi,p_*)=\frac{1}{2\pi i}(f(1)-f(0))=0$, since $f$ is clearly
$1$-periodic. So $\nu(\xi,p_*)=0$, and
then $\nu(\xi,p)=0$ for all $p\in{\rm Out}(\xi)$.

Now choose a point $\hat p\in{\rm Ins}(\xi)$.
Fix a point $\hat q$ of $|\xi|$ closest to $\hat p$. Let 
$\hat r=|\hat p-\hat q|$. Then 
the open disc $\hat D=\{z:|z-\hat p|<\hat r\}$ 
does not intersect $|\xi|$ and
is entirely contained in ${\rm Ins}(\xi)$.  After a translation 
and a rotation  of the 
coordinate axes of the plane, we may assume that $\hat p=0$
and $\hat q=\hat r$. Fix a $\hat t\in\rr$ such that,
$\xi(e^{2\pi i\hat t})=\hat r$. Fix a number $\hat\eps$
such that $0<\hat\eps<\frac{1}{2}$,  and two continuous functions 
\mbox{$\rho:[\hat t-\hat\eps,\hat t+\hat\eps]\mapsto]0,\infty[$}, 
$\varphi:[\hat t-\hat\eps,\hat t+\hat\eps]\mapsto\rr$, such that
\mbox{$\xi(e^{2\pi it})=\rho(t)e^{2\pi i\varphi(t)}$} 
whenever $t\in[\hat t-\hat\eps,\hat t+\hat\eps]$, and
$\varphi(\hat t)=0$. Then $\rho(t)\ge\hat r$ for all
$\in[\hat t-\hat\eps,\hat t+\hat\eps]$, and $\rho(\hat t)=\hat r$.

Then, if $0<\eps\le\hat\eps$, it cannot be the case that $\varphi(t)$ 
vanishes for all \mbox{$t\in[\hat t-\eps,\hat t+\eps]$}. 
(Indeed, suppose that $\varphi(t)=0$ whenever $t\in[\hat t-\eps,\hat t+\eps]$.
Then 
$\xi(e^{2\pi it})=\rho(t)\in\rr$ for $t\in[\hat t-\eps,\hat t+\eps]$.
Therefore 
the function \mbox{$[\hat t-\eps,\hat t+\eps]\ni t\mapsto\xi(e^{2\pi it})$} 
would be real-valued and would have a local 
minimum at $\hat t$, contradicting the fact that $\xi$ is one-to-one
on some neighborhood of $\hat t$.) It follows that there exists
a sequence $\{\tau_j\}_{j=1}^\infty$ in $\rr$ such that
$\tau_j\to\hat t$ as $j\to\infty$ and $\varphi(\tau_j)\neq 0$ for all $j$.
If we let $L_j$ be the closed half-line $\{re^{2\pi i\varphi(\tau_j)}:
r\ge 0\}$, then $L_j\cap|\xi|$ is a compact set.
Furthermore,  $L_j\cap|\xi|$ is nonempty, because
$\xi(\tau_j)=\rho(\tau_j)e^{2\pi i\varphi(\tau_j)}\in L_j\cap|\xi|$. 
So $L_j\cap|\xi|$ has a point $q_j$ closest to $0$. 
Write $q_j=\xi(e^{2\pi i t_j})$, 
$t_j\in[\hat t-\frac{1}{2},\hat t+\frac{1}{2}]$. 
Also, let $q_j=r_je^{2\pi i\varphi(t_j)}$,
so $\hat r\le r_j\le\rho(\tau_j)$. 

We claim that $\lim_{j\to\infty}t_j=\hat t$. Indeed, if this was not so
there would exist a subsequence $\{t_{j(\ell)}\}_{\ell=1}^\infty$
of $\{t_{j}\}_{j=1}^\infty$ that converges to a 
$\tilde t\in[\hat t-\frac{1}{2},\hat t+\frac{1}{2}]\backslash\{\hat t\}$.
Let $\tilde q=\xi(e^{2\pi i\tilde t})$. Then 
$\tilde q=\lim_{\ell\to\infty}q_{j(\ell)}$.
Since $e^{2\pi i\varphi(\tau_j)}\to 1$ as $j\to\infty$, 
and $e^{2\pi i\varphi(t_j)}=e^{2\pi i\varphi(\tau_j)}$
(because $q_j\in L_j$),  we conclude that $e^{2\pi i\varphi(t_j)}\to 1$ 
as $j\to\infty$, so $\tilde q$ is a positive real number
$\tilde r$, and $\tilde r=\lim_{\ell\to\infty}r_{j(\ell)}$.
Since $\hat r\le r_{j(\ell)}\le\rho(\tau_{j(\ell)})$, we have
$\tilde r=\hat r$, and then 
$\tilde q=\hat q$. Since $|\tilde t-\bar t|\le\frac{1}{2}$, the fact that
$\xi(e^{2\pi i\tilde t})=\tilde q=\hat q=\xi(e^{2\pi i\bar t})$ 
implies $\tilde t=\bar t$, 
which is a contradiction, proving our claim.

By passing to a subsequence we may assume that $|t_j-\hat t|<\hat\eps$
for all $j$, and either $t_j<\hat t$ for all $j$ or $t_j>\hat t$ for 
all $j$. Then $q_j=\rho(t_j)e^{2\pi i\varphi(t_j)}$---so 
$r_j=\rho(t_j)$ and $\varphi(t_j)=\varphi(\tau_j)$---and 
$re^{2\pi i\varphi(t_j)}\notin|\xi|$ whenever $0\le r<r_j$. 
By passing to a subsequence,  we assume that 
either $\varphi(t_j)<0$ for all $j$ or $\varphi(t_j)>0$
for all $j$.

Under our conditions,  one of the following possibilities
occurs:
\begin{itemize}
\item[I.] $t_j<\hat t$ and $\varphi(t_j)<0$ for all $j$,
\item[II.] $t_j<\hat t$ and $\varphi(t_j)>0$ for all $j$,
\item[III.] $t_j>\hat t$ and $\varphi(t_j)<0$ for all $j$,
\item[IV.] $t_j>\hat t$ and $\varphi(t_j)>0$ for all $j$.
\end{itemize}
We will discuss Case I, and prove that $\nu(\xi,0)=1$. (The other
three cases are similar. In Cases II and III one can 
prove that $\nu(\xi,0)=-1$, and in Case IV that 
$\nu(\xi,0)=1$.)  

Write $\varphi_j=-\varphi(t_j)$, so $\varphi_j>0$. 
Fix a positive real number $\delta$ such that 
\mbox{$\delta<\hat r={\rm dist}(0,|\xi|)$}. 
%For each $j$, let
%$S_j$ be the sector $\{re^{2\pi i\theta}:r\ge 0\wedge
%-\varphi_j\le \theta\le 0\}$. Let $\Delta$ be the closed
%disc $\{z\in\mathbb{C}:|z|\le\delta\}$. Let  $T_j=S_j\cup\Delta$.
Let $\eta_j:[0,3]\mapsto\mathbb{C}$ be the curve given by
$$
\eta_j(u)=\left\{\begin{array}{lcl}
\Big((1-u)r_j+u\delta\Big)e^{-2\pi i\varphi_j}&{\rm if}&
0\le u\le 1\,,\\
\delta e^{-2\pi i((2-u)\varphi_j+1-u)}&{\rm if}&1\le u\le 2\,,\\
(u-2)\hat r+(3-u)\delta&{\rm if}&2\le u\le 3\,.\end{array}\right.
$$
Then, as $u$ goes from $0$ to $3$, $\eta_j(u)$ starts at
the point $\xi(e^{2\pi i t_j})=r_je^{-2\pi i\varphi_j}$,
moves along the straight-line segment $M_j^1$ from 
$r_je^{-2\pi i\varphi_j}$ to $\delta e^{-2\pi i\varphi_j}$, 
then follows an arc $M_j^2$ of length 
$\delta.(2\pi-\varphi(t_j))$ of the circle $\{z:|z|=\delta\}$, clockwiset
from $\delta e^{-2\pi i\varphi_j}$ to $\delta$, 
and finally moves along the straight-line segment $M_3$ from 
$\delta$ to $\hat r$, ending at $\hat r=\xi(e^{2\pi i\hat t})$. 
It is clear that $\eta_j$ is a one-to-one map. Furthermore,
$\eta_j(u)\notin|\xi|$ whenever $0<u<3$. 

Define $\hat\zeta_j:[\hat t-\frac{1}{2},\hat t+\frac{1}{2}]
\mapsto\mathbb{C}$  by letting
$$
\hat\zeta_j(t)=\left\{\begin{array}{lcl}
\xi(e^{2\pi it})&{\rm if}&t\in
[\hat t-\frac{1}{2},\hat t+\frac{1}{2}]\backslash
[t_j,\hat t]\,,\\
\eta_j\Big(\frac{3(t-t_j)}{\hat t-t_j}\Big)
&{\rm if}&t\in[t_j,\hat t]\,.\end{array}\right.
$$
Then $\hat\zeta_j$ is continuous, and
$\hat\zeta_j(\hat t+\frac{1}{2})=\hat\zeta_j(\hat t+\frac{1}{2})$,
so we can extend $\hat\zeta_j$ to a $1$-periodic map $\zeta_j$ 
from $\rr$ to $\mathbb{C}$. Write $\zeta_j(t)=\omega_j(e^{2\pi it})$
for $t\in\rr$. Then $\omega_j$ is a simple loop. Furthermore,
$|\omega_j|\subseteq\{z:|z|\le R\}$, so $p_*\in{\rm Out}(\omega_j)$, and then
$\nu(\omega_j,p_*)=0$. 

Let $I_j=\{\xi(e^{2\pi it}):t_j<t<\hat t\}$. Let $K_j=|\xi|\backslash I_j$.
Let $U_j=\mathbb{C}\backslash K_j$.
Then $K_j$ is a compact subset of $\mathbb{C}$ which is homeomorphic
to the interval $[0,1]$. Therefore $K_j$ does not disconnect $U_j$. 
Since $p_*$ and $0$ obviously belong to $U_j$, there exists
a continuous path $\gamma_j:[0,1]\mapsto U_j$ such that 
$\gamma_j(0)=p_*$ and $\hat\gamma_j(1)=0$. This path 
cannot be contained in $\mathbb{C}\backslash|\xi|$, because
$p_*$ and $0$ belong to different connected components 
of $\mathbb{C}\backslash|\xi|$. Therefore the set
$G_j=\{s\in[0,1]:\hat\gamma_j(s)\in|\xi|\}$ is nonempty.
Since $G_j$ is obviously compact, it has a smallest 
member $s_j$. Then $\gamma_j(s_j)\in|\xi|$ and
$\gamma_j(s)\notin|\xi|$ whenever $0\le s<s_j$. Since
$\gamma_j$ is contained in $U_j$, $\gamma_j(s_j)$ cannot belong
to $K_j$, so $\gamma_j(s_j)\in I_j$. Therefore
$\gamma_j(s_j)=\xi(e^{2\pi iv_j})$ for some
$v_j\in I_j$. Pick an $\ell(j)$ such that
$t_j<t_{\ell(j)}<\hat t$. Define a curve
$\mu_j:[0,3]\mapsto\mathbb{C}$ by letting
$$
\mu_j(u)=\left\{\begin{array}{lcl}
\gamma_j(s_ju) &{\rm if}&0\le u\in\le 1\,,\\
\xi(e^{2\pi i((2-u)v_j+(u-1)t_{\ell(j)})})
&{\rm if}&1\le u\le 2\,,\\
(3-u)r_je^{2\pi i\varphi(t_{\ell(j)})}
&{\rm if}&2\le u\le 3\,.\end{array}\right.
$$
Then $\mu_j$ is continuous, $\mu_j(0)=p_*$, and
$\mu_j(3)=0$. Moreover, $\mu_j$ never goes through
$|\omega_j|$.  Therefore $\nu(\omega_j,0)=\nu(\omega_j,p_*)=0$.

We now relate $\nu(\omega_j,0)$ to $\nu(\xi,0)$.  Let $f:\rr\mapsto\mathbb{C}$
be a continuous map such that $e^{f(t)}=\xi(e^{2\pi it})$ for 
all $t\in\rr$. Define $g_j:[0,3]\mapsto\mathbb{C}$ by 
$$
g_j(u)=\left\{\begin{array}{lcl}
{\rm ln}\,\Big((1-u)r_j+u\delta\Big)-2\pi i\varphi_j&{\rm if}&
0\le u\le 1\,,\\
{\rm ln}\,\delta-2\pi i((2-u)\varphi_j+1-u)&{\rm if}&1\le u\le 2\,,\\
{\rm ln}\,\Big((u-2)\hat r+(3-u)\delta\Big)+2\pi i
&{\rm if}&2\le u\le 3\,.\end{array}\right.
$$
Then $g_j$ is continuous and $e^{g_j(u)}=\eta_j(u)$ for
$0\le u\le 3$. Define
$$
h_j(t)=\left\{\begin{array}{lcl}
f(t)&{\rm if}&\hat t-\frac{1}{2}\le t\le t_j,\,,\\
g_j\Big(\frac{3(t-t_j)}{\hat t-t_j}\Big)+f(t_j)-g_j(0)
&{\rm if}&t_j\le t\le\hat t\,,\\
f(t)+f(t_j)-g_j(0)+g_j(3)-f(\hat t)
&{\rm if}&\hat t\le t\le\hat t+\frac{1}{2}\,.
\end{array}\right.
$$
Then $h_j$ is continuous and $e^{h_j(t)}=\zeta_j(t)$ whenever
$t\in[\hat t-\frac{1}{2},\hat t+\frac{1}{2}]$. Therefore
$h_j(\hat t+\frac{1}{2})-h_j(\hat t-\frac{1}{2})=2\pi i\,\nu(\omega_j,0)=0$. 
Then 

\noindent
$\quad f\Big(\hat t\!+\!\frac{1}{2}\Big)\!+\!f(t_j)\!-\!g_j(0)\!+\!g_j(3)
\!-\!f(\hat t)=
h_j\Big(\hat t\!+\!\frac{1}{2}\Big)=h_j\Big(\hat t\!-\!\frac{1}{2}\Big)
=f\Big(\hat t\!-\!\frac{1}{2}\Big)\,,$

\noindent so that

\noindent 
$\qquad\qquad\quad f\Big(\hat t+\frac{1}{2}\Big)-f\Big(\hat t-\frac{1}{2}\Big)=
f(t_j)-g_j(0)+g_j(3)-f(\hat t)\,.$

\noindent It follows from the definition of $g_j$ that 
$$
g_j(0)={\rm ln}\,r_j-2\pi i\varphi_j\qquad{\rm and}\qquad
g_j(3)={\rm ln}\,\hat r+2\pi i\,.
$$

\noindent Since $\lim_{j\to\infty}r_j=\hat r$ (because
$\hat r\le r_j\le\rho(\tau_j)$, $\tau_j\to\hat t$, and
$\rho(\hat t)=\hat r$), and 
$\lim_{j\to\infty}\varphi_j=\lim_{j\to\infty}(-\varphi(\tau_j))=
-\varphi(\hat t)=0$, we can conclude that
$\lim_{j\to\infty}(g_j(3)-g_j(0))=2\pi i$. Therefore

\noindent 
$\quad\,\,f\Big(\hat t+\frac{1}{2}\Big)-f\Big(\hat t-\frac{1}{2}\Big)=
\lim_{j\to\infty}\Big(f(t_j)-g_j(0)+g_j(3)-f(\hat t)\Big)=2\pi i\,,$

\noindent since $\lim_{j\to\infty}t_j=\hat t=2\pi i$. Hence
$$
\nu(\xi,0)=\frac{1}{2\pi i}
\Big(\,f\Big(\hat t+\frac{1}{2}\Big)-f\Big(\hat t-\frac{1}{2}\Big)\,\Big)=1\,.
$$
Therefore $\nu(\xi,z)=1$ for every $z\in{\rm Ins}(\xi)$, and our proof
is complete. \proofend

\ssection{Green's formula}


If $\xi\in C^0(\mathbb{S}^1,\mathbb{C})$ is a loop 
in $\mathbb{C})$, the {\em winding number function}
of $\xi$ is the function
$$
\mathbb{C}\backslash|\xi|\ni z\mapsto 
\nu_\xi(z)\equaldef \nu(\xi,z)\in\zz\,.
$$
We want to prove that $\nu_\xi$ is a well defined member 
of $L^1(\rr^2,\rr)$ if $\xi$ is rectifiable.   For this purpose, 
we have to know first of all that the set $|\xi|$ has measure
zero, so the fact that $\nu_\xi$ is not well defined on
$|\xi$ does not matter, and $\nu_\xi$ is well defined as
a measurable function from $\rr^2$ to $\rr$ up to 
equality almost everywhere.

\begin{proposition}\label{measiszero}
Let $\xi\in C^0(\mathbb{S}^1,\mathbb{C})$ be a rectifiable loop.
Then the \mbox{2-dimensional} Lebesgue measure of $|\xi|$ is equal to zero.
\end{proposition}
\medbreak\noindent PROOF.\ In view of Proposition vaca,
we may assume that $\xi$ is Lipschitz. Let $C$ be a Lipschitz constant
for the map $\rr\ni t\mapsto\eta(t)=\xi(e^{2\pi it})$. For each $n,k\in\nn$
such that $1\le k\le n$, let $D_{k,n}$ be the disc
with center $\eta\Big(\frac{2k-1}{2n}\Big)$ and radius 
$\frac{C}{2n}$.  Then $|\xi|\subseteq\bigcup_{k=1}^nD_{k,n}$,
so 
$$
{\rm meas}(|\xi|)\subseteq\sum_{k=1}^n{\rm meas}(D_{k,n})
=n\pi\frac{C^2}{4n^2}=\frac{\pi C^2}{4n}\,.
$$
Since $n$ is arbitrary, ${\rm meas}(|\xi|)=0$, as stated.\proofend

\begin{proposition}\label{fewdays}
Let $C>0$, and let $\xi\in C^0(\mathbb{S}^1,\mathbb{C})$ be a Lipschitz loop
with Lipschitz constant $C$. Then 
\begin{itemize}
\item[1.] The winding number function $\nu_\xi$ is Lebesgue 
integrable.
\item[2.] There exists a sequence $\{\xi_j\}_{j=1}^\infty$ 
of polygonal loops in $\mathbb{C}$ with Lipschitz \mbox{constant} $C$ 
such that, as $j\to\infty$, 
\begin{itemize}
\item[2.a.] $\xi_j\to\xi$ uniformly, 
\item[2.b.] the functions $\nu_{\xi_j}$ converge to 
$\nu_\xi$ in $L^1(\rr^2,\rr)$.
\end{itemize}\end{itemize}
\end{proposition}

\medbreak\noindent PROOF.\ Let $\eta(t)=\xi(e^{2\pi i t})$, so 
$\eta:\rr\mapsto\mathbb{C}$ is a $1$-periodic function.
For $j=1,2,\ldots$, we define $\eta_j:\rr\mapsto\mathbb{C}$ 
by letting
$$
\eta_j(t)= (k-2^{j}t)\xi\Big(\frac{k-1}{2^{j}}\Big)+
(2^{j}t+1-k)\xi\Big(\frac{k}{2^{j}}\Big)\quad{\rm if}\quad k\in\zz\,,\,\,
\frac{k-1}{2^{j}}\le t\le\frac{k}{2^{j}}\,.
$$
Then each $\eta_j$ is $1$-periodic, and linear on each interval
$I_{j,k}\equaldef\Big[\frac{k-1}{2^{j}},\frac{k}{2^{j}}\Big]$.
If we define $\xi_j:\mathbb{S}^1\mapsto\mathbb{C}$ by letting
$\xi_j(e^{2\pi it})=\eta_j(t)$, then $\xi_j$ is a polygonal
loop with $2^j$ sides 
which is Lipschitz with constant $C$. Furthermore, the $\xi_j$
converge uniformly to $\xi$ as $j\to\infty$.

For $k=0,1,\ldots,2^j$, let $A_k^j=\xi\Big(\frac{k}{2^{j}}\Big)$, so 
$A_j^0,\ldots,A_j^{2^j-1}$ are the vertices of $\xi_j$, 
$A_j^{2^j}=A_j^0$, and the segments 
$\sigma_j^k=\overline{A_j^{k-1}A_j^{k}}$, for 
$k=1,\ldots,2^j$,  are the sides of $\xi_j$.
Then $\xi_{j+1}$ is obtained from $\xi_j$ by replacing 
each side $\sigma_j^k$ by the concatenation of the
two segments $\sigma_{j+1}^{2k-1}$  and $\sigma_{j+1}^{2k}$.
That means that the polygonal $\xi_{j+1}$ 
is obtained from $\xi_j$ by adjoining to it the 
$2^j$ triangles $T_j^k=A_j^{k-1}A_{j+1}^{2k-1}A_j^k$, for 
$k=1,\ldots,2^j$.  Therefore
$$
\nu_{\xi_{j+1}}=\nu_{\xi_j}+\sum_{k=1}^{2^j}\nu_{T_j^k}\,.
$$
For given $j,k$, The function $\nu_{T_j^k}$ has the value $1$
or $-1$ inside $T_j^k$, and $0$ outside. Therefore,
$\nu_{T_j^k}$ is integrable, and its $L^1$ norm 
is equal to the area inside $T_j^k$, which is bounded by 
$C^22^{-2j-2}$. It follows that
$$
\Big\|\nu_{\xi_{j+1}}-\nu_{\xi_j}\Big\|_{L^1}\le C^22^{-j-2}\,.
$$
This shows that the sequence $\{\nu_{\xi_j}\}_{j=1}^\infty$ 
converges in $L^1(\rr^2,\rr)$ to a limit $\hat\nu$.  

On the other hand, if $z\notin|\xi|$, then $z\notin|\xi_j|$
if $j$ is large enough, and $\lim_{j\to\infty}\nu_{\xi_j}(z)=\nu_\xi(z)$.
In view of Proposition \ref{measiszero}, $\nu_{\xi_j}(z)\to\nu_\xi(z)$
for almost all $z\in\mathbb{C}$. Therefore $\nu_\xi=\hat\nu$,
and our conclusion is proved.\proofend

\begin{remark}
{\rm The results of Propositions \ref{measiszero} and \ref{fewdays}
are also true, with almost identical proof, if the curve 
$\xi$ is H\"older continuous
with exponent $\alpha$, as long as $\alpha>\frac{1}{2}$. Indeed,
to prove Proposition \ref{measiszero} in this case it suffices 
to let $D_{k,n}$ be 
the disc with center $\eta\Big(\frac{2k-1}{2n}\Big)$ and radius 
$\Big(\frac{C}{2n}\Big)^\alpha$.  To prove 
Proposition \ref{fewdays}, we use the same $\xi_j\,$s, and observe 
that the area inside $T_j^k$ is bounded by 
$C^22^{-2\alpha j-2}$. Then 
$$
\Big\|\nu_{\xi_{j+1}}-\nu_{\xi_j}\Big\|_{L^1}\le C^22^{-(2\alpha-1)j-2}\,,
$$
from which it still follows that $\{\nu_{\xi_j}\}_{j=1}^\infty$ 
converges in $L^1(\rr^2,\rr)$.}\proofend
\end{remark}

\begin{theorem}\label{green}
(Green's formula) Let $\xi\in C^0(\mathbb{S}^1,\mathbb{C})$ 
be a rectifiable loop. Then 
\begin{equation}\label{flag}
A(\xi)=\int_{-\infty}^{\infty}\int_{-\infty}^{\infty}
\nu_\xi(x+iy)\,dx\,dy\,.
\end{equation}
\end{theorem}

\medbreak\noindent PROOF.\ Both sides of (\ref{flag}) are invariant 
under time reparametrization, so we may replace $\xi$ by 
its arc-length reparametrization, and assume that $\xi$ is a
Lipschitz loop with Lipschitz constant $1$. 

By Proposition \ref{fewdays}, there exists a 
sequence $\{\xi_j\}_{j=1}^\infty$ of polygonal loops that are
Lipschitz with constant $1$, converge uniformly to
$\xi$, and are such that the functions $\nu_{\xi_j}$ converge to
$\nu_{\xi}$ in $L^1$. Then Proposition vaca implies that
$A(\xi_j)\to A(\xi)$. Therefore the validity of (\ref{flag})
for all rectifiable loops $\xi$ follows if (\ref{flag}) is true
for all polygonal loops. 

To prove (\ref{flag}) for polygonal loops, it suffices to observe that
each polygonal loop can be obtained from a triangle by successive
adjunctions of triangles, and both sides of (\ref{flag}) are additive
with respect to adjunction.  Hence it suffices to prove 
(\ref{flag}) for triangles. Finally, the proof of (\ref{flag}) for triangles
is an elementary direct calculation. \proofend

\begin{corollary}\label{greencor}
Let $\xi\in C^0(\mathbb{S}^1,\mathbb{C})$ be a simple rectifiable loop. 
Let $\omega=1$ if $\xi$ is oriented cuonterclockwise,
and $\omega=-1$ if $\xi$ is oriented clockwise. Then the area 
of the region ${\rm Ins}(\xi)$ is equal to $\omega A(\xi)$.
\end{corollary}


\cchapter[$\quad$The value function]{The value function}
\ssection{Introduction}
\label{sec-intro}

\noindent The purpose of this note is to present some new \mbox{results},
together with a number of particularly \mbox{simple} and user-friendly 
versions of results obtained in \mbox{recent} years by the \mbox{author} 
and M. Malisoff, on the \mbox{uniqueness} of \mbox{solutions} of the 
Hamilton-Jacobi-Bellman \mbox{equation} (HJBE) 
for \mbox{deterministic} finite-dimensional optimal control
\mbox{problems} under \mbox{non-standard hypotheses}. Our
\mbox{approach} is completely control-theoretic and totally
self-contained, using the systematic \mbox{construction} of 
\mbox{special} trajectories of \mbox{various} kinds, and not
\mbox{involving} any PDE methods.  We will %definitely 
{\em not} assume that the \mbox{Lagrangian} is positive, or 
that the \mbox{dynamics} is Lipschitz-continuous.

We will consider autonomous {\em Lagrangian \mbox{optimization} problems}
involving a {\em state variable} $x$ which takes values in an open subset 
$\Omega$ of $\rr^n$,  a {\em control variable} $u$ taking values in a 
{\em control space} $U$, and a {\em target set} ${\cal T}$, which is a 
closed subset of the closure of $\Omega$
disjoint from $\Omega$ itself.  The dynamics is given by
an ordinary differential equation
\begin{equation}\label{dyna}
\dot x=f(x,u)\,,
\end{equation}
the cost functional to be minimized is
\begin{equation}\label{lagra}
J=\int_{\tau_-(\xi)}^{\tau_+(\xi)}L(\xi(t),\eta(t))\,dt\,,
\end{equation}
(where $\tau_-(\xi)$, $\tau_+(\xi)$ are, respectively, 
the initial and \mbox{terminal} times of the trajectory $\xi$),
and the \mbox{minimization} is supposed to be, for each \mbox{initial}
state \mbox{$x\in\Omega$}, over the set $\axts$ of all pairs 
\mbox{$\Xi=(\xi,\xi_0)$} such that
\begin{itemize}
\item[(i)]  \mbox{$\Xi$} \mbox{consists}  of a trajectory
$\xi$ of (\ref{dyna}) (i.e., a \mbox{locally} \mbox{absolutely} 
continuous function  $\xi$ that satisfies $\dot\xi(t)=f(\xi(t),\eta(t))$
for almost all $t$) \mbox{corresponding} to some $U$-valued control 
$\eta$, and a ``\mbox{running} cost'' function $\xi_0$ corresponding to 
$\xi$ and $\eta$ (i.e., a \mbox{locally} absolutely continuous 
function  $\xi_0$ such that \mbox{$\dot\xi_0(t)=L(\xi(t),\eta(t))$}
for almost all $t$);
\item[(ii)]  \mbox{$\xi$} starts at $x$,  and ``ends at the target'' in a 
sense  to be defined precisely later.
\end{itemize}
We will refer to a pair $\Xi=(\xi,\xi_0)$ for which (i) above 
holds as an ``augmented trajectory'' of our \mbox{system}, 
\mbox{because} it really is a trajectory of the ``\mbox{augmented} 
\mbox{control} system''  

\begin{equation}\label{dynaaug}
\dot x=f(x,u)\,,\qquad\dot x_0=L(x,u)
\end{equation}

\medbreak\noindent obtained from (\ref{dyna}) by ``adding the cost as
an extra \mbox{variable}'' in a well known way.  We will write
$J(\Xi)$, rather than just $J$, for the left-hand side of
(\ref{lagra}), \mbox{because} it is easy to see that the natural
arguments for our cost functional $J$ are really augmented
\mbox{trajectories}, \mbox{since} (i)~one cannot just regard $J$ as a
functional of $\xi$ \mbox{only}, because the integral of (\ref{lagra})
involves the control $\eta$ as well as $\xi$, but on the other hand
(ii)~the cost is \mbox{completely} \mbox{determined} once we know
$\xi$ and a \mbox{running} cost $\xi_0$, \mbox{because} in that case
$L(\xi(t),\eta(t))=\dot\xi_0(t)$. 

The infimum $V(x)$ of the costs $J(\Xi)$ of all 
\mbox{augmented} trajectories
$\Xi\in\axts$ is the {\em value}  of our problem at $x$. (If the
set $\axts$ is itself empty, then  of course 
\mbox{$V(x)=+\infty$}.)  The function 
$V:\Omega\mapsto\rr\cup\{-\infty,+\infty\}$ is the {\em value
function} of our problem.

Our goal is to prove uniqueness theorems, \mbox{showing} that a
viscosity solution of the HJBE that satisfies an \mbox{appropriate}
boundary condition is necessarily the value function. 
``\mbox{Uniqueness}'' is to be understood as ``\mbox{uniqueness}
within a class defined by some additional properties,'' such as the
class of all functions that are continuous and bounded below.

We will work with a class of systems which is \mbox{sufficiently} general to
capture some \mbox{interesting} \mbox{phenomena} not commonly addressed in the
\mbox{literature}, and at the same time restricted \mbox{enough}
 to make it \mbox{possible} to prove strong \mbox{theorems}.  In 
particular,  we will asume that the sets 
\mbox{$\fsh(x,U)=\{(f(x,u),L(x,u)):u\in U\}$} are closed and convex,
but will not require them to be \mbox{compact}, and will instead
impose a ``local coerciveness'' \mbox{condition}, \mbox{according} to which, 
locally, an inequality of the form $\|f(x,u)\|^r\le L(x,u)+C$,
with $C>0$ and $r>1$,  holds uniformly with respect to $u$.  We will
also require $f(x,u)$ and $L(x,u)$ to be continuous 
with respect to $x$, with the continuity being uniform 
with respect to $u$ locally in $x$. 

On the other hand, we will most definitely {\em not} require 
that the dynamics $f(x,u)$ be Lipschitz-continuous with respect to $x$,
since one of the main purposes of this work is to clarify the exact
role of the Lipschitz-continuity assumptions often made in the viscosity
literature.  The answer we will propose is as follows: 
\begin{itemize}
\item[(a)] Without any Lipschitz-continuity hypotheses, one can 
prove, for continuous viscosity solutions $V$ of the HJBE, an 
{\em existence theorem for \mbox{trajectories}}, \mbox{asserting}
that, starting 
at every point of $x$ of $\Omega$, there is a maximally defined
``augmented \mbox{trajectory} of steepest descent,'' that is, 
a maximally defined pair $\Xi=(\xi,\xi_0)$ defined on a interval 
$I$ such that \mbox{$0=\min\,I$}, and having the property 
that the \mbox{inequality} 

\begin{equation}\label{steepdesc}
V(\xi(t))+\xi_0(t)\ge V(\xi(s))+\xi_0(s)
\end{equation}

\medbreak\noindent holds whenever $s,t\in I$ and $s\le t$. (This is Theorem 
\ref{maintech} below.)
\item[(b)] As a trivial corollary of the existence of \mbox{steepest} 
\mbox{descent} trajectories (applied to $-V$ and $-L$), we get the 
existence of ``DPI \mbox{trajectories}'' (where ``DPI'' is an acronym 
for ``\mbox{Dynamic} \mbox{Programming} \mbox{Inequality}''), i.e., 
\mbox{augmented}
\mbox{trajectories} \mbox{$\Xi=(\xi,\xi_0)$} along which 
the inequality 

\begin{equation}\label{steepasc}
V(\xi(t))+\xi_0(t)\le V(\xi(s))+\xi_0(s)
\end{equation}

\medbreak\noindent (that is, the exact opposite of (\ref{steepdesc}))
is satisfied.
\item[(c)] The existence result of (b) says that for every 
``\mbox{sufficiently} nice'' (e.g., piecewise constant) \mbox{control}
$\eta$ {\em there exists a trajectory of $\eta$ 
with the given \mbox{initial} condition along which the DPI holds}.
This is \mbox{almost}, but not quite, 
what is needed to prove that $V$ is bounded above by the 
value function. 
\item[(d)] The gap between the existence result for DPI 
\mbox{trajectories} and what would actually 
be needed to prove that $V$ is bounded above by the 
value \mbox{function} is that to achieve the latter goal one needs 
the DPI to hold for {\em all} trajectories, and it is not enough
to have just one DPI trajectory for every intial condition and every 
control. 
\item[(e)] The gap described in (d) clearly does not exist when 
there is uniqueness of trajectories for every given control and initial 
condition. 
\item[(f)] In particular, the gap does not exist when,
for \mbox{every} admissible control $\eta$,  the corresponding 
time-varying vector field $(x,t)\mapsto f(x,\eta(t))$ 
satisfies a Lipschitz-Carath\'eodory condition that guarantees
uniqueness of trajectories. 
\item[(g)] Naturally, the Lipschitz-Carath\'eodory condition 
can be replaced by weaker conditions that \mbox{guarantee}
uniqueness, such as the requirement that a bound
\begin{equation}\label{weakerbo}
\langle f(x,\eta(t))-f(y,\eta(t)),x-y\rangle 
\le k(t)\|x-y\|^2\,,
\end{equation}
\item[] with $k$ integrable, hold locally. 
\item[(h)] Even more generally, the only property that really matters
is that, if we pick a sequence $\{\eta_j\}_{j=1}$ of 
piecewise constant controls such that our \mbox{augmented}
trajectory $\Xi$ can be approximated by  \mbox{augmented}
trajectories $\Xi_j$ corresponding to the $\eta_j$---with, say, 
the same initial condition---then the $\Xi_j$ converge to $\Xi$ uniformly 
{\em no matter how the $\Xi_j$ are chosen}. We call such trajectories 
``uniquely \mbox{limiting},'' and use this concept in the statement of our main 
theorem. 
\end{itemize}

\medbreak The following important issues will not be discussed here:

\begin{itemize}
\item[(1)] Whether  the value function itself satisfies the 
\mbox{conditions} of our main theorem, i.e., whether it is a \mbox{continuous}
viscosity solution of the HJBE and whether it is bounded below.

\item[(2)] What happens when the sets $F(x,U)$ are not closed and convex.
(This would require \mbox{considering} \mbox{relaxed} controls, and using 
trajectories for which the steepest descent property holds
\mbox{approximately} rather than exactly. It turns out to be
\mbox{possible} to extend our results  under fairly \mbox{general}
\mbox{conditions}, provided our system has 
appropriate \mbox{local} \mbox{controllability} properties.) 

\end{itemize}


\begin{remark}
The approach followed here owes a great deal to the book \cite{subbotin} by
A.I. Subbotin. We point out, however, that Subbotin considers
viscosity \mbox{solutions} of PDEs of the form $F(x,u(x),Du(x))=0$, where
the Hamiltonian $F(x,u,p)$ is required to be globally 
\mbox{Lipschitz} with respect 
to the momentum variable $p$ (cf. Equation (2.2) in page 9 of 
\cite{subbotin}).  A somewhat weaker \mbox{hypothesis} is also considered 
later, in which the \mbox{Lipschitz} requirement is replaced by the condition
that for any $\Lambda>0$ there exists a positive constant
$\mu(\Lambda)$ such that the estimate
$$
|F(x,z,s)-F(x,z,p)|\le\mu(\Lambda)(1+\|s-p\|)
$$ 
holds for all $s\in\rr^n$ such that $\|s\|\le\Lambda$ and all $p\in\rr^n$ 
(cf. page 37 of \cite{subbotin}). In particular, even with the 
\mbox{weakened}
requirements, these hypotheses are not \mbox{sufficient} to cover, for
example, coercive problems of the kind discussed here, such as linear 
quadratic \mbox{optimal} \mbox{control}. (For example, for the optimal 
\mbox{control} \mbox{problem} of minimizing the integral
$\frac{1}{2}\int(x^2+u^2)$, with a scalar state $x$ and a scalar control
$u$, and dynamics $\dot x=u$, the \mbox{function} $F$ is given by 
$F(x,z,p)=\frac{1}{2}(p^2-x^2)$. Therefore 
$F(x,z,s)-F(x,z,p)=\frac{1}{2}(s^2-p^2)=\frac{1}{2}(s+p)(s-p)$, 
and for the desired estimate to be satisfied  the sum  $s+p$ would have 
to be bounded by a constant $\mu(\Lambda)$ for all
$s\in\rr^n$ such that $\|s\|\le\Lambda$ and all $p\in\rr^n$, and such a 
bound obviously does not hold. \proofend
\end{remark}


\ssection{The main theorem}\label{sec-standard}
\subsection{Systems and augmented systems.} 
\noindent If $n$ is a positive  integer, an {\em $n$-dimensional
control \mbox{system}} is a triple $\Sigma=(\Omega,U,f)$ such that
$\Omega$ (the {\em \mbox{state} space} of $\Sigma$) is an open subset 
of $\R^n$,  $U$ (the {\em control \mbox{space}} of $\Sigma$) 
is a nonempty set, and $f$ (the {\em dynamics} of $\Sigma$) 
is a map \mbox{$\Omega\times U\ni(x,u)\mapsto f(x,u)\in\R^n$}. 

An {\em $n$-dimensional augmented 
control system} is a \mbox{4-tuple} $\hat\Sigma=(\Omega,U,f,L)$ such that
$\Sigma=(\Omega,U,f,L)$ is an $n$-dimensional control system 
and $L$ (the {\em Lagrangian} of $\hat\Sigma$) is a map 
$\Omega\times U\ni(x,u)\mapsto L(x,u)\in\rr$. 
(In that case, the state space, control space, and dynamics 
of $\Sigma$ are also called the state space, control space, and dynamics 
of $\hat\Sigma$.)

An augmented control system $\hat\Sigma=(\Omega,U,f,L)$ is 
{\em \mbox{continuous}} if the maps 
$\Omega\ni x\mapsto f(x,u)\in\R^n$ and 
$\Omega\ni x\mapsto L(x,u)\in\R$ 
are continuous for each fixed $u$. We call $\hat\Sigma$
{\em \mbox{uniformly} continuous} on a subset $S$ of $\Omega$
if there exists a function $\omega:\,]0,\infty[\mapsto[0,\infty]$ 
such that $\lim_{s\downarrow 0}\omega(s)=0$, having the property that

$$
\|f(x,u)-f(y,u)\|+|L(x,u)-L(y,u)|\le\omega(\|x-y\|)
$$

\medbreak\noindent whenever $x,y\in S$ and $u\in U$. We call 
$\hat\Sigma$ {\em \mbox{locally} \mbox{uniformly} continuous} if it is 
uniformly \mbox{continuous} on every compact subset of $\Omega$, and
{\em globally  \mbox{uniformly} \mbox{continuous}} if it is 
uniformly continuous on $\Omega$. 
We say that $\hat\Sigma$ is 
{\em Lipschitz continuous} if the maps 
\mbox{$\Omega\ni x\mapsto f(x,u)\in\R^n$} and 
$\Omega\ni x\mapsto L(x,u)\in\R$ 
are Lipschitz continuous for each fixed $u$. We call $\hat\Sigma$
{\em \mbox{uniformly} Lipschitz continuous} on a subset $S$ of $\Omega$
if there exists a positive constant $C$ such that

$$
\|f(x,u)-f(y,u)\|+|L(x,u)-L(y,u)|\le C\|x-y\|
$$

\medbreak\noindent whenever $x,y\in S$ and $u\in U$. We call $\hat\Sigma$
{\em \mbox{locally} \mbox{uniformly} Lipschitz continuous} if it is 
uniformly \mbox{Lipschitz} continuous on every compact subset of $\Omega$, and
{\em globally  uniformly Lipschitz continuous} if it is 
uniformly Lipschitz continuous on $\Omega$. 

\begin{remark}
Naturally, the concepts of continuity,   \mbox{uniform} continuity,
Lipschitz continuity,  and uniform Lipschitz continuity,  
also make sense for a control \mbox{system} $\Sigma=(\Omega,U,f)$,
by taking the same definitions given above and omitting 
the parts that refer to $L$. In order to avoid having to make 
a similar remark for other concepts to be introduced
in the future, we adopt the convention that any concept $X$
that we \mbox{define} for an augmented control system 
$\hat\Sigma=(\Omega,U,f,L)$ is \mbox{automatically}
\mbox{understood} to apply to a control system $\Sigma=(\Omega,U,f)$, 
in the sense that ``$X$ of $\Sigma$'' means 
``$X$ of the augmented system $\hat\Sigma=(\Omega,U,f,0)$.''\proofend
\end{remark}

The augmented control system $\hat\Sigma=(\Omega,U,f,L)$ is {\em
\mbox{coercive}} on a subset $S$ of $\Omega$ if there exist real constants
$r$, $A$, $C$, such that $A>0$, $C>0$, $r>1$, and 
\begin{equation}\label{iscoer}
\|f(x,u)\|^{r}\le AL(x,u)+C\,\,{\rm for\,\,all}\,\,
x\in S\,,\,u\in U\,.
\end{equation}
We call $\hat\Sigma$ {\em locally coercive} if it is coercive on every 
\mbox{compact} subset of $\Omega$, and {\em globally coercive} if 
it is \mbox{coercive} on $\Omega$.

\begin{remark}
If $\hat\Sigma$ is coercive on a set $S$, then it is 
\mbox{always} possible
to choose $C$, $r$, such that $C>0$, $r>1$, and 
(\ref{iscoer}) holds with $A=1$. Indeed, let $A$, $C$, $r$ be such that
$A>0$, $C>0$, $r>1$, and (\ref{iscoer}) holds. Pick $\rho$ such that
$1<\rho<r$,  and let$s=\frac{\rho}{r-\rho}$. 
Let $x\in\Omega$, $u\in U$. Then, 
if $\|f(x,u)\|^{r-\rho}>A$, we have 
\begin{eqnarray*}
\|f(x,u)\|^{\rho}&=&\frac{\|f(x,u)\|^{r}}{\|f(x,u)\|^{r-\rho}}\\
&\le&A^{-1}(AL(x,u)+C)\\
&=&L(x,u)+\frac{C}{A}\\
&\le&L(x,u)+\frac{C}{A}+A^s\,,
\end{eqnarray*}
while on the other hand, if $\|f(x,u)\|^{r-\rho}\le A$, we find that
\begin{eqnarray*}
\|f(x,u)\|^{\rho}&=&(\|f(x,u)\|^{r-\rho})^{s}\\
&\le& A^s\\
&\le&A^{-1}\|f(x,u)\|^{r}+A^s\\
&\le&A^{-1}(AL(x,u)+C)+A^s\\
&=&L(x,u)+\frac{C}{A}+A^s.
\end{eqnarray*}
Hence (\ref{iscoer}) holds if $r$, $C$, $A$ are replaced 
by $\rho$, $\frac{C}{A}+A^s$, and $1$.\proofend
\end{remark}

For an augmented control system $\hat\Sigma=(\Omega,U,f,L)$, 
we define a map $\fsh:\Om\times U\mapsto\R^{n+1}$
(called the {\em augmented dynamics} of $\hat\Sigma$) by 
letting $\fsh(x,u)=(f(x,u),L(x,u))$ for $x\in\Omega$, $u\in U$. 
We say that $\hat\Sigma$ {\em satisfies the convexity and 
upper semicontinuity condition} if, for each $x\in\Omega$, 
\begin{equation}\label{convus}
\fsh(x,U)=\bigcap_{\eps>0}\overline{{\rm co}}\Big(\bigcup\{\fsh(x',U):
\|x'-x\|\le\eps\}\Big)\,,
\end{equation}
where ``$\overline{{\rm co}}$'' stands for ``closed convex hull,''
and $\fsh(x,U)\equaldef\{(f(x,u),L(x,u)):u\in U\}$.

\begin{remark}
If $\hat\Sigma=(\Omega,U,f,L)$ is locally uniformly 
continuous, then $\hat\Sigma$ satisfies the convexity and 
upper semicontinuity condition if and only if 
the set  $\fsh(x,U)$ is closed and convex for every $x\in\Omega$.
Indeed, the ``only if'' assertion is trivial, since the right-hand side
of (\ref{convus}) is obviously closed and convex. To prove the 
``if'' part, we fix $x\in\Omega$ and assume that 
$\fsh(x,U)$ is closed and convex. We choose a $\delta$
such that $\delta>0$ and $B_\delta(x)\equaldef
\{x'\in\R^n:\|x'-x\|\le\delta\}\subseteq\Omega$ and a function
$\omega:]0,\infty[\mapsto[0,\infty]$ such that 
$\lim_{s\downarrow 0}\omega(s)=0$ and 
$\|\fsh(y,u)-\fsh(z,u)\|\le\omega(\|y-z\|)$ whenever 
$u\in U$ and $y,z\in B_\delta(x)$. We then pick 
$$
v\in\bigcap_{\eps>0}\overline{{\rm co}}\Big(\bigcup\{\fsh(x',U):
\|x'-x\|\le\eps\}\Big)
$$
and prove that $v\in\fsh(x,U)$. We let $\eps_k=2^{-k}$, 
and use the fact that 
\mbox{$v\in\overline{{\rm co}}\Big(\bigcup\{
\fsh(',U):\|x'-x\|\le\eps_k\}\Big)$}
to find, for each sufficiently large $k$, a member 
$v_k$ of the convex hull of 
\mbox{$\bigcup\{\fsh(x',U):\|x'-x\|\le\eps_k\}$}
such that \mbox{$\|v-v_k\|<\eps_k$}. We then write
\mbox{$v_k=\sum_{j=0}^{n+1}\alpha_{k,j}\fsh(x_{k,j},u_{k,j})$}, 
where the $x_{k,j}$ \mbox{belong} to $\Omega$ and satisfy 
$\|x_{k,j}-x\|\le\eps_k$, the $u_{k,j}$ belong to $U$,
and the $\alpha_{k,j}$ are nonnegative and satisfy 
$\sum_{j=0}^{n+1}\alpha_{k,j}=1$.  Then 
$\|\fsh(x,u_{k,j})-\fsh(x,u_{k,j})\|\le\omega(\eps_k)$. 
\mbox{Therefore},
if we let $w_k=\sum_{j=0}^{n+1}\alpha_{k,j}\fsh(x,u_{k,j})$, 
we have $\|w_k-v_k\|\le\omega(\eps_k)$. Hence 
$w_k\to v$ as $k\to\infty$. Since $\fsh(x,U)$ is convex, the 
$w_k$ belong to $\fsh(x,U)$. Since $\fsh(x,U)$ is closed, 
$v\in\fsh(x,U)$, and the proof is complete.\proofend
\end{remark}

A {\em target} for an augmented control system 
\mbox{$\hat\Sigma=(\Omega,U,f,L)$}
is a closed subset ${\cal T}$ of $\R^n$ 
such that \mbox{${\cal T}\subseteq{\rm Closure}\,\Omega$} and 
${\cal T}\cap\Omega=\emptyset$.

\subsection{Trajectories}

A {\em trajectory} of $\hat\Sigma=(\Omega,U,f,L)$ 
is a locally absolutely continuous curve 
\begin{equation}\label{trajo}
I\ni t\mapsto\xi(t)\in\Omega\,,
\end{equation}
defined on a nonempty subinterval $I$ of $\R$, 
having the property that 
$\dot\xi(t)\in f(\xi(t),U)$ for almost every \mbox{$t\in I$}. 
An {\em augmented trajectory} of $\hat\Sigma$ is a locally 
absolutely \mbox{continuous} curve 
\begin{equation}\label{augmtrajo}
I\ni t\mapsto\Xi(t)=(\xi(t),\xi_0(t))\in\Omega\times\R\,,
\end{equation}
defined on a subinterval $I$ of $\R$,
having the property that $\dot\Xi(t)\in\fsh(\xi(t),U)$ for
almost every $t\in I$. 

The {\em initial time}, or {\em starting time} of a trajectory $\xi$ 
(\mbox{resp.} an augmented trajectory $\Xi=(\xi,\xi_0)$)  with domain $I$ 
is the time $\tau_-(\xi)\equaldef\min\,I$ (resp.
$\tau_-(\Xi)\equaldef\min\,I$)), if the \mbox{minimum} exists,
i.e., if $I$ is bounded below and \mbox{its} infimum belongs to $I$.  If
the initial time of $\xi$ (\mbox{resp.} $\Xi$) exists, 
then (i)~the point $x_-(\xi)\equaldef\xi(\tau_-(\xi))$
(\mbox{resp.} $x_-(\Xi)\equaldef\xi(\tau_-(\Xi))$)
is the {\em starting point}, or {\em \mbox{initial}  point}, of
$\xi$ (resp. $\Xi$), and (ii)~the ordered pair 
$\partial_-(\xi)\equaldef(\tau_-(\xi),x_-(\xi))$ 
(resp. $\partial_-(\Xi)\equaldef(\tau_-(\Xi),x_-(\Xi))$) is the
{\em initial condition} of $\xi$ (resp.  $\Xi$). 
If $\partial_-(\xi)=(t,x)$ (resp. 
$\partial_-(\Xi)=(t,x)$), we say that $\xi$ (resp. $\Xi$) 
{\em starts at $x$ at time $t$}. 

If ${\cal T}$ is a target for $\hat\Sigma=(\Omega,U,f,L)$, then 
a trajectory $\xi$ or augmented trajectory $\Xi=(\xi,\xi_0)$
with domain $I$ {\em ends at  ${\cal T}$} if the limit
$$
\xi(\uparrow)\equaldef\lim_{t\uparrow\sup\,I}\xi(t)
$$
exists and belongs to ${\cal T}$. 

For each $x\in\Omega$, we let $\axts$ be the set of \mbox{all}
\mbox{augmented} trajectories $\,\Xi=(\xi,\xi_0)$ of $\hat\Sigma$ such that 

\medbreak
(i)~$\partial_-(\Xi)=(0,x)$, 

\medbreak
(ii)~$\xi_0(0)=0$,

\medbreak
(iii)~$\Xi$ ends at the target,

\medbreak
(iv)~the limit 
$\xi_0(\uparrow)\equaldef\lim_{t\uparrow\sup\,{\rm domain}\,\Xi}\xi_0(t)$
exists. 

\medbreak\noindent If $\Xi=(\xi,\xi_0)\in\axts$ then the {\em cost}
of $\Xi$ is the number
\begin{equation}\label{thecost}
J(\Xi)\equaldef \xi_0(\uparrow)\,.
\end{equation}

\medbreak The {\em value function} of the optimal control \mbox{problem}
\mbox{defined}
by $\hat\Sigma$ and the target ${\cal T}$ 
is the function 
\mbox{$\vts:\Omega\cup{\cal T}\mapsto\R\cup\{-\infty,+\infty\}$}
given by
$$
\vts(x)=\left\{\begin{array}{lll}
\inf\{J(\Xi):\Xi\in\axts\}&{\rm if}&x\in\Omega\\
0&{\rm if}&x\in{\cal T}\,.\end{array}\right.
$$

\def\sdsv{{\cal SD}_{\hat\Sigma,V}}
\def\sdsvx{{\cal SD}_{\hat\Sigma,V,x}}
\def\msdsv{{\cal MSD}_{\hat\Sigma,V}}
\def\msdsvx{{\cal MSD}_{\hat\Sigma,V,x}}

If $V:\Omega\mapsto\R$ is a function, then an {\em augmented 
\mbox{trajectory} of $\hat\Sigma$ of steepest descent with respect to $V$} 
is an \mbox{augmented} trajectory $\Xi=(\xi,\xi_0)$ of $\hat\Sigma$ 
%that has a \mbox{starting} time equal to zero, and is 
such that 

\begin{eqnarray}
\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\xi_0(s)+V(\xi(s))&\ge&\xi_0(t)+V(\xi(t))\nonumber\\
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\!\!\!\!\!\!\!\!
{\rm whenever}\,\,s,t\in{\rm domain}\,\Xi\,\,{\rm and}\,\,s\le t\,.
\label{steep}
\end{eqnarray}

\medbreak\noindent 
We use $\sdsv$ to denote the set of all  augmented 
\mbox{trajectories} of $\hat\Sigma$ of steepest descent with respect to $V$,
and $\sdsvx$ to denote the set of all  $\Xi\in\sdsv$ 
such that $\partial_-(\Xi)=(0,x)$. 

\medbreak If $x\in\Omega$, a {\em maximal augmented trajectory of
$\hat\Sigma$ from $x$ of steepest \mbox{descent} with respect to $V$}
is a $\Xi\in\sdsvx$ that \mbox{cannot} be \mbox{extended} to a
$\tilde\Xi\in\sdsvx$ defined on an interval which is strictly larger
that the domain of $\Xi$.  We use $\msdsvx$ to denote the set of all
maximal augmented \mbox{trajectories} of $\hat\Sigma$ from $x$ of steepest
descent with respect to $V$.

\begin{remark}
We have set up our definitions in such a way that the notion of 
``maximal augmented \mbox{trajectory} of steepest descent''
always means ``maximal within the class $\sdsvx$ 
of augmented \mbox{trajectories} $\Xi$ of steepest descent with
initial condition $\partial_-(\Xi)=(0,x)$, for a fixed $x\in\Omega$.''
(In other words, there is no such thing as a 
``maximal augmented \mbox{trajectory};'' there are \mbox{only}
``\mbox{maximal} augmented \mbox{trajectories} {\em from a given point $x$}.'')
Since all the members of $\sdsvx$ start at time $0$ at the point $x$, 
the only way that a $\Xi\in\sdsvx$, \mbox{defined}
on an interval $I$ (that necessarily starts at time $0$), could fail 
to \mbox{belong} to $\msdsvx$,
would be for $\Xi$ to be ``extendable to the right,'' that is, 
such that there exists an extension
$\tilde\Xi\in\sdsvx$ which is defined on an 
interval $\tilde I$ that also 
starts at $0$ and is strictly larger than $I$. Naturally, it may also happen
that $\Xi$ can be extended on the left, to an augmented 
trajectory $\hat\Xi$ of steepest descent defined on an interval
$\hat I=\,]$-$\eps,0]\cup I$ for some positive $\eps$. 
But such an extension will of course no longer start at time $0$, 
so its existence does not affect the possibility that $\Xi$ might 
belong to $\msdsvx$. \proofend
\end{remark}

\begin{definition}\label{rightunb}.
If $\Omega$ is an open subset of $\R^n$, and 
\mbox{$\xi\!:\!I\!\mapsto\!\Omega$} is a curve, we say that $\xi$ is 
\mbox{{\em right-unbounded} if}

\begin{itemize}
\vspace{-2mm}
\item[(i)] the interval $I$ is open on the right (that is, 
if \mbox{$\tau=\sup\,I$}, then either (a)~$\tau=+\infty$ or
(b)~$\tau$ is finite and does not belong to $I$),
\end{itemize}

\vspace{-2.8mm}
\noindent and 


\begin{itemize}

\vspace{-2.3mm}
\item[(ii)] if $\tau$ is finite, then for every compact subset $K$
of $\Omega$ there exists a $\tau_K$ such that $0\le\tau_K<\tau$ and
\mbox{$\xi(t)\notin K$} whenever $\tau_K<t<\tau$. 
\end{itemize}

\vspace{-2mm}\noindent (Equivalently, condition (ii) asserts 
that \mbox{$\lim_{t\uparrow\tau}\xi(t)=\infty_\Omega$}, where $\infty_\Omega$ is
the point at \mbox{infinity} of the one-point 
\mbox{compactification} of $\Omega$.)\proofend
\end{definition}

\medbreak The following observation is completely trivial
\mbox{given} our definitions, but we state it explicitly as a
\mbox{separate} \mbox{result} for future reference. We emphasize that
this \mbox{trivial} result is valid {\em under no technical hypotheses
\mbox{whatsoever} on $\hat\Sigma$ or $V$}.  The reader is warned that the 
\mbox{result} is {\em not} a true ``existence theorem'' for 
\mbox{trajectories} of \mbox{steepest} descent, even though at first sight 
it may \mbox{appear} to be, 
because the member $\Xi$ of $\msdsvx$ whose \mbox{existence}
it asserts could very well turn out to be the trivial trajectory
$\Xi_x^{triv}$, where $\Xi_x^{triv}$ is the map 
\mbox{$\Xi:\{0\}\mapsto\Omega\times\R$} such that $\Xi(0)=(x,0)$.  The true
\mbox{``existence} \mbox{theorem},'' yielding the existence of a 
\mbox{{\em nontrivial}} maximal trajectory of steepest descent and, 
in \mbox{fact}, \mbox{asserting} the stronger conclusion that 
{\em \mbox{every}} \mbox{maximal} \mbox{trajectory} of steepest descent 
is right-unbounded in the sense of Definition \ref{rightunb}.
This will be proved later (cf. Theorem
(\ref{maintech}) and, naturally, will \mbox{depend} on our technical hypotheses
on $\hat\Sigma$ and $V$. 

\begin{proposition}\label{trivialprop}
If $\hat\Sigma=(\Omega,U,f,L)$ is an \mbox{augmented} control system,
$V:\Omega\mapsto\R$ is a function, 
and $x\in\Omega$, then the set $\msdsvx$ of maximal \mbox{augmented}
\mbox{trajectories} of $\hat\Sigma$ from $x$ of steepest \mbox{descent}
with respect to $V$ is nonempty.
\end{proposition}

\begin{proof} 
Fix $x$. Let ${\cal Z}$ be the set of all pairs $(I,\Xi)$ such that
$I$ is a subinterval of $[0,\infty[$, $0\in I$, and 
\mbox{$\Xi=(\xi,\xi_0):I\mapsto\Omega\times\R$} is an augmented trajectory 
of $\hat\Sigma$ which is of steepest descent with respect
to $V$ and such that $\xi(0)=x$. We partially order  
${\cal Z}$ by stipulating that, if $(I_i,\Xi_i)\in{\cal Z}$ 
for $i=1,2$, then $(I_1,\Xi_1)\preceq(I_2,\Xi_2)$ iff
$I_1\subseteq I_2$ and $\Xi_1$ is the restriction of
$\Xi_2$ to $I_1$. 

It is clear that ${\cal Z}\neq\emptyset$, because
the pair $(\{0\},\Xi_x^{triv})$\mbox{---where} $\Xi_x^{triv}$ is the map 
defined above, in the paragraph preceding 
the statement of our proposition---belongs to 
${\cal Z}$.  If $Z$ is a totally ordered \mbox{subset}
of ${\cal Z}$, we show that $Z$ has an upper bound 
$(I_*,\Xi_*)$ in ${\cal Z}$. 
This conclusion is trivial if $Z=\emptyset$, for in that 
case we can take $(I_*,\Xi_*)=(\{0\},\Xi_x^{triv})$. Assume that 
$Z\neq\emptyset$. Let $I_*$ be the union of the intervals $I$ for all
the \mbox{members} $(I,\Xi)$ of $Z$. Then $I_*$ is obviously a subinterval
of $[0,\infty[$, and $0\in I$. If $t\in I_*$, we define $\Xi_*(t)=\Xi(t)$,
where $(I,\Xi)$ is any member of $Z$ such that $t\in I$.
Write $\Xi_*=(\xi_*,\xi_{0,*})$.  Then $\Xi_*$ is
obviously well defined, and is an augmented trajectory of $\hat\Sigma$
such that $\xi_*(0)=x$. If $t\in I_*$, then we can pick 
$(I,\Xi)\in Z$ such that $t\in I$, and then 
$\Xi_*(s)=\Xi(s)$ for all $s\in I$, and in \mbox{particular} for all
$s\in[0,t]$, since $[0,t]\subseteq I$. Furthemore, 
if we write $\Xi=(\xi,\xi_{0})$, then the  fact that 
$\Xi\in\sdsv$ \mbox{implies} that 
$V(x)\ge\xi_0(t)+V(\xi(t))=\xi_{0,*}(t)+V(\xi_*(t))$. \mbox{Since}
$t$ is an arbitrary member of $I_*$, we have shown that
$\Xi_*\in\sdsv$. Therefore $(I_*,\xi_*)\in {\cal Z}$. 
Furthermore, it is clear that
$(I_*,\Xi_*)$ is an upper bound for $Z$. So
we have shown that every totally ordered subset of ${\cal Z}$ has
an \mbox{upper} bound, and that ${\cal Z}\neq\emptyset$. Therefore Zorn's
Lemma implies that ${\cal Z}$ has a maximal element 
$(I_*,\Xi_*)$.  Clearly, such a maximal element 
is a member of $\msdsvx$, and our proof is complete.
\end{proof}


\medbreak An {\em augmented arc} is an augmented trajectory whose domain 
is a compact interval. If $\Xi=(\xi,\xi_0)$ is an \mbox{augmented} arc 
with domain $[a,b]$, then an \mbox{{\em improvement}} of $\Xi$ 
is an augmented arc  $\Xi'=(\xi',\xi'_0)$, with domain
$[a',b']$, such that $\xi'(a')=\xi(a)$, $\xi'(b')=\xi(b)$, 
and $\xi_0'(b')-\xi_0'(a')\le\xi_0(b)-\xi_0(a)$.



\medbreak If $\hat\Sigma=(\Omega,U,f,L)$ is an \mbox{augmented} 
control system, then an augmented arc $\Xi=(\xi,\xi_0)$ of
$\hat\Sigma$ with \mbox{domain} $[a,b]$ is 
{\em uniquely limiting} if there exists a sequence 
$\{\eta_j\}_{j=1}^\infty$ of piecewise constant functions
$\eta_j:[a,b]\mapsto U$ such that 
\begin{itemize}
\item[(*)] {\em if $\{\Xi_j\}_{j=1}^\infty$ is an arbitrary sequence
of \mbox{maximally} defined augmented trajectories of $\hat\Sigma$ 
such that \mbox{$a\in{\rm domain}(\Xi_j)$} and $\Xi_j(a)=\Xi(a)$
for every $j$, then $[a,b]\subseteq{\rm domain}(\Xi_j)$ if $j$ is large enough,
and $\Xi_j\to\Xi$ uniformly on $[a,b]$ as $j\to\infty$.}
\end{itemize}

\begin{example}\label{exalips}
Suppose $\Xi=(\xi,\xi_0)$  is an augmented arc  of
$\hat\Sigma$ with \mbox{domain} $[a,b]$ such that 
\begin{itemize}
\item[(\#)] {\em there exist a positive number $\delta$, 
a function \mbox{$\eta:[a,b]\mapsto U$}, 
and a function $\varphi:[a,b]\mapsto[0,\infty]$, such that
\begin{itemize}
\item[(i)] $\dot\Xi(t)=\Big(f(x,\eta(t)),L(x,\eta(t))\Big)\,\,$ for almost
\mbox{every} $t\in[a,b]$, 
\item[(ii)] the map $t\mapsto\Big(f(x,\eta(t)),L(x,\eta(t))\Big)\,\,$, on the 
\mbox{compact}
set 
\item[] $\qquad I_x\equaldef\{t:a\le t\le b\wedge \|x-\xi(t)\|\le\delta\}$, 
\item[] is measurable for each $x\in\Omega$, 
\item[(iii)] the map $x\mapsto\Big(f(x,\eta(t)),L(x,\eta(t))\Big)\,\,$, on the 
\mbox{compact}
set 
\item[] $\qquad I^t\equaldef\{x\in\Omega:\|x-\xi(t)\|\le\delta\}$, 
\item[] is measurable for each $t\in[a,b]$, 
\item[(iv)] $\varphi$ is integrable,
\item[(v)] the inequality
\item[] $\Big\langle f(x,\eta(t))-f(x',\eta(t)),x-x'\Big\rangle
\le\varphi(t)\|x-x'\|^2$ 
\item[] holds whenever $t\in[a,b]$, $\|x-\xi(t)\|\le\delta$, and
$\|x'-\xi(t)\|\le\delta$,
\item[(vi)] the inequality $|L(x,\eta(t))|\le\varphi(t)$ 
holds \mbox{whenever} $t\in[a,b]$ and $\|x-\xi(t)\|\le\delta$.
\end{itemize}}\end{itemize}

\medbreak\noindent Then $\Xi$ is uniquely limiting. The proof is 
essentially as follows.  By dividing $[a,b]$ into small intervals,
we can assume that there is a fixed compact ball $B$ 
such that $B\subseteq\Omega$, $\xi$ is entirely contained in the interior
of $B$, and the bound of {\em (v)} holds whenever $t\in[a,b]$ and 
$x,x'\in B$. We then write $F^u(x)=(f(x,u,L(x,u))$
for each $x\in B$, $u\in U$, 
and observe that the set ${\cal F}=\{F^{\eta(t)}:t\in[a,b]\}$ 
is a subset of the separable Banach space $C^0(B,\rr^{n+1})$. Then 
$[a,b]\ni t\mapsto F^{\eta(t)}$ is an $L^1$ $C^0(B,\rr^{n+1})$-valued map.
Therefore one can approximate this map 
in $L^1$ by piecewise constant ${\cal F}$-valued 
maps. In other words, one can find a sequence
$\{\eta_j\}_{j=1}^\infty$ of piecewise constant 
${\cal G}$-valued functions (where 
${\cal G}=\{\eta(t):t\in[a,b]\}$), and integrable functions
$k_j:[a,b]\mapsto[0,+\infty]$,  such that 

\medbreak\noindent $\|f(x,\eta(t))\!-\!f(x,\eta_j(t))\|\!+\!
|L(x,\eta(t))\!-\!L(x,\eta_j(t))|\!\le\!k_j(t)$

\medbreak\noindent whenever $a\le t\le b$, $x\in B$, and $j\in\nn$, and 
$\lim_{j\to\infty}\int_a^bk_j(t)\,dt=0$. Then, if $a\le c\le d\le b$, 
and $\zeta:[c,d]\mapsto B$, $\theta:[c,d]\mapsto B$, 
are trajectories of $\eta$, $\eta_j$, respectively, Gronwall's inequality 
yields the bound 
$$
\|\zeta(t)-\theta(t)\|\le e^{\int_a^b\varphi(s)ds}\big(\|k_j\|_{L^1}+
\|\zeta(c)-\theta(c)\|\Big)
$$
if $t\in[c,d]$. If we apply this with $c=a$ and $\zeta=\xi$, letting
$\theta$ be any trajectory $\xi_j$ of $\eta_j$ 
\mbox{starting} at $\xi(a)$ 
at time $a$, and defined on some \mbox{subinterval} $[a,d]$ of 
$[a,b]$, we see that, as long as $j$ is large \mbox{enough}, the 
\mbox{maximum} of the $\|\xi(t)-\xi_j(t)\|$ is \mbox{bounded} by a small
\mbox{constant}. This guarantees that $\xi_j$ 
\mbox{actually} exists 
on the \mbox{whole} \mbox{interval} $[a,b]$, and then the Gronwall bound
implies that $\xi_j\to \xi$ uniformly as $j\to\infty$.
Then the \mbox{integrals} $\int_a^tL(\xi_j(s),\eta_j(s))\,ds$
differ from $\int_a^tL(\xi_j(s),\eta(s))\,ds$
by less than $\|k_j\|_{L^1}$, in view of the 
bound 
$$
|L(x,\eta(t))-L(x,\eta_j(t))|\le k_j(t)\,,
$$
and 
$\int_a^tL(\xi_j(s),\eta(s))\,ds\to\int_a^tL(\xi(s),\eta(s))\,ds$
as $j\to\infty$,
because $L(\xi_j(s),\eta(s))\to L(\xi(s),\eta(s))$ for each $s$, and 
$|L(\xi_j(s),\eta(s))|\le\varphi(s)$. \proofend
\end{example}


\medbreak An augmented trajectory $\Xi=(\xi,\xi_0)$ with domain $I$ is 
{\em locally uniquely limiting} if for every compact \mbox{subinterval} 
$I'$  of $I$ the restriction of $\Xi$ to 
$I'$ is uniquely limiting.

\medbreak\noindent
An augmented trajectory $\Xi=(\xi,\xi_0)$ with domain $I$ is {\em almost
uniquely limiting} if there exists 
a finite subset $B$ of $I$ such that the restriction of
$\Xi$ to every subinterval of $I\backslash B$ is 
locally uniquely limiting.

\medbreak\noindent If $\Omega$ is an open subset of $\R^n$, 
we say that a function $V:\Omega\mapsto\R$ {\em satisfies the inequality}

\begin{equation}\label{loweq}
\sup\{-\langle \nabla V(x),f(x,u)\rangle-L(x,u):u\in U\}\ge 0
\end{equation}

\medbreak\noindent on $\Omega$ {\em in the viscosity sense} if 

\begin{center}\begin{minipage}{7.4cm}
\begin{itemize}
\item[(V$_+$)] {\em whenever $x\in\Omega$ and $p\in\R^n$ is a 
\mbox{subdifferential} of $V$ at $x$, it follows that}

$$
\sup\{-\langle p, f(x,u)\rangle-L(x,u):u\in U\}\ge 0\,.
$$
\end{itemize}
\end{minipage}\end{center}

\medbreak\noindent(We recall that, if $\Omega$ is open in $\R^n$, then
a {\em subdifferential} of a function $V:\Omega\mapsto\R$ at a point
$\bar x\in\Omega$ is a vector $p\in\R^n$ such that

$$
\lim\inf_{x\to\bar x}\frac{V(x)-V(\bar x)-p\cdot(x-\bar x)}{
\|x-\bar x\|}\ge 0\,.\,)
$$

\medbreak Similarly, we say that $V$ {\em satisfies the inequality}

\begin{equation}\label{upeq}
\sup\{-\langle \nabla V(x),f(x,u)\rangle-L(x,u):u\in U\}\le 0
\end{equation}

\medbreak\noindent on $\Omega$ {\em in the viscosity sense} if 

\begin{center}\begin{minipage}{7.4cm}
\begin{itemize}
\item[(V$_-$)] {\em whenever $x\in\Omega$ and $p\in\R^n$ is a 
\mbox{superdifferential} of $V$ at $x$, it follows that}

$$
\sup\{-\langle p, f(x,u)\rangle-L(x,u):u\in U\}\le 0\,.
$$
\end{itemize}
\end{minipage}\end{center}

\medbreak\noindent 
(A {\em superdifferential} of $V$ at $x_*$ is a vector $p$ 
such that $-p$ is a subdifferential of $-V$ at $x_*$.) 

\medbreak We say that $V$ {\em satisfies the equation}

\begin{equation}\label{eqeq}
\sup\{-\langle \nabla V(x),f(x,u)\rangle-L(x,u):u\in U\}=0
\end{equation}

\medbreak\noindent on $\Omega$ {\em in the viscosity sense} if 
it satisfies (\ref{loweq}) and (\ref{upeq}) in the viscosity 
sense. 

\begin{remark}
The definition of ``viscosity solution'' given here 
is known to be equivalent to the more \mbox{common} one involving 
test functions, cf. \cite{BARD}.\proofend
\end{remark}

Our main result is the following theorem:

\begin{theorem}\label{mainres}
Let $\hat\Sigma=(\Omega,U,f,L)$ be an \mbox{augmented} control 
system, let ${\cal T}$ be a target for $\hat\Sigma$, and
let \mbox{$V:\Omega\cup{\cal T}\mapsto\R$} be a function. Assume that
\begin{itemize}
\item[(1)] $\hat\Sigma$ is locally uniformly continuous, locally coercive,
and such that $F_{\hat\Sigma}(x,U)$ is closed and convex for \mbox{every} 
$x\in\Omega$. 
\item[(2)] $V$ is \mbox{continuous}.
\item[(3)] $V$ satisfies (\ref{eqeq}) on $\Omega$ in the
\mbox{viscosity} sense.
\item[(4)] $V$ vanishes on ${\cal T}$.
%\item[(5)] $V$ is is bounded below.
\item[(5)] Every augmented arc has an almost locally uniquely 
limiting improvement. 
\item[(6)] Whenever $x\in\Omega$, $\Xi=(\xi,\xi_0)\in\msdsvx$, 
and $\xi$ is right-unbounded, it follows that $\Xi\in\axts$.
\end{itemize}
Then $V\equiv\vts$.
\end{theorem}

\begin{remark} 
Condition (6) was essentially introduced by M. Malisoff, cf.  
especially \cite{MALI}.\proofend
\end{remark}

\ssection{Examples}\label{exasec}

\begin{example} {\em (Linear-quadratic optimal control.)} 
\mbox{Consider} the standard linear-quadratic optimal \mbox{control} problem, in which 
$x$, $u$ take values in $\R^n$, $\R^m$, \mbox{respectively},
the dynamical law is
\begin{equation}\label{dynaquad}
\dot x=Ax+Bu\,,
\end{equation}
the Lagrangian is given by 
$$
L(x,u)=x^\dagger Rx+u^\dagger Su\,,
$$
the square matrices $R$, $S$ are strictly positive \mbox{definite}, and the pair 
$(A,B)$ is stabilizable. We take the \mbox{target} set ${\cal T}$ to consist 
of the origin of $\R^n$.  (In order to \mbox{satisfy} the condition 
that $F_{\hat\Sigma}(x,U)$ is convex for \mbox{every} 
$x\in\Omega$, we add a new scalar nonnegative control \mbox{variable}
$v$, in such a way that the dynamical law (\ref{dynaquad}) remains \mbox{unchanged}
but the Lagrangian $L$ is replaced by $\tilde L$, where 
$\tilde L(x,u,v)\equaldef L(x,u)+v$.) The crucial \mbox{technical} issue here is the
fact that the Lagrangian is not bounded away from zero. 
The \mbox{hypotheses} of our main \mbox{theorem} (including the coerciveness,
which \mbox{follows} from the \mbox{positive} \mbox{definiteness} of $S$) are easily 
verified as long as $V$ is bounded below.  The only 
nontrivial point is the \mbox{verification} of condition (6). To prove that 
this  holds, let $\Xi:[0,\tau\,[\,\mapsto\R^n\times\R$ be a 
right-unbounded \mbox{maximal} \mbox{trajectory}
of steepest descent with respect to $V$ that does not end
at the target, and write $\Xi=(\xi,\xi_0)$ in the usual way. Then 
$\tau$ has to be infinite, because if $\tau$ was \mbox{finite} then
the boundedness of the cost (\mbox{arising} from the fact that 
$\Xi=(\xi,\xi_0)$ is of steepest descent and $V$ is bounded below)
would \mbox{trivially} imply an $L^2$ bound on the control, from which
it would follow that 
$\Xi$ can be extended to the closed \mbox{interval} $[0,\tau]$, and 
then the \mbox{assumption} that $\Xi$ does not end at the 
\mbox{target} would
\mbox{enable} us to use Proposition \ref{trivialprop} and
Theorem \ref{maintech} (\mbox{applied} with $\Omega=\R^n\backslash\{0\}$)
to \mbox{extend} $\Xi$ even \mbox{further}, contradicting maximality. 
So $\tau$ is \mbox{infinite}. On the other hand, the fact that
$V$ is bounded \mbox{below} and $\Xi$ is of \mbox{steepest} descent implies
that the integral 
$$
\int_0^\infty\Big(\xi(t)^\dagger R\xi(t)+
\eta(t)^\dagger S\eta(t)\Big)dt
$$
is finite, if $\eta$ is an open-loop control that generates $\Xi$.
But then $\xi$ and $\eta$ are square-integrable, so the 
\mbox{condition}
that $\dot\xi=A\xi+B\eta$ implies that $\xi$ is square-integrable and has
a square-integrable derivative, and then Barbalat's lemma
implies that  $\xi$  ends at the \mbox{target}, as desired. \proofend
\end{example}

\begin{example} {\em (Fuller's problem, cf., e.g., 
\cite{ZB}.)}  This is the optimal control problem for the dynamical law
\begin{eqnarray*}
\dot x&=&y\,,\\
\dot y&=&u\,,
\end{eqnarray*}
with control constraint $-1\le u\le 1$. The target set ${\cal T}$ consists 
of the origin of $\R^2$.  The Lagragian is $L(x,y,u)=x^2$.
The crucial \mbox{technical} issue here is the
fact that the Lagrangian is not bounded away from zero, and in fact has a 
whole line of zeros.  The \mbox{hypotheses} of our main theorem are easily 
verified as long as $V$ is bounded below. The only 
nontrivial point is the \mbox{verification} of condition (6). To prove that 
this  holds, let $\Xi:[0,\tau\,[\,\mapsto\R^3$ be a right-unbounded
\mbox{maximal} \mbox{trajectory}
of steepest descent with respect to $V$ that does not end
at the target, and write $\Xi=(\xi,\xi_0)$ in the usual way. Then 
$\tau$ has to be infinite, because if $\tau$ was \mbox{finite} then
the boundedness of the control would \mbox{trivially} imply that 
$\Xi$ can be extended to the closed \mbox{interval} $[0,\tau]$, and 
then the assumption that $\Xi$ does not end at the target
enables us to use Proposition \ref{trivialprop} and
Theorem \ref{maintech} (applied with $\Omega=\R^2\backslash\{(0,0)\}$)
to \mbox{extend} $\Xi$ even \mbox{further}, contradicting maximality. 
So $\tau$ is \mbox{infinite}. On the other hand, the fact that 
$V$ is bounded \mbox{below} and $\Xi$ is of \mbox{steepest} descent implies
that the integral $\int_0^\infty x(t)^2dt$ is finite, if we write
\mbox{$\xi(t)=(x(t),y(t))$}. But then $x(\cdot)$ is a square-integrable function
on $[0,\infty\,[\,$  whose second derivative is bounded.�
By a \mbox{straightforward}  \mbox{generalization} of Barbalat's lemma, 
this implies that 
both $x(\cdot)$ and $y(\cdot)$ go to zero, i.e., that 
$\xi$  ends at the target, as desired. \proofend
\end{example} 

\begin{example} {\em (The reflected brachistochrone problem.)} This is
the minimum time problem for the dynamical law
\begin{eqnarray*}
\dot x&=&u\sqrt{|y|}\,,\\
\dot y&=&v\sqrt{|y|}\,,
\end{eqnarray*}
with control constraint $u^2+v^2\le 1$. The target set ${\cal T}$ consists 
of a single point $B\in\R^2$.  The crucial \mbox{technical} issue here is the
fact that the dynamical law is not Lipschitz-continuous with respect to the 
state.  The \mbox{hypotheses} of our main theorem are easily verified. The only 
nontrivial point is the verification of condition (5). To prove that this 
holds, we pick an arbitrary \mbox{integral} arc $\xi:[a,b]\mapsto\R^2$,
and observe that either (i)~$\xi(t)$ never belongs to the $x$ axis
\mbox{$X=\{(x,y)\in\R^2:y=0\}$} or (ii)~there exist $t_-,t_+\in[a,b]$
such that $t_-\le t_+$, \mbox{$\xi(t)\notin X$} whenever $a\le t<t_-$
or $t_+>t\le b$, $\xi(t_-)\in X$, and $\xi(t_+)\in X$. If (i) holds, 
then $\xi$ \mbox{satisfies} the conditions  of  Example \ref{exalips}, so 
$\xi$ is uniquely \mbox{limiting}.  If (ii) holds and $t_+=t_-$, then the 
\mbox{restriction} of $\xi$ to each of the \mbox{intervals} $[a,t_-\,[+,$, 
$\,]\,t_+,b]$, is \mbox{locally} uniquely \mbox{limiting}, so 
$\xi$ is \mbox{almost} locally uniquely \mbox{limiting}.  Finally, if (ii) 
holds and $t_+<t_-$, then the restriction $\tilde\xi$ 
of $\xi$ to the interval $[t_-,t_+]$ is such that
the set \mbox{$S=\{t\in[a,b]:\xi(t)\notin X\}$} is the union of a finite
or countable infinite collection ${\cal I}$ of pairwise disjoint 
relatively open subintervals of $[a,b]$. If $I\in{\cal I}$, then 
the restriction $\tilde\xi_I$ of $\tilde\xi$ 
to $I$ is entirely contained in the
open upper half-plane or in the open lower half-plane. By reflecting 
$\tilde\xi_I$ with respect to $X$, if necessary, we may 
assume that $\tilde\xi_I$ is entirely contained in the
open upper half-plane for every $I\in{\cal I}$. Then 
$\tilde\xi$ is a trajectory of our system entirely 
\mbox{contained}
in the closed upper half-plane. It is well known that the problem 
in the closed upper half-plane $H_+$ 
is the famous ``brachistochrone problem,''
whose time-optimal \mbox{trajectories} $\zeta:[\alpha,\beta]\mapsto H_+$
are cycloids such that $\zeta(t)$ is an interior point of $H_+$ 
whenever $\alpha<t<\beta$.  It follows that we can always replace 
$\tilde\xi$ by a cycloid $\zeta$, thereby \mbox{obtaining} an
almost locally uniquely limiting \mbox{improvement} of $\xi$. \proofend
\end{example} 

\begin{example} {\em (An example with a continuous \mbox{non-Lipschitz}
dynamics where uniqueness fails.)} Let $\varphi:[0,1]\mapsto\rr$ be a 
nonnegative continuous function such that (a)~the set $\{x\in[0,1]:
\varphi(x)=0\}$ is exactly the Cantor set, and
(b)~$\int_0^1\frac{dx}{\varphi(x)}<\infty$.  (For example, we may take
$\varphi$ to be given by $\varphi(x)={\rm dist}(x,C)^\rho$, where $C$
is the Cantor set and $\rho$ is a positive number such that
$\rho<1-{\rm log}_{_3}2$.  An explicit calculation shows that
$\int_0^1\frac{dx}{\varphi(x)}=(1-\rho)^{-1}2^\rho\sum_{j=1}^{\infty}
\theta^j$, where $\theta=\frac{2}{3}\times 3^\rho$. Our choice of
$\rho$ guarantees that $\theta<1$, so the integral is finite.)  Extend
$\varphi$ to a function defined on $\rr$ by making it periodic of
period $1$. Then consider the optimal control problem on $\rr$ whose
dynamics is $\dot x=u\varphi(x)$, $|u|\le 1$, and where the goal is to
reach the origin in minimum time.  It is easy to see that the optimal 
trajectory from each point $x$ exists and is obtained by ``moving towards the 
target as fast as possible.'' Precisely, this means the we use the control 
$u=-1$ as long as we are to the right of the origin, and we use
$u=1$ if we are to the left. This, however, does not suffice to specify 
the optimal trajectories, because of the lack of uniqueness of solutions. 
The complete specification of the optimal trajectories is as follows.
Suppose $\bar x<0$. Define a function 
$\tau:[\bar x,0]\mapsto\rr$ by letting 
$\tau(x)=\int_{\bar x}^x\frac{dy}{\varphi(y)}$. Then 
$\tau$ is aboslutely continuous, strictly increasing, and such that
$\tau(\bar x)=0$. Therefore $\tau$ maps the interval 
$[\bar x,0]$ homeomorphically onto the interval 
$[0,\tau(0)]$. Let $\xi$ be the inverse function, so 
$\xi$ maps  $[0,\tau(0)]$ homeomorphically onto $[\bar x,0]$. 
Then $\xi$ is absolutely continuous, and 
$\dot\xi(t)=\varphi(\xi(t))$ for almost all $t\in[0,\tau(0)]$.
So $\xi$ is a trajectory of our system which goes from 
$\bar x$ to $0$ in time $\tau(0)$, and it is easy to see that $\xi$ 
is the optimal trajectory from $\bar x$ to $0$. It follows that optimal
time to go from $\bar x$ to $0$ is $\tau(0)$, that is, 
$\int_{\bar x}^0\frac{dy}{\varphi(y)}$.  A similar contruction
applies when  $\bar x>0$. Then the value function $\bar V$ for our problem
is given by
$$
\bar V(x)=\int_{\min(x,0)}^{\max(x,0)}\frac{dy}{\varphi(y)}\,.
$$
The HJBE for our problem is
\begin{equation}\label{isasol}
|V'(x)|\varphi(x)-1=0\,.
\end{equation}
The function $\bar V$ is a solution of this equation on $\rr\backslash\{0\}$
in the \mbox{viscosity} sense.  (This follows from the fact that, for
problems such as this one, the value function is \mbox{automatically} a
viscosity solution of the HJBE. In \mbox{addition}, one can also verify this
directly.  Let ${\cal O}$ be the set of points where
$\varphi(x)>0$. Then on ${\cal O}$ the function $\bar V$ is smooth,
and its derivative is $\frac{1}{\varphi}$ when $x<0$, and
$-\frac{1}{\varphi}$ when $x>0$, so (\ref{isasol}) holds. At points
$x$ where $\varphi(x)=0$, the viscosity solution requirements say that
$-1\ge 0$ whenever $p$ is a subdifferential of $\bar V$ at $x$, and
$-1\le 0$ whenever $p$ is a superdifferential of $\bar V$ at $x$. The
second condition is trivially true. To verify the first condition,
we need to show that it is satisfied vacuously, i.e., that there are 
no subdifferentials of $\bar V$ at $x$. But this easy. Suppose, say, that 
$x<0$. The difference quotient 
$\frac{1}{h}(V(x+h)-V(x))$ is equal, if $h>0$, to 
$-\frac{1}{h}\int_x^{x+h}\frac{dy}{\varphi(y)}$, which is bounded above 
by $\zeta(h)=-\frac{1}{\max\{\varphi(y):y\in[x,x+h]\}}$. 
Since $\varphi(x)=0$, $\zeta(h)$ goes to 
$-\infty$ as $h\to 0$.  This shows that the right derivative of $\bar V$ 
at $x$ is equal to $�\infty$, from which it follows easily 
that there exist no \mbox{subdifferentials} of $\bar V$ at $x$. 
A similar argument shows that if $x>0$ the left derivative of $\bar V$ 
at $x$ is equal to $+\infty$, from which it follows once again 
that there are no subdifferentials of $\bar V$ at $x$.

We now show that there exist nonnegative \mbox{continuous} functions
$\hat V:\rr\mapsto\rr$ other than $\bar V$ that satisfy the HJBE on
$\rr\backslash\{0\}$ and are such that $\hat V(0)=0$.  To see this, we
let $W$ be a continuous monotonically nondecreasing real-valued
function on $[0,\infty\,[\,$ such that (a)~$W(0)=0$, (b)~$W$ is
constant on each connected component of the set
$\{x:x>0\wedge\varphi(x)>0\}$, and (c)~$W(x)<W(y)$ \mbox{whenever}
$0\le x<y$ and the interval $[x,y]$ contains a zero of $\varphi$.
(Such a function is easily constructed using the well known Cantor
function.)  We then \mbox{extend} $W$ to all of $\rr$ by defining
$W(x)=W(-x)$ when $x<0$. Using $W$, we define $\hat V=\bar V+W$.  Then
$\hat V$ is \mbox{continuous}, \mbox{$\hat V(0)=0$}, and $\hat
V(x)>\bar V(x)$ whenever $x\neq 0$.  Let us show that $\hat V$ is also
a solution of the HJBE for our \mbox{problem} on $\rr\backslash\{0\}$.  Near
points $x$ such that $\varphi(x)>0$, the functions $\bar V$ and $\hat
V$ differ by a constant, so the fact that $\bar V$ satisfies the HJBE
implies that the same is true for $\hat V$.  If $x\neq 0$ but
$\varphi(x)=0$, the \mbox{viscosity} \mbox{solution} requirements say that 
$-1\ge 0$ whenever $p$ is a \mbox{subdifferential} of $\hat V$ at $x$,
and $-1\le 0$ whenever $p$ is a superdifferential of $\hat V$ at $x$,
and the second one of these conditions is trivially true.  As for the
first \mbox{condition}, if $x<0$ then we have already shown that the
right \mbox{derivative} of $\bar V$ at $x$ is equal to $�\infty$, and
this clearly \mbox{implies} that the right derivative of $\hat V$ at
$x$ is equal to $�\infty$ as well, since $\hat V=\bar V+W$ and $W$ is
\mbox{monotonically} \mbox{nonincreasing} near $x$. Hence there 
\mbox{exist} no \mbox{subdifferentials} of $\hat V$ at $x$. A similar
argument applies if $x>0$, and we conclude that first one of the
viscosity requirements is satisfied vacuously.

It follows that for our example the value function is not the unique
continuous nonnegative function that vanishes at the target and
satisfies the HJBE. In the example, the reason for the failure of
uniqueness is easy to \mbox{understand}, and clearly related to the
non-uniqueness of trajectories. Notice that the spurious value
\mbox{function} $\hat V$ is bounded below by the true value function,
so what goes wrong is the other inequality, which is related to the
\mbox{dynamic} programming inequality (DPI). And, \mbox{indeed}, the
DPI fails, and this makes it impossible to draw the conclusion that
$\hat V\le\bar V$. Furthermore, the failure of the DPI happens exactly
as described in our general \mbox{analysis}: given any control
$u(\cdot)$ and any initial \mbox{condition} $x_0$, it is easy to
construct a maximal trajectory $\xi$ for $u(\cdot)$ starting at $x_0$
along which the DPI for $\hat V$ holds.  (It \mbox{suffices} to follow
the only possible trajectory for $u(\cdot)$ as long as $\varphi\neq
0$, and stopping at $\bar x$ and staying there for ever as soon as we
reach the first point $\bar x$ where $\varphi$ \mbox{vanishes}.)  This
$\xi$ is not, however, the only \mbox{trajectory} for $u(\cdot)$
\mbox{starting} at $x_0$.  And the fact that the DPI holds along $\xi$
does not imply that that the DPI holds for \mbox{every} trajectory for
$u(\cdot)$ that starts at $x_0$.  (Indeed, if for \mbox{example}
$x_0<0$ and $u(t)\equiv 1$, then in addition to the $\xi$ given by our
construction we could also consider $\xi_{opt}$, the optimal
trajectory described earlier. The DPI for $\hat V$ clearly fails along
$\xi_{opt}$, because if it was true it would imply that $\hat
V(x_0)\le\bar V(x_0)$, whereas we know that $\hat V(x_0)>\bar
V(x_0)$.) \proofend
\end{example} 


\ssection{The main technical lemma}\label{maintechsec}

\def\Phids{\Phi_{\delta,\hat\Sigma}}

\subsection{Infinitesimal steepest descent.}
\noindent Let \mbox{$\hat\Sigma=(\Omega,U,f,L)$} be an augmented control 
system. For every $x\in\Omega$ and every positive number $\delta$ such that 
${\rm dist}(x,\R^n\backslash\Omega)>\delta$, we let
$\Phids(x)$ be the closed \mbox{convex} hull of all the vectors
$\fsh(x',u)$, for all pairs $(x',u)$ such that $x'\in\Omega$,
$\|x'-x\|\le\delta$, and $u\in U$. Then $\Phids(x)$ is a closed
convex subset of $\R^{n+1}$.  Clearly, 
\mbox{$\Phids(x)\subseteq\Phi_{\delta',\hat\Sigma}(x)$} whenever
$0<\delta\le\delta'$. 

Let $V:\Omega\mapsto\bar\R$ be a real-valued function,
and let $x_*\in\Omega$. We say that $V$ {\em satisfies the infinitesimal 
steepest descent condition for $\hat\Sigma$ at $x_*$} if 

\medbreak
\begin{minipage}{8cm}
\begin{itemize}%\item[]\begin{itemize}
\item[(ISD)] {\em there exist 
sequences $\{x_j\}_{j=1}^\infty$, $\{v_j\}_{j=1}^\infty$,
$\{\lambda_j\}_{j=1}^\infty$, $\{h_j\}_{j=1}^\infty$,
$\{\gamma_j\}_{j=1}^\infty$
in $\Omega$, $\R^n$, $\R$, $\R$, and $\R$, 
\mbox{respectively}, such that 
\begin{itemize}
\item[(1)] $h_j>0$, $\gamma_j>0$, $(v_j,\lambda_j)\in\fsh(x_*,U)$, and
$\|x_j-x_*-h_jv_j\|\le h_j\gamma_j$ for all $j$, 
\item[(2)]
$h_j\downarrow 0$, $x_j\to x_*$, 
and $\gamma_j\downarrow 0$ as $j\to\infty$,
\end{itemize}
and}
\begin{itemize}
\item[(3)] $V(x_j)\le V(x_*)-h_j\lambda_j+h_j\gamma_j\,\,\,
{\rm for\,\,\, all}\,\,\,j$.
\end{itemize}\end{itemize}%\end{itemize}
\end{minipage}

\medbreak\noindent We say that $V$ {\em satisfies the weak infinitesimal 
steepest descent condition for $\hat\Sigma$ at $x_*$} if 

\medbreak
\begin{minipage}{8cm}
\begin{itemize}\item[]\begin{itemize}
\item[(WISD)] {\em there exist 
sequences $\{x_j\}_{j=1}^\infty$, $\{v_j\}_{j=1}^\infty$,
$\{\lambda_j\}_{j=1}^\infty$, $\{h_j\}_{j=1}^\infty$,
$\{\delta_j\}_{j=1}^\infty$, $\{\gamma_j\}_{j=1}^\infty$
in $\Omega$, $\R^n$, $\R$, $\R$, $\R$, and $\R$, respectively, such that 
\begin{itemize}
\item[(1)] $h_j>0$, $\delta_j>0$, $\gamma_j>0$, 
\mbox{$(v_j,\lambda_j)\in\Phi_{\delta_j,\hat\Sigma}(x_*)$}, $\|x_j-x_*\|\le\delta_j$, and $\|x_j-x_*-h_jv_j\|\le h_j\gamma_j$ for all $j$, 
\item[(2)]
$h_j\downarrow 0$, $\delta_j\downarrow 0$, and 
$\gamma_j\downarrow 0$ as $j\to\infty$,
\end{itemize}
and}
\begin{itemize}
\item[(3)] $V(x_j)\le V(x_*)-h_j\lambda_j+h_j\gamma_j\,\,\,
{\rm for\,\,\, all}\,\,\,j$.
\end{itemize}\end{itemize}\end{itemize}
\end{minipage}

\medbreak\noindent We say that $V$ {\em satisfies (\ref{loweq}) on $\Omega$ 
in the ISD sense} if $V$ satisfies the infinitesimal steepest descent
condition for $\hat\Sigma$ at $x$ for every \mbox{$x\in\Omega$}.  We
say that $V$ {\em satisfies (\ref{loweq}) on $\Omega$ in the WISD
sense} if $V$ satisfies the weak infinitesimal steepest descent
condition for $\hat\Sigma$ at $x$ for every \mbox{$x\in\Omega$}.

\begin{theorem}\label{satinfi}
Let $\hat\Sigma=(\Omega,U,f,L)$ be an $n$-dimensional augmented control 
system and let \mbox{$V:\Omega\mapsto\R$} be a \mbox{continuous} function. 
Then
\begin{itemize}
\item[(1)] Condition (WISD) holds at every point $x_*\in\Omega$ where 
(ISD) holds. In particular, if $V$ \mbox{satisfies} (\ref{loweq}) on $\Omega$ 
in the ISD sense then $V$ \mbox{satisfies} (\ref{loweq}) on $\Omega$ 
in the WISD sense. 
\item[(2)] If $\hat\Sigma$ is locally \mbox{uniformly} continuous
and such that $\fsh(x,U)$ is convex for every $x\in\Omega$,
then (ISD) holds at every point $x_*$ where (WISD) holds, and in particular
if $V$ \mbox{satisfies} (\ref{loweq}) on $\Omega$ 
in the WISD sense then 
$V$ \mbox{satisfies} (\ref{loweq}) on $\Omega$ in the ISD sense. 
\item[(3)] If $\hat\Sigma$ is locally \mbox{coercive}, and
and such that $\fsh(x,U)$ is closed and convex
for every $x\in\Omega$, then
\begin{itemize}
\item[(3.i)] if $V$ \mbox{satisfies} (\ref{loweq}) on $\Omega$ 
in the ISD sense then 
$V$ \mbox{satisfies} (\ref{loweq}) on $\Omega$ 
in the \mbox{viscosity} sense;
\item[(3.ii)] if $V$ \mbox{satisfies} 
(\ref{loweq}) on $\Omega$ in the \mbox{viscosity} sense it follows 
that if $V$ \mbox{satisfies} (\ref{loweq}) on $\Omega$ in the WISD sense. 
\end{itemize}\end{itemize}
In particular, if $\hat\Sigma$ is  locally \mbox{uniformly} continuous,
locally coercive, and such that $\fsh(x,U)$ is closed and convex
for every $x\in\Omega$, then the three concepts of solution 
of (\ref{loweq}) on $\Omega$ (viscosity, ISD, and WISD) are equivalent.
\end{theorem}
\begin{proof}
We first prove (1). We assume that $V$ $x_*$ is a point of $\Omega$
where (ISD) holds, and prove that (WISD) is true as well. 
Let $\{x_j\}_{j=1}^\infty$, $\{v_j\}_{j=1}^\infty$,
$\{\lambda_j\}_{j=1}^\infty$, $\{h_j\}_{j=1}^\infty$,
$\{\gamma_j\}_{j=1}^\infty$ be sequences that satisfy the
\mbox{properties} of (ISD). Define $\delta_j=\|x_j-x_*\|$. Then
all the \mbox{conclusions} of (WISD) are true, so (WISD) holds,
and (1) is proved. 

\medbreak  Next, we prove (2). We assume that $\hat\Sigma$ is locally 
\mbox{uniformly} continuous and (WISD) holds at a point $x_*\in\Omega$,
and prove that (ISD) \mbox{holds} at $x_*$ as well.  To do
this, we pick sequences $\{x_j\}_{j=1}^\infty$,
$\{v_j\}_{j=1}^\infty$, $\{\lambda_j\}_{j=1}^\infty$,
$\{h_j\}_{j=1}^\infty$, $\{\delta_j\}_{j=1}^\infty$,
$\{\gamma_j\}_{j=1}^\infty$ with the properties specified in
(WISD). We pick $\delta$ such that

$$
B_\delta(x_*)=\{x\in\R^n:\|x-x_*\|\le\delta\}\subseteq\Omega\,,
$$

\medbreak\noindent and a \mbox{function} 
$\omega:\,\,]\,0,+\infty[\,\,\mapsto[0,+\infty]$ 
such that $\lim_{s\downarrow 0}\omega(s)=0$ and

$$
\|f(x,u)-f(x',u)\|+|L(x,u)-L(x',u)|\le\omega(\|x-x'\|)
$$

\medbreak\noindent whenever $x,x'\in B_\delta(x_*)$. We then 
pass to a \mbox{subsequence}, if necessary, and assume that 
$\delta_j\le\delta$ for all $j$. We use the fact that
$(v_j,\lambda_j)\in\Phi_{\delta_j,\hat\Sigma}(x_*)$ to find a 
$(w_j,\ell_j)$ of the form $\sum_{k=0}^{n+1}\alpha_{j,k}\fsh(x_{j,k},u_{j,k})$ 
such that $\alpha_{j,k}\ge 0$, 
$\sum_{k=0}^{n+1}\alpha_{j,k}=1$, $\|x_{j,k}-x_*\|\le\delta_j$,
and $\|w_j-v_j\|+|\ell_j-\lambda_j|\le 2^{-j}$. 
We then define 
\begin{eqnarray*}
\tilde v_j&=&\sum_{k=0}^{n+1}\alpha_{j,k}f(x_*,u_{j,k})\,,\\
\tilde \lambda_j&=&\sum_{k=0}^{n+1}\alpha_{j,k}L(x_*,u_{j,k})\,,
\end{eqnarray*}
and conclude that $\|\tilde v_j-w_j\|\le\omega(\delta_j)$
and 
\mbox{$\|\tilde\lambda_j-\ell_j\|\le\omega(\delta_j)$}. We then let
$$
\tilde\gamma_j=\gamma_j+2^{-j}+\omega(\delta_j)\,,
$$
so $\tilde\gamma_j\to 0$ as $j\to\infty$. It is then clear that 
\mbox{$\|x_j=x_*-h_j\tilde v_j\|\le h_j\tilde\gamma_j$}
and \mbox{$V(x_j)\le V(x_*)-h\tilde\lambda_j+h_j\tilde\gamma_j$}
for all $j$. \mbox{Finally,} we have
$(\tilde v_j,\tilde\lambda_j)=
\sum_{k=0}^{n+1}\alpha_{j,k}\fsh(x_*,u_{j,k})$, so 
$(\tilde v_j,\tilde\lambda_j)\in\fsh(x_*,U)$, since 
$\fsh(x_*,U)$ is convex. This completes the proof that (WISD) implies
(ISD). 

\medbreak We now turn to the proof of (3), for which purpose we assume that 
$\hat\Sigma$ is locally \mbox{coercive}
and such that $\fsh(x,U)$ is closed and convex
for every $x\in\Omega$. To prove (3.i), we assume in addition
that $V$ \mbox{satisfies}
(\ref{loweq}) on $\Omega$ in the ISD sense,  and 
show that $V$ \mbox{satisfies} (\ref{loweq}) on $\Omega$ in the
\mbox{viscosity} sense.  To prove this, we pick $x_*\in\Omega$
and a subdifferential $p$ of $V$ at $x_*$, and show that
\begin{equation}\label{topro}
\sup\{-p\cdot f(x_*,u)-L(x_*,u):u\in U\}\ge 0\,.
\end{equation}

\medbreak\noindent Using (ISD), we pick \mbox{sequences} 
$\{x_j\}_{j=1}^\infty$, 
$\{v_j\}_{j=1}^\infty$, $\{\lambda_j\}_{j=1}^\infty$, $\{h_j\}_{j=1}^\infty$,
$\{\gamma_j\}_{j=1}^\infty$
in $\Omega$, $\R^n$, $\R$, $\R$, and $\R$, 
\mbox{respectively}, such that 
$h_j>0$, $\gamma_j>0$, \mbox{$(v_j,\lambda_j)\in\fsh(x_*,U)$}, 
$\|x_j-x_*-h_jv_j\|\le h_j\gamma_j$,
and \mbox{$V(x_j)\le V(x_*)-h_j\lambda_j+h_j\gamma_j$}
for all $j$, and \mbox{$h_j\downarrow 0$}, \mbox{$x_j\to x_*$}, 
and $\gamma_j\downarrow 0$ as $j\to\infty$. Let 
\mbox{$J=\{j:x_j=x_*\}$}.  Then for $j\in J$ the \mbox{inequality}
$\|x_j-x_*-h_jv_j\|\le h_j\gamma_j$ implies 
$\|v_j\|\le\gamma_j$, so $\lim_{j\to\infty,j\in J}v_j=0$. 
On the \mbox{other} hand, the \mbox{inequality}
\mbox{$V(x_j)\le V(x_*)-h_j\lambda_j+h_j\gamma_j$} 
\mbox{implies} $h_j\lambda_j\le h_j\gamma_j$, i.e., 
$\lambda_j\le \gamma_j$. Since the sequence
$\{\lambda_j\}_{j=1}^\infty$  is bounded \mbox{below} 
(for example, because the \mbox{local} coercivity \mbox{implies} a
bound $\lambda_j\ge\|v_j\|^r-C$), we may assume, after 
\mbox{replacing} $J$ by a smaller infinite set, if necessary, 
that $\lambda=\lim_{j\to\infty,j\in J}\lambda_j$ exists.
Since $\lambda_j\le \gamma_j$, $\lambda$ must be $\le 0$. 
Furthermore, the vector $(0,\lambda)$ is a limit 
of vectors $(v_j,\lambda_j)\in\fsh(x_*)$, so 
$(0,\lambda)\in\fsh(x_*,U)$, since $\fsh(x_*,U)$ is closed. 
Hence there exists $\bar u\in U$ such that 
\mbox{$f(x_*,\bar u)=0$} and $L(x_*,\bar u)\le 0$. But then 
$-p\cdot f(x_*,\bar u)-L(x_*,\bar u)\ge 0$, so 
(\ref{topro}) holds. 

We now consider the case when the set $J$ is finite. In this case, after 
passing to a subsequence, if necessary, we may assume that $J$ is empty, 
i.e., that $x_j\neq x_*$ for all $j$. Since $p$ is a 
subdifferential of $V$ at $x_*$, we have
\begin{equation}\label{eqa-a}
\lim\inf_{x\to x_*,x\neq x_*}\frac{V(x)-V(x_*)-p\cdot(x-x_*)}
{\|x-x_*\|}\ge 0\,.
\end{equation}
Since the $x_j$ converge to $x_*$ and are different from $x_*$, 
(\ref{eqa-a}) implies
\begin{equation}\label{eqa-b}
\lim\inf_{j\to\infty}\frac{V(x_j)-V(x_*)-p\cdot(x_j-x_*)}
{\|x_j-x_*\|}\ge 0\,.
\end{equation}
Since \mbox{$V(x_j)\le V(x_*)-h_j\lambda_j+h_j\gamma_j$}, 
(\ref{eqa-b}) implies
\begin{equation}\label{eqa-c}
\lim\inf_{j\to\infty}\frac{-h_j\lambda_j+h_j\gamma_j-p\cdot(x_j-x_*)}
{\|x_j-x_*\|}\ge 0\,.
\end{equation}
Now, $x_j-x_*=x_j-x_*-h_jv_j+h_jv_j=h_j(w_j+v_j)$, where
$w_j=h_j^{-1}(x_j-x_*-h_jv_j)$. Hence 
\begin{equation}\label{eqa-d}
\lim\inf_{j\to\infty}\frac{-\lambda_j+\gamma_j-p\cdot(w_j+v_j)}
{\|w_j+v_j\|}\ge 0\,.
\end{equation}
Hence, given a positive $\eps$ there exists a $j(\eps)$ such that 
$$
-\lambda_j+\gamma_j-p\cdot(w_j+v_j)\ge-\eps\|w_j+v_j\|\,\,{\rm whenever}
\,\,j\ge j(\eps)\,.
$$
Therefore

$$
-\lambda_j-p\cdot v_j\ge-\eps\|v_j\|-\eps\|w_j\|
+p\cdot w_j-\gamma_j\,\,{\rm if}\,\,j\ge j(\eps)\,.
$$

\medbreak\noindent Since $w_j\to 0$, there is---for each $\eps$---a 
$j'(\eps)$ such that

\begin{equation}\label{eqa-i}
-\lambda_j-p\cdot v_j\ge-\eps\|v_j\|-\eps
\,\,{\rm whenever}\,\,j\ge j'(\eps)\,.
\end{equation}

\medbreak\noindent The coercivity bound yields $\|v_j\|^r\le\lambda_j+C$, 
so \mbox{$-\lambda_j\le-\|v_j\|^r+C$}. Hence 

$$
-\|v_j\|^r+C-p\cdot v_j\ge-\eps\|v_j\|-\eps
\,\,{\rm whenever}\,\,j\ge j'(\eps)\,,
$$

\medbreak\noindent so

\begin{equation}\label{eqa-e}
-\|v_j\|^r+\eps\|v_j\|+\|p\|.\|v_j\|\ge-\eps-C
\,{\rm if}\,j\ge j'(\eps)\,.
\end{equation}

\medbreak\noindent 
Now, if the sequence $\{\|v_j\|\}_{j=1}^\infty$ was unbounded, then 
we could pick an infinite subset $J$ of $\nn$ such that 
$\|v_j\|\to+\infty$ as $j\to\infty$ via values in $J$. But then, taking 
for example $\eps=1$, we would contradict (\ref{eqa-e}), because 
the number $-\|v_j\|^r+\|v_j\|+\|p\|.\|v_j\|$ is equal to 
\mbox{$-\|v_j\|^r\Big(1-(1+\|p\|)\|v_j\|^{1-r}\big)$}, which 
goes to $-\infty$ as $j\to\infty$ via values in $J$.  Therefore 
the sequence $\{\|v_j\|\}_{j=1}^\infty$ is bounded. Pick a
constant $K$ such that $\|v_j\|\le K$ for all 
$j$. Then, for each $\eps$, if $u_\eps$ is such that
$v_{j'(\eps)}=f(x_*,u_\eps)$ and $\lambda_{j'(\eps)}=L(x_*,u_\eps)$,
(\ref{eqa-i}) implies 
$$
-p\cdot f(x_*,u_\eps)-L(x_*,u_\eps)\ge -\eps(K+1)\,.
$$
Hence 
$$
\sup\{-p\cdot f(x_*,u)-L(x_*,u):u\in U\}\ge-\eps(K+1)\,.
$$
Since $\eps$ is arbitrary, we see that (\ref{topro}) holds. 
This \mbox{concludes} the proof of (3.i).

\medbreak We now proceed to proving (3.ii).  We assume that
$V$ satisfies (\ref{loweq}) on $\Omega$ in the viscosity sense, pick
an $x_*\in\Omega$, and prove that condition \mbox{(WISD)} holds. We do
this by \mbox{assuming} that the sequences whose \mbox{existence} is
asserted by (WISD) do not exist and \mbox{deriving} a
\mbox{contradiction}.  We will \mbox{assume}, as we clearly may
\mbox{without} loss of \mbox{generality}, that $x^*=0$ and $V(x^*)=0$,
i.e., $V(0)=0$.  In particular, this implies of course that $0\in\Omega$.

Since (WISD) is not satisfied, there must exist a $\bar\gamma$ such that 
$0<\bar\gamma$, 
\begin{equation}\label{traf-uxux}
\{x\in\rr^n:\|x\|\le\bar\gamma\}\subseteq\Omega\,,
\end{equation} 
and
\begin{equation}\label{traf}
V(h(v+v'))+h\lambda>h\bar\gamma
\end{equation} 
whenever $0<h\le\bar\gamma$, %$0<\delta\le\bar\gamma$, 
%$(v,\lambda)\in\Phi_{\delta,\hat\Sigma}(0)$, $h\|v\|\le\delta$, 
$(v,\lambda)\in\Phi_{\bar\gamma,\hat\Sigma}(0)$, 
\mbox{$h\|v+v'\|\le\bar\gamma$},  and $\|v'\|\le\bar\gamma$.
(Indeed, if $\bar\gamma$ did not exist, then for each sufficiently
large natural number
$j$ we could \mbox{define} $\gamma_j=2^{-j}$, and find
$h_j$, %$\delta_j$,  
$v_j$, $\lambda_j$, $v'_j$, $x_j$,
such that $0<h_j\le\gamma_j$, %$0<\delta_j<2^{-j}$, 
$(v_j,\lambda_j)\in\Phi_{\gamma_j,\hat\Sigma}(0)$, 
\mbox{$h_j\|v_j+v'_j\|\le\gamma_j$},  
%$(v_j,\lambda_j)\in\Phi_{\delta_j,\hat\Sigma}(0)$, $h_j\|v_j\|<\delta_j$,  
$\|v'_j\|\le\gamma_j$, 
and \mbox{$V(h_j(v_j+v'_j))+h_j\lambda_j\le\gamma_jh_j$}.  But then, if we take
\mbox{$x_j=h_j(v_j+v'_j)$, $\delta_j=\gamma_j$}, the sequence
$\{(x_j,v_j,\lambda_j,h_j,\delta_j,\gamma_j)\}_{j=1}^\infty$ 
satisfies conditions (1), (2) and (3) of (WISD), contradicting the fact 
that $V$ does not satisfy (WISD).)

By making $\bar\gamma$ smaller, if necessary, we can assume
that there exist real numbers $C$, $r$, such that $C>0$, $r>1$ and
$\|f(x,u)\|^r\le L(x,u)+C$ whenever $\|x\|\le\bar\gamma$
and $u\in U$.  It then follows, if we let
$$
\psi(v,\lambda)=\|v\|^r-\lambda-C\,\,{\rm for}\,\,(v,\lambda)\in
\R^n\times\R\,,
$$
that 
\begin{equation}\label{itthefol}
\!\!\!\!\psi(w)\le 0\,\,{\rm whenever}\,\,w=F(x,u)\,,\|x\|\le\bar\gamma,
u\in U\,.
\end{equation} 
Since $\psi$ is convex, the inequality
$\psi(w)\le 0$ holds whenever $w\in\Phi_{\bar\gamma,\hat\Sigma}(0)$, and then
\begin{equation}\label{coer-a}
\|v\|^r\le\lambda+C\quad{\rm whenever}\,\,\,
(v,\lambda)\in\Phi_{\bar\gamma,\hat\Sigma}(0)\,.
\end{equation}
If $(v,\lambda)\in\Phi_{\bar\gamma,\hat\Sigma}(0)$, $v'\in\R^n$, and
$\|v'\|\le\bar\gamma$, then the inequality 
$(\alpha+\beta)^r\le 2^r(\alpha^r+\beta^r)$, valid 
for nonnegative $\alpha$, $\beta$, implies
\begin{eqnarray*}
\|v+v'\|^r&\le&(\|v\|+\|v'\|)^r\\
&\le&2^r(\|v\|^r+\|v'\|^r)\\
&\le&2^{r}(\lambda+C+\bar\gamma^r)\,.
\end{eqnarray*}
Pick $\tilde r$ such that $1<\tilde r<r$. Let $A=2^{\frac{r}{r-\tilde r}}$
Then
\begin{eqnarray*}
\|v+v'\|^{\tilde r}&=&\|v+v'\|^{\tilde r-r}\cdot\|v+v'\|^{r}\\
&\le& A^{\tilde r-r}\|v+v'\|^{r}\\
&\le& A^{\tilde r-r}2^{r}(\lambda+C+\bar\gamma^r)\\
&=&2^{-r}2^{r}(\lambda+C+\bar\gamma^r)\\
&=&\lambda+C+\bar\gamma^r
\end{eqnarray*}
if $\|v+v'\|\ge A$. On the other hand, 
if $\|v+v'\|\le A$, then 
$\|v+v'\|^{\tilde r}\le A^{\tilde r}\le \|v\|^r+A^{\tilde r}\le\lambda+C+
A^{\tilde r}$. 
Therefore, if we let $\tilde C=C+\max(A^{\tilde r},\bar\gamma^r)$, and then 
relabel $\tilde r$, $\tilde C$ as our new $r$ and $C$, we have shown that 
\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\Big((v,\lambda)\in\Phi_{\bar\gamma,\hat\Sigma}(0)\wedge
v'\in\R^n\wedge\|v'\|\le\bar\gamma\Big)\nonumber\\
&\,&\,\,\Rightarrow \|v+v'\|^r\le\lambda+C\,.\label{coer-ai}
\end{eqnarray}



For $0<\delta\le\bar\gamma$, we define
\begin{eqnarray*}
&\,&\Xi_\delta(0)\equaldef\Big\{(v+v',\lambda+\lambda'):\\
&\,&(v,\lambda)\in\Phi_{\delta,\hat\Sigma}(0), v'\in\R^n\,,\,
\|v'\|\le\bar\gamma\,,\,\lambda'\in\R\,,\,\lambda'\ge 0\Big\}\,.
\end{eqnarray*}
Then 
\begin{itemize}
\item[(a)]  {\em If $0<\delta\le\bar\gamma$, then $\Xi_{\delta}(0)$ is 
a closed, convex, nonempty subset of $\R^{n+1}$}
\end{itemize}
(The fact that $\Xi_\delta(0)$ is nonempty follows because
\mbox{$F_{\hat\Sigma}(0,U)\subseteq\Phi_{\delta,\hat\Sigma}(0)
\subseteq\Xi_\delta(0)$}, and $F_{\hat\Sigma}(0,U)\neq\emptyset$ because 
$U\neq\emptyset$. 
The fact that $\Xi_\delta(0)$ is closed follows because, if 
a \mbox{sequence} $\{(v_j+v_j',\lambda_j+\lambda_j')\}_{j=1}^\infty$
with the property that $(v_j,\lambda_j)\in\Phi_{\delta,\hat\Sigma}(0)$,
$v_j'\in\R^n$, $\|v_j'\|\le\bar\gamma\in\R^n$, 
$\lambda_j'\ge 0$ converges to a limit 
$(\hat v,\hat\lambda)$, then the sequence 
$\{v_j'\}_{j=1}^\infty$ is bounded, so we may assume
after passing to a \mbox{subsequence} that $\lim_{j\to\infty}v_j'=v'$
exists, and then of course $\|v'\|\le\bar\gamma\in\R^n$, 
and $\lim_{j\to\infty}v_j=v$ \mbox{exists} as well, since 
$v_j+v'_j\to \hat v$, and then $\hat v=v+v'$. Furthermore, the bound
\mbox{$\|v_j\|^r\le\lambda_j+C$}  implies 
that \mbox{$\lambda_j\ge\|v_j\|^r-C\ge -C$}, so 
$$
\lambda_j'=(\lambda_j+\lambda_j')-\lambda_j\le (\lambda_j+\lambda_j')+C\,.
$$
Hence the sequence
$\{\lambda_j'\}_{j=1}^\infty$ is bounded above,
because $\{\lambda_j+\lambda_j'\}_{j=1}^\infty$ is 
convergent. Since $\lambda_j'\ge 0$, the sequence
$\{\lambda_j'\}_{j=1}^\infty$ is bounded, so we may assume it is
convergent to a limit $\lambda'$, after passing to a subsequence.
Clearly, then, $\lambda'\ge 0$, and the limit
$\lim_{j\to\infty}\lambda_j=\lambda$ exists as well,
and satisfies $\hat\lambda=\lambda+\lambda'$. Since
$(v,\lambda)=\lim_{j\to\infty}(v_j,\lambda_j)$, 
$(v_j,\lambda_j)\in\Phi_{\delta,\hat\Sigma}(0)$, and 
$\Phi_{\delta,\hat\Sigma}(0)$ is closed, we see that 
$(v,\lambda)\in\Phi_{\delta,\hat\Sigma}(0)$. Since 
$\|v'\|\le\bar\gamma\in\R^n$, and $\lambda'\ge 0$, we see that
$(\hat v,\hat\lambda)\in\Xi_\delta(0)$. The 
convexity of $\Xi_\delta(0)$ is trivial.)

Let $\Lambda=\{0\}\times[\bar\gamma,+\infty\,[\,$, so 
$\Lambda\subseteq\R^{n+1}$.  We 
let $\Psi_{\delta}(0)$ be the convex hull of
$\Lambda\cup\Xi_\delta(0)$. We show that
\begin{itemize}
\item[(b)] {\em If $0<\delta\le\bar\gamma$ then
$\Psi_{\delta}(0)$ is a nonempty closed \mbox{convex} subset 
of $\R^{n+1}$.}
\item[(c)] {\em There exist real numbers $r,C$ such that 
$r>1$ and}
\begin{equation}\label{coer-ao}
\|v\|^r\le\lambda+C\quad{\rm whenever}\,\,\,
(v,\lambda)\in\Psi_{\bar\gamma}(0)\,.
\end{equation}
\item[(d)] {\em The inequality
\begin{equation}\label{traf-a}
V(hv)+h\lambda\ge h\bar\gamma
\end{equation} 
holds whenever $0<h\le\bar\gamma$, $(v,\lambda)\in\Psi_{\bar\gamma}(0)$,
and $h\|v\|\le\bar\gamma$.} 
\item[(e)] {\em $(0,\ell)\notin\Psi_{\bar\gamma}(0)$ whenever
$\ell<\bar\gamma$}
\end{itemize}
We will prove the above assertions in order, except for the statement
that $\Psi_{\delta}(0)$ is  closed if $0<\delta\le\bar\gamma$, which will 
be proved last.

The fact that $\Psi_{\delta}(0)$ is convex is trivial,
and the fact that $\Psi_{\delta}(0)$ is nonempty 
follows from (a), \mbox{because} $\Xi_{\delta}(0)\subseteq\Psi_{\delta}(0)$.

To prove (c), we choose $r,C$ such that $r>1$, $C>0$, and 
(\ref{coer-ai}) holds,
and observe that (\ref{coer-ai}) trivially \mbox{implies} 
that the inequality $\|v\|^r\le\lambda+C$ is true 
\mbox{whenever} $(v,\lambda)\in\Xi_{\bar\gamma}(0)$.
Since $C>0$, the inequality is also 
true whenever $(v,\lambda)\in\Lambda$.
Hence  $\|v\|^r\le\lambda+C$ \mbox{whenever} $(v,\lambda)\in
\Xi_{\bar\gamma}(0)\cup\Lambda$, from which it follows that
$\|v\|^r\le\lambda+C$ whenever $(v,\lambda)\in\Psi_{\delta}(0)$,
since the function 
$(v,\lambda)\mapsto\|v\|^r-\lambda-C$ is convex.

To prove (d), we observe that (\ref{traf}) trivially 
\mbox{implies} that $V(hv)+h\lambda> h\bar\gamma$ whenever
$(v,\lambda)\in\Xi_{\bar\gamma}(0)$, \mbox{$0<h\le\bar h$},  
and $h\|v\|\le\bar\gamma$.  (Indeed, if 
$(v,\lambda)\in\Xi_{\bar\gamma}(0)$, \mbox{$0<h\le\bar h$},  
and $h\|v\|\le\bar\gamma$, then 
$(v,\lambda)=(\bar v,\bar\lambda)+(v',\lambda')$, with 
$(\bar v,\bar\lambda)\in\Phi_{\bar\gamma,\hat\Sigma}(0)$, 
$v'\in\R^n$, $\|v'\|\le\bar\gamma$, $\lambda'\in\R$, 
and $\lambda'\ge 0$. Then we can apply (\ref{traf}), with 
$\bar v$, $\bar\lambda$ in the roles of $v$, $\lambda$, and conclude that
$V(h(\bar v+v'))+h\bar\lambda>h\bar\gamma$, since 
$h\|\bar v+v'\|\le\bar\gamma$. Therefore $V(hv)+h\bar\lambda>h\bar\gamma$,
and then {\it a fortiori}  $V(hv)+h\lambda>h\bar\gamma$, since 
$\lambda=\bar\lambda+\lambda'$ and $\lambda'\ge 0$.)
Assume that $(v,\lambda)\in\Psi_{\bar\gamma}(0)$. Then we can write 
$(v,\lambda)=\alpha(v',\lambda')+(1-\alpha)(0,\ell)$, with
$(v',\lambda')\in\Xi_{\bar\gamma}(0)$, $\ell\ge\bar\gamma$,
and $0\le\alpha\le 1$. Let $h$ be such that
$0<h\le\bar\gamma$ and $h\|v\|\le\bar\gamma$. 
If $\alpha=0$, then $hv=0$ and
$\lambda=\ell\ge\bar\gamma$, so 
$V(hv)+h\lambda=V(0)+h\ell=h\ell\ge h\bar\gamma$.
If $\alpha>0$, define $\tilde h=\alpha h$. Then 
$0<\tilde h\le\bar\gamma$ and $\tilde h\|v'\|=\alpha h\|v'\|
=h\|v\|\le\bar\gamma$, since $v=\alpha v'$. Therefore 
$V(\tilde h v')+\tilde h\lambda'\ge\tilde h\bar\gamma$, since
$(v',\lambda')\in\Psi_{\bar\gamma}(0)$. On  the other hand,
$(1-\alpha)h\ell\ge(1-\alpha)h\bar\gamma$, since $\ell\ge\bar\gamma$.
Therefore 
\begin{eqnarray*}
V(hv)+h\lambda&=&V(h\alpha v')+h\alpha\lambda'+h(1-\alpha)\ell\\
&=&V(\tilde hv')+\tilde h\lambda'+h(1-\alpha)\ell\\
&\ge&\tilde h\bar\gamma+h(1-\alpha)\bar\gamma\\
&=&h\alpha\bar\gamma+h(1-\alpha)\bar\gamma\\
&=&h\bar\gamma\,,
\end{eqnarray*}
completing the proof of (d).

Statement (e) now follows easily: if $(0,\ell)\in\Psi_{\bar\gamma}(0)$, then
we can apply (d) taking $h=\bar\gamma$, $v=0$, $\lambda=\ell$, and conclude that
$\bar\gamma\ell\ge\bar\gamma^2$, so that $\ell\ge\bar\gamma$.

We now prove that $\Psi_{\delta}(0)$ is closed if $0<\delta\le\bar\gamma$.
Let $\{w_j\}_{j=1}^\infty$ be a sequence of points of
$\Psi_{\delta}(0)$ that converges to a limit
$w\in\R^{n+1}$. We will show that $w\in\Psi_{\delta}(0)$.
Let $w_j=\alpha_j(v_j,\lambda_j)+(1-\alpha_j)(0,\ell_j)$,
where $(v_j,\lambda_j)\in\Xi_\delta(0)$, $\ell_j\ge\bar\gamma$,
and $0\le\alpha_j\le 1$. Let $w=(v,\lambda)$. By 
\mbox{passing} to a
subsequence, if necessary, we may assume that the $\alpha_j$ converge to
a limit $\hat\alpha$. If the sequence $\{(v_j,\lambda_j)\}$ is bounded,
then we may pass to a subsequence and assume that 
$(\hat v,\hat\lambda)=\lim_{j\to\infty}(v_j,\lambda_j)$ exists.
Then $(\hat v,\hat\lambda)\in\Xi_\delta(0)$. Furthermore,
the limit $\lim_{j\to\infty}\alpha_j(v_j,\lambda_j)$ exists, so
$\mu=\lim_{j\to\infty}(1-\alpha_j)\ell_j$ exists as well, 
and $\mu\ge 0$. Clearly,
$w=\hat\alpha(\hat v,\hat\lambda)+(0,\mu)$. If $\hat\alpha=1$, then
$w=(\hat v,\hat\lambda)+(0,\mu)$, so $w\in\Xi_\delta(0)$---and {\it 
a \mbox{fortiori}} $w\in\Psi_\delta(0)$---because 
$(\hat v,\hat\lambda)\in\Xi_\delta(0)$ and $\mu\ge 0$.   If 
$\hat\alpha<1$, then 
$\ell=\lim_{j\to\infty}\ell_j$ exists and satisfies
$\ell=\frac{\mu}{1-\hat\alpha}$ and $\ell\ge\bar\gamma$. 
Then $w=\hat\alpha(\hat v,\hat\lambda)+(1-\hat\alpha)(0,\ell)$, 
and $(\hat v,\hat\lambda)$, $(0,\ell)$, belong to 
$\Xi_\delta(0)$ and $\Lambda$, respectively, so 
$w\in\Psi_\delta(0)$.  Now suppose that the sequence 
$\{(v_j,\lambda_j)\}$ is unbounded. Then (\ref{coer-ao}) implies that
$\{\lambda_j\}$ is unbounded. Since the $\lambda_j$ are bounded below,
we may assume, after passing to a subsequence, that 
$\lambda_j\to+\infty$ as  $j\to\infty$. On the other hand,
$\alpha_jv_j\to v$ and $\alpha_j\lambda_j+(1-\alpha_j)\ell_j
\to\lambda$. Since both sequences $\{\alpha_j\lambda_j\}$, 
$\{(1-\alpha_j)\ell_j\}$ are bounded \mbox{below},
we may pass to a subsequence and assume that the limits
$\mu=\lim_{j\to\infty}(1-\alpha_j)\ell_j$ and 
$\nu=\lim_{j\to\infty}\alpha_j\lambda_j$ exist.
But then 
\begin{eqnarray*}
\alpha_j\|v_j\|\le\alpha_j(\lambda_j+C)^{1/r}&=&
\alpha_j\lambda_j^{1/r}\Big(1+\frac{C}{\lambda_j}\Big)^{1/r}\\
&=&\alpha_j\lambda_j\lambda_j^{1/r-1}\Big(1+\frac{C}{\lambda_j}\Big)^{1/r}\\
&\asjgoes& 0\,,
\end{eqnarray*}
since $\lambda_j\asjgoes+\infty$, $\alpha_j\lambda_j\asjgoes\nu$,
and $r>1$.  Then $v=\lim_{j\to\infty}\alpha_jv_j=0$.
This implies, in particular, that $\bar\gamma\alpha_j\|v_j\|\le\bar\gamma$ 
if $j$ is large enough. So we can apply (d) with $h=\bar\gamma\alpha_j$
and $(v_j,\lambda_j)$ in the role of $(v,\lambda)$, and conclude that 
$V(\bar\gamma\alpha_jv_j)+\bar\gamma\alpha_j\lambda_j\ge\bar\gamma\alpha_j\bar\gamma$.
On the other hand, $(1-\alpha_j)\ell_j\ge(1-\alpha_j)\bar\gamma$, because
$\ell_j\ge\bar\gamma$. Therefore
$$
V(\bar\gamma\alpha_jv_j)+\bar\gamma\alpha_j\lambda_j+\bar\gamma(1-\alpha_j)\ell_j
\ge\bar\gamma\alpha_j\bar\gamma+\bar\gamma(1-\alpha_j)\bar\gamma
=\bar\gamma^2
$$ 
for large enough $j$. If we let $j\to\infty$, and use the facts that 
$\alpha_jv_j\asjgoes 0$, $V$ is continuous, and $V(0)=0$,
we find that $\bar\gamma\lambda=\lim_{j\to\infty}
(\bar\gamma\alpha_j\lambda_j+\bar\gamma(1-\alpha_j)\ell_j)
\ge\bar\gamma^2$, so $\lambda\ge\bar\gamma$. Therefore 
$w=(0,\lambda)$ and $\lambda\ge\bar\gamma$, so $w\in\Lambda$ 
and then $w\in\Psi_\delta(0)$.

We have now completed the proofs of (b), (c), (d) and (e). 
Let $w^*$ be the member of $\Psi_{\bar\gamma}(0)$ such that

$$
\|w^*\|\le\|w\|\qquad{\rm whenever}\qquad w\in\Psi_{\bar\gamma}(0)\,.
$$

\medbreak\noindent (The existence and uniqueness of $w^*$ 
follows from the fact that $\Psi_{\bar\gamma}(0)$ is closed and convex.)
Since \mbox{$(0,0)\notin\Psi_{\bar\gamma}(0)$}, it follows that 
$w^*\neq(0,0)$. Furthermore, the inequality 

\begin{equation}\label{ineqtriv}
\langle w^*,w\rangle\ge\|w^*\|^2
\end{equation}

\medbreak\noindent holds whenever $w\in\Psi_{\bar\gamma}(0)$, because 
if $w\in\Psi_{\bar\gamma}(0)$ then $\|w^*+t(w-w^*)\|^2\ge\|w^*\|^2$ 
whenever $0\le t\le 1$, since 
$w^*+t(w-w^*)\in\Psi_{\bar\gamma}(0)$ for such $t$, and then

$$
\|w^*\|^2+t^2\|w-w^*\|^2+2t\langle w^*,w-w^*\rangle\ge\|w^*\|^2\,,
$$

\medbreak\noindent so $t^2\|w-w^*\|^2+2t\langle w^*,w-w^*\rangle\ge 0$,
which implies $t\|w-w^*\|^2+2\langle w^*,w-w^*\rangle\ge 0$ if
$0<t\le 1$, and then $\langle w^*,w-w^*\rangle\ge 0$ (since we can let 
$t\downarrow 0$), so (\ref{ineqtriv}) holds. 

\medbreak We now let $Q$ be the set of all vectors $q\in\R^{n+1}$ such that
$q=hw$ for some $h,w$ such that $h\in[0,\infty[$ and
$w\in\Psi_{\bar\gamma}(0)$.
%, and write $\qint$ to denote the interior of $Q$.  
We will show that
\begin{itemize}
\item[(f)] {\em $Q$ is a closed convex cone such that 
$Q\backslash\{(0,0)\}\neq\emptyset$.}
\item[(g)] {\em $\langle w^*,q\rangle\ge 0$ for all $q\in Q$ and 
$\langle w^*,q\rangle>0$ for all $q\in Q\backslash\{(0,0)\}$.}
%\item[(h)] $(0,0)\notin\qint$.
\item[(h)] {\em there exist real constants $\kappa_-$, $\kappa_+$, 
such that \mbox{$0<\kappa_-\le\kappa_+$} and 
$\kappa_-\|q\|\le\langle w^*,q\rangle\le\kappa_+\|q\|$
\mbox{whenever} $q\in Q$.}
\end{itemize} 
Indeed, $Q$ is obviously a convex cone. The fact that
$Q\backslash\{(0,0)\}\neq\emptyset$ follows because 
$\Psi_{\bar\gamma}(0)\subseteq Q$,  $\Psi_{\bar\gamma}(0)\neq\emptyset$, 
and $(0,0)\notin\Psi_{\bar\gamma}(0)$. To show that $Q$ is closed, we 
pick a sequence $\{q_j\}_{j=1}^\infty$of points of $Q$ that converges to 
a limit $q\in\R^{n+1}$, and show that $q\in Q$. Write $q_j=h_jw_j$, 
\mbox{$h_j\ge 0$}, 
$w_j\in\Psi_{\bar\gamma}(0)$.  If $q=(0,0)$ then $q\in Q$, so we may assume 
that $q\neq (0,0)$ and that $q_j\neq (0,0)$ for all $j$. Then $h_j\neq 0$ as
well. If the sequence 
$\{w_j\}_{j=1}^\infty$ is bounded, then we may pass to a subsequence
and assume that the $w_j$ converge to a limit $w$, which must belong to
$\Psi_{\bar\gamma}(0)$ because $\Psi_{\bar\gamma}(0)$ is closed. In particular,
$w$ and the $w_j$ are $\neq(0,0)$. But then
$h_j=\frac{\|q_j\|}{\|w_j\|}\asjgoes\frac{\|q\|}{\|w\|}\equaldef h$. Therefore
$q=hw$, so $q\in Q$. Now suppose that  
the sequence $\{w_j\}_{j=1}^\infty$ is unbounded.  Write 
$w_j=(v_j,\lambda_j)$, and use (c) to conclude that
$\|v_j\|^r\le\lambda_j+C$ for all $j$. Then 
the sequence $\{\lambda_j\}_{j=1}^\infty$ is unbounded, and we may assume,
after passing to a subsequence, that $\lambda_j\asjgoes+\infty$.
Since $q=\lim_{j\to\infty}(h_jv_j,h_j\lambda_j)$, the sequence
$\{h_j\lambda_j\}_{j=1}^\infty$ converges to a finite limit $\mu$,
so $h_j\asjgoes 0$. Then 
$h_j\|v_j\|\le h_j(\lambda_j+C)^{1/r}=
h_j\lambda_j\lambda_j^{\frac{1}{r}-1}\Big(1+\frac{C}{\lambda_j}\Big)^{1/r}\asjgoes 0$.
So $h_jv_j\asjgoes 0$. Therefore $q=(0,\mu)$, so 
$q=h(0,\bar\gamma)$, where $h=\frac{\mu}{\bar\gamma}$. 
Since $\mu\ge 0$ and $(0,\bar\gamma)\in\Psi_{\bar\gamma}(0)$, 
it is now clear that $q\in Q$. This completes the proof of (f).

The fact that $\langle w^*,q\rangle\ge 0$ for all $q\in Q$ follows 
\mbox{trivially}
from the definition of $Q$, because if $q\in Q$ then 
\mbox{$q=hw$} 
for some $w\in\Psi_{\bar\gamma}(0)$ and some 
\mbox{nonnegative} $h$, so
$\langle w^*,q\rangle=h\langle w^*,w\rangle\ge h\|w^*\|^2\ge 0$.
\mbox{Furthermore}, if $q\neq(0,0)$ then $h\neq 0$, so $h>0$,
and then \mbox{$\langle w^*,q\rangle\ge h\|w^*\|^2>0$}, since
$w^*\neq 0$.  This proves (g). 

Let $K=\{q\in Q:\|q\|=1\}$. Then $K$ is 
\mbox{compact}, so the continuous
function $K\ni q\mapsto \langle w^*,q\rangle\in\rr$ \mbox{attains} a
minimum value $\kappa_-$ and a maximum value $\kappa_+$ on
$K$. Clearly, $\kappa_->0$, because $\langle w^*,q\rangle>0$ for all
$q\in K$.  Furthermore, $\kappa_-\|q\|\le\langle
w^*,q\rangle\le\kappa_+\|q\|$ for all $q\in Q$, because the
inequalities hold when $\|q\|=1$ and \mbox{involve} functions of $q$
that are positively homogeneous of \mbox{degree $1$}. This proves (h).

\medbreak Next, we define a function $\sigma:Q\mapsto\R$ by letting $\sigma(q)$ be,
if $q\in Q$, the largest $h\in\R$ such that $q=hw$ for some 
$w\in\Psi_{\bar\gamma}(0)$. (The existence of such a largest
$h$ is trivial if $q=0$, for in that case the only possible value of
$h$ is $0$, since $(0,0)\notin\Psi_{\bar\gamma}(0)$. If $q\in Q$ and 
$q\neq 0$, let $H=\{h\in\R:h>0,\,h^{-1}q\in\Psi_{\bar\gamma}(0)\}$. Then 
$H$ must be bounded, for otherwise $(0,0)$ would be a limit of
points of $\Psi_{\bar\gamma}(0)$, and then 
$(0,0)$ would have to belong to $\Psi_{\bar\gamma}(0)$.
If $h=\sup\,H$, then the fact that $\Psi_{\bar\gamma}(0)$ is closed
implies that $h\in H$, so $\sigma(q)$ exists and is equal to $h$.)
We prove the following properties of $\sigma$.

\begin{itemize}
\item[(i)] {\em $\sigma$ is strictly positive on $Q\backslash\{(0,0)\}$.}

\vspace{-1mm}
\item[(j)] {\em $\sigma$ is positively homogeneous of degree $1$ (that is, 
$\sigma(rq)=r\sigma(q)$ whenever $q\in Q$ and $r\ge 0$).} 

\vspace{-1mm}
\item[(k)] {\em There exists a constant $\kappa\in\R$ such that
\mbox{$\sigma(q)\le\kappa\|q\|$} whenever $q\in Q$.}
\end{itemize} 


Statements (i) and (j) are immediate consequences of the definition
of $\sigma$. To prove (k), we assume it is not true, and 
find a sequence $\{q_m\}_{m=1}^{\infty}$ of points of $Q$ such that
\mbox{$\sigma(q_m)> m\|q_m\|$} for all $m$. We then write 
\mbox{$q_m=\sigma(q_m)w_m$}, with $w_m\in\Psi_{\bar\gamma}(0)$, and use (j)
to \mbox{conclude} that $\sigma(w_m)=1$ and 
\mbox{$\sigma(w_m)> m\|w_m\|$} for all $m$. Then $\|w_m\|<\frac{1}{m}$,
so $w_m\to(0,0)$ as $m\to\infty$.  Since $w_m\in\Psi_{\bar\gamma}(0)$,
and $\Psi_{\bar\gamma}(0)$ is closed, we conclude that 
$(0,0)\in\Psi_{\bar\gamma}(0)$, contradicting (e). This completes the proof
of (k). 

\medbreak Next, we define 
\begin{eqnarray*}
\gamma^*&\equaldef&\frac{\bar\gamma}{\max(1,\kappa)}\\
Q^*&\equaldef&\Big\{q\in Q:\|q\|\le\gamma^*\Big\}\,.
\end{eqnarray*}

\medbreak\noindent Then $Q^*$ is a convex, compact subset of
$\rr^{n+1}$ such that $(0,0)\in Q^*$ but 
$Q^*$ contains at least one point other than $(0,0)$ (because of (f)). In 
addition, if $q=(x,x_0)\in Q^*$, then $\|x\|\le\gamma^*\le\bar\gamma$, so 
(\ref{traf-uxux}) tells us that $x\in\Omega$, and then 
$q\in\Omega\times\rr$. Hence $Q^*\subseteq\Omega\times\rr$. 

We then define a function  $W:\Omega\times\R\mapsto\R$ by \mbox{letting}
$W(x,x_0)=V(x)+x_0$ for $(x,x_0)\in\Omega\times\R$, and observe that
$W(q)$ is defined whenever $q\in Q^*$, because 
\mbox{$Q^*\subseteq\Omega\times\rr$}. 

We then claim that

\begin{equation}\label{traf-i}
W(q)\ge\bar\gamma\sigma(q)\quad
{\rm whenever}\,\,\,q\in Q^*\,.
\end{equation} 

\medbreak\noindent To prove (\ref{traf-i}), we first observe that the 
inequality is clearly true if $q=0$. Let us pick $q\in Q^*\backslash\{0\}$ 
and write $q=hw$, where $h=\sigma(q)$ and
\mbox{$w=(v,\lambda)\in \Psi_{\bar\gamma}(0)$.}
Then $0<h$, since $q\neq 0$, and $h\le\bar\gamma$, 
\mbox{because} \mbox{$\sigma(q)\le\kappa\|q\|\le\kappa\gamma^*\le\bar\gamma$.}
Furthermore, $\|q\|\le\bar\gamma$ (because $\gamma^*\le\bar\gamma$),
and $q=(hv,h\lambda)$, so 
$h\|v\|\le\bar\gamma$. It then follows from (\ref{traf}) that
$W(q)=V(hv)+h\lambda\ge h\bar\gamma=\bar\gamma\sigma(q)$,
completing the proof of (\ref{traf-i}).

\medbreak It follows from (\ref{traf-i}) and (i) that 

\begin{equation}\label{traf-ixx}
W(q)>0\quad{\rm whenever}\,\,\,q\in Q^*\backslash\{(0,0)\}\,.
\end{equation} 

\medbreak Now fix a number $\rho$ such that $0<\rho\le\kappa_-\gamma^*$, 
and define 

\begin{equation}\label{traf-izz}
Q^{\#}=\{q\in Q^*:\langle w^*,q\rangle\ge\rho\}\,.
\end{equation}

\medbreak\noindent If $q$  is any member
of $Q$ such that $\|q\|=\gamma^*$, then \mbox{$q\in Q^{*}$}, and in addition 
\mbox{$\langle w^*,q\rangle\ge\kappa_-\|q\|=\kappa_-\gamma^*\ge\rho$}, so 
\mbox{$q\in Q^{\#}$}. Hence $Q^{\#}$ is nonempty, and it is clear that 
$Q^{\#}$ is compact and convex.  If \mbox{$q\in Q^{\#}$}, then 
\mbox{$q\in Q^{*}\backslash\{(0,0)\}$}, so $W(q)>0$. It follows that 
$\bar\mu>0$, if we let \mbox{$\bar\mu=\min\{W(q):q\in
Q^{\#}\}$}. Since $W(0,0)=0$, we may pick a $\mu$ such that
\mbox{$0<\mu<\bar\mu$}, and use the Clarke-Ledyaev mean value theorem
(cf. \mbox{\cite{Cla-Le-a,Cla-Le-b,clarkebooknew}})
to conclude that if $\beta$ is any positive number, and we use ${\cal
N}_\beta$ to denote the \mbox{$\beta$-neighborhood} of the set
\mbox{${\cal N}_0\equaldef\{hq:q\in Q^{\#},0\le h\le 1\}$}, then there
exists a \mbox{subdifferential} $(\pi,\pi_0)$ of $W$ at some point $q^{\#}$
belonging to ${\cal N}_\beta$ such that


\begin{equation}\label{traf-iuu}
\langle (\pi,\pi_0),q\rangle>\mu\quad{\rm for\,\,all}\quad
q\in Q^{\#}\,.
\end{equation}

\medbreak\noindent Write $q^{\#}=(x^{\#},x_0^{\#})$.
Then, if we write $q=(x,x_0)$ for $q$ near $q^{\#}$ , we have 

$$
\liminf_{q\to q^{\#}}
\frac{W(q)\!-\!W(q^{\#})\!-\!\pi\cdot(x\!-\!x^{\#})
\!-\!\pi_0(x_0\!-\!x_0^{\#})}{\|x\!-\!x^{\#}\|\!+\!|x_0-x_0^{\#}|}\!\ge\!0.
$$

\medbreak\noindent Taking $q=(x,x_0^{\#})$, this implies 

$$
\lim\inf_{x\to x^{\#}}
\frac{V(x)-V(x^{\#})-\pi\cdot(x-x^{\#})}{\|x-x^{\#}\|}\ge 0\,,
$$

\medbreak\noindent so $\pi$ is a subdifferential of $V$ at $x^{\#}$. 

Taking  $q=(x^{\#},x_0)$, we get

$$
\lim\inf_{x_0\to x_0^{\#}}\frac{x_0-x_0^{\#}-\pi_0(x_0-x_0^{\#})}
{|x_0-x_0^{\#}|}\ge 0\,,
$$

\medbreak\noindent so $\pi_0=1$.

Now, if $u\in U$, and we let $w=(f(x^{\#},u),L(x^{\#},u))$, then 
$w\in\Phi_{\bar\gamma,\hat\Sigma}(0)$, so 
$w\in Q$. Let $q=\frac{\gamma^*w}{\|w\|}$. Then 
$\|q\|=\gamma^*$, so $q\in Q^*$. Furthermore, 
\mbox{$\langle w^*,q\rangle\ge\kappa_-\|q\|=\kappa_-\gamma^*\ge\rho$}, 
so $q\in Q^{\#}$. Therefore

\begin{equation}\label{traf-iuuu}
\langle (\pi,\pi_0),q\rangle>\mu\,,
\end{equation}

\medbreak\noindent that is, 

\begin{equation}\label{traf-iuuuaa}
\frac{\gamma^*}{\|w\|}\Big(\langle\pi,f(x^{\#},u)\rangle+L(x^{\#},u)\Big)
>\mu\,.
\end{equation}

\medbreak\noindent Therefore

\begin{equation}\label{traf-iuux}
\langle\pi,f(x^{\#},u)\rangle+L(x^{\#},u)
>\frac{\|w\|\mu}{\gamma^*}\ge\frac{\|w^*\|\mu}{\gamma^*}\,, 
\end{equation}

\medbreak\noindent so

\begin{equation}\label{traf-iuuxax}
-\langle\pi,f(x^{\#},u)\rangle-L(x^{\#},u)
<-\frac{\|w^*\|\mu}{\gamma^*}\,.
\end{equation}

\medbreak\noindent Since (\ref{traf-iuuxax}) is true for every $u\in U$,
we can conclude that 

\begin{equation}\label{traf-iuuxaxai}
\sup\Big\{-\langle\pi,f(x^{\#},u)\rangle-L(x^{\#},u):u\in U\Big\}
\le\frac{\|w^*\|\mu}{\gamma^*}<0\,.
\end{equation}

\medbreak\noindent But $\pi$ is a subdifferential of $V$
at $x^{\#}$, and then (\ref{traf-iuuxaxai}) contradicts the fact that
$V$ is a solution of (\ref{loweq}) on $\Omega$ in the viscosity sense.
This contradiction establishes (3.ii) and completes our proof.
\end{proof}	

\ssection{The compactness theorem.}

\noindent If \mbox{$\hat\Sigma=(\Omega,U,f,L)$} is an  augmented control
system, 
and $\eps$ is a positive \mbox{number}, an {\em $\eps$-approximate
\mbox{augmented} \mbox{trajectory} of $\hat\Sigma$} is a locally absolutely 
\mbox{continuous} curve 
\mbox{$I\ni t\mapsto\Xi(t)=(\xi(t),\xi_0(t))\in\R^{n+1}$}
having the property that there exists a measurable function 
\mbox{$I\ni t\mapsto v(t)\in\R^{n}$} such that 

\begin{itemize}
\item[(1)] $\|v(t)\|\le\eps$ for almost all $t\in I$, 

\item[(2)] $\dot\Xi(t)-(v(t),0)\in\fsh(\xi(t),U)$ 
for almost all $t\in I$.
\end{itemize}

\begin{remark}
Roughly speaking, an $\eps$-approximate
\mbox{augmented} \mbox{trajectory} of $\hat\Sigma$ is an augmented
\mbox{trajectory} of the ``$\eps$-extended system'' $\hat\Sigma^{\eps}
=(\Omega,U^{\eps},f^{\eps},L^{\eps})$ whose control space 
$U^{\eps}$ is the Cartesian product 
\mbox{$U\times\{v\in\R^{n}:\|v\|\le\eps\}$},
and whose dynamics $f^{\eps}$ and Lagrangian $L^{\eps}$ 
are given by 

$$
f^{\eps}(x,u,v)=f(x,u)+v\,,\quad L^{\eps}(x,u,v)=L(x,u)\,. 
$$

\medbreak\noindent More precisely, a curve 
\mbox{$\Xi=(\xi,\xi_0):I\mapsto\R^{n+1}$ is an} \mbox{$\eps$-approximate}
\mbox{augmented} \mbox{trajectory} of $\hat\Sigma$ if and only if 
$\,\Xi$ is locally absolutely \mbox{continuous} and there exist functions 
\mbox{$I\ni t\mapsto\eta(t)\in U$}, 
\mbox{$I\ni t\mapsto v(t)\in\{v\in\R^{n}:\|v\|\le\eps\}$}
such that $v$ is measurable and $\dot\Xi(t)\in
F_{\hat\Sigma^{\eps}}(\xi(t),U^{\eps})$ for almost every $t\in I$.
The definition of an augmented trajectory of $\hat\Sigma^{\eps}$ is
exactly the same, except that in that case the requirement
that $v$ be measurable is omitted.\proofend
\end{remark}

\begin{theorem}\label{compacttheo}
Let $\hat\Sigma=(\Omega,U,f,L)$ be an $n$-dimensional locally
uniformly continuous, locally coercive \mbox{augmented} control system such
that $\fsh(x,U)$ is closed and convex for every $x\in\Omega$.  Let $K$
be a compact subset of $\Omega$, let $T$ be a positive time, and let
$k\in\R$. Let $\{\eps_j\}_{j=1}^\infty$ be a 
\mbox{sequence} of positive
numbers such that $\eps_j\to 0$ as $j\to\infty$, and let
$\Xi^j=(\xi^j,\xi_{0}^j)$ be, for each $j$, an \mbox{$\eps_j$-approximate}
\mbox{augmented} trajectory of $\hat\Sigma$, defined on the interval $[0,T]$,
such that $\xi^{j}(t)\in K$ for all $t\in[0,T]$, $\xi_{0}^j(0)=0$,  
and $\xi_{0}^j(T)\le k$. Then there exist a \mbox{subsequence}
$\{\Xi^{j(k)}\}_{k=1}^\infty$ of the  \mbox{sequence}
$\{\Xi^j\}_{j=1}^\infty$ and an augmented
trajectory $\Xi^\infty=(\xi^\infty,\xi_{0}^\infty)$ of 
$\hat\Sigma=(\Omega,U,f,L)$ such that
\begin{itemize}
\item[(i)] $\xi_0^\infty(0)=0$, 
\item[(ii)] the sequence $\{\xi^{j(k)}\}_{k=1}^\infty$
converges uniformly to $\xi^\infty$, and 
\item[(iii)] 
$\liminf_{k\to\infty}\inf\big\{\Delta^{j(k),\infty}(t,s)\!:\!
0\!\le\!s\!\le\!t\!\le\!T\big\}\!\ge\!0$,
where, if $j,j'\in\nn\cup\{+\infty\}$, we define
\end{itemize}

\vspace{-2.5mm}
\begin{equation}\label{defodel}
\Delta^{j,j'}(t,s)\equaldef(\xi_0^j(t)-\xi_0^j(s))-
(\xi_0^{j'}(t)-\xi_0^{j'}(s))\,.
\end{equation}
\end{theorem}

\bigbreak \vspace{1mm}
\begin{proof}
Pick constants $r$, $C$, such that $r>1$, 
\mbox{$C>0$},
and $\|f(x,u)\|^r\le L(x,u)+C$ whenever $x\in K$, $u\in U$,
and a function $\omega:\,]\,0,+\infty\,[\,\mapsto[0,+\infty]$ 
such that $\lim_{s\downarrow 0}\omega(s)$ and 
$\|\fsh(x,u)-\fsh(x',u)\|\le\omega(s)$ whenever
$x,x'\in K$ and $\|x-x'\|\le s$. Choose,
for each $j$, a measurable function \mbox{$v^j:[0,T]\mapsto\R^n$} and a
function $\eta^j:[0,T]\mapsto U$ such that $\|v(t)\|\le\eps_j$,
$\dot\xi^j(t)=f(\xi^j(t),\eta^j(t))+v(t)$, and
$\dot\xi_0^j(t)=L(\xi^j(s),\eta^j(s))$ for almost all $t\in[0,T]$.

Let $C'=2^r(C+\eps^r)$. 
Then for each $j$ the coerciveness condition implies the inequality

$$
\|\dot\xi^j(t)\|^{r}\le 2^rL(\xi^j(t),\eta^j(t)) +C'\,\,
{\rm for\,\,almost\,\,all}\,\,t\,,
$$

\medbreak\noindent because 
\begin{eqnarray*}
\|\dot\xi^j(t)\|^{r}&=&\|f(\xi^j(t),\eta^j(t))+v(t)\|^r\\
&\le&\big(\|f(\xi^j(t),\eta^j(t))\|+\|v(t)\|\big)^r\\
&\le&2^r\|f(\xi^j(t),\eta^j(t))\|^r+2^r\|v(t)\|^r\\
&\le&2^r(L(\xi^j(t),\eta^j(t))+C)+2^r\eps^r\\
&=&2^rL(\xi^j(t),\eta^j(t))+C'\,.
\end{eqnarray*}

\medbreak\noindent from which it follows that
$\int_0^T\|\dot\xi^j(t)\|^{r}dt\le 2^rk+C'T$. Then the sequence
$\{\dot\xi^j(t)\}_{j=1}^\infty$ is uniformly bounded in $L^{r}$, so
we may assume, after passing to a subsequence if necessary,
that the weak $L^{r}$-limit $\zeta=\,$w-$\lim_{j\to\infty}\dot\xi^j$
\mbox{exists}.  Then the $\xi^j$ converge uniformly as $j\to\infty$
to a limit $\xi^\infty$ such that 
$\xi^\infty(t)-\xi^\infty(s)=\int_s^t\zeta(r)\,dr$ for all
$s,t\in[0,T]$.  After passing to a subsequence once more, if necessary,
we assume that

\begin{equation}\label{twoj}
\|\xi^j(t)-\xi^\infty(t)\|\le 2^{-j}\,\,{\rm for\,\,all}\,\,j\in\nn\,,
\,\,t\in[0,T]\,.
\end{equation}

\medbreak\noindent
Let $\theta^j(t)=\dot\xi_0^j(t)+C$.  Then the $\theta^j$ 
are nonnegative, because 
\mbox{$\dot\xi_0^j(t)=L(\xi^j(t),\eta^j(t))\ge-C$}.
Furthermore, the sequence $\{\theta^j\}_{j=1}^\infty$ 
is bounded in $L^1([0,T],\rr)$, \mbox{because}
\begin{eqnarray*}
\|\theta^j\|_{L^1}&=&\int_0^T\theta^j(t)\,dt\\
&=&\int_0^T(\dot\xi_0^j(t)+C)\,dt\\
&\le&k+CT\,.
\end{eqnarray*}
The space $L^1([0,T],\rr)$ can be embedded in the usual way in
$C^0([0,T],\rr)^\dagger$ (the dual of $C^0([0,T],\rr)$), which is the
space of finite Borel measures on $[0,T]$, by \mbox{means} of 
the map $\psi\mapsto\mu_\psi$ that assigns to
each function \mbox{$\psi\in L^1([0,T],\rr)$} the Borel measure $\mu_\psi$
such that 
$$
\mu_\psi(\varphi)= \int_0^T\psi(t)\varphi(t)dt\,\,{\rm for\,\,every}
\,\,\varphi\in C^0([0,T],\rr)\,.
$$

Then we may assume, after passing to a subsequence for a third time,
if \mbox{necessary}, that the weak* limit $\mu^\infty$ 
of the measures $\mu^j\equaldef\mu_{\theta^j}$ 
defined by the $\theta^j$ exists as $j\to\infty$.
The measure $\mu^\infty$ then has a decomposition
$$
\mu^\infty=\mu^{\infty,ac}+\mu^{\infty,at}+\mu^{\infty,sing}
$$
into the sum of an absolutely continuous part, an \mbox{atomic} part, and 
a singular part. Since the measure $\mu^\infty$ is 
\mbox{positive}, because 
it is a limit of positive measures, the three components
$\mu^{\infty,ac}$, $\mu^{\infty,at}$, $\mu^{\infty,sing}$ are 
positive as well.  Let $\theta^\infty$ be the Radon-Nikodym 
derivative of $\mu^{\infty}$, so $\theta^\infty$ is an integrable 
function on $[0,T]$ such that 
$$
\mu^{\infty,ac}(\varphi)=\int_0^T\theta^\infty(t)\varphi(t)\,dt\,\,
{\rm for}\,\,\varphi\in C^0([0,T],\rr)\,.
$$
Define 
\begin{equation}\label{defdef}
\xi_0^\infty(t)=-Ct+\int_0^t\theta^\infty(s)ds
\end{equation}
and then set $\Xi^\infty=(\xi^\infty,\xi_0^\infty)$. 

\medbreak We will show that 
$\Xi^\infty$ is an augmented \mbox{trajectory} of the system
$\hat\Sigma$. To see this, 
we observe first of \mbox{all} that by \mbox{construction}
the functions \mbox{$\xi^\infty:[0,T]\mapsto\rr^n$} and 
\mbox{$\xi_0^\infty:[0,T]\mapsto\rr$} are absolutely continuous,
and their
\mbox{derivatives} at $t$  are \mbox{equal} 
to $\zeta(t)$ and $\theta^\infty(t)$,
\mbox{respectively}, for all $t$ in a subset $G$ of $[0,T]$ 
such that $[0,T]\backslash G$ has \mbox{measure} zero. Let 
$A$ be the set of atoms of $\mu^{\infty, at}$, and let 
$B$ be a subset of $[0,T]$ of Lebesgue measure zero such that
\mbox{$\mu^{\infty,sing}([0,T]\backslash B)=0$}. Then the set
\mbox{$G'=G\backslash(A\cup B\cup\{T\})$} has measure $T$ and 
\mbox{$\mu^{\infty,at}(G')=\mu^{\infty,sing}(G')=0$}. Let
$G''$ be the set of points of density of $G'$ that are Lebesgue points
of $\zeta$ and $\theta^\infty$,  so 
$G''$ has measure $T$ as well. (\mbox{Recall} that a {\em Lebesgue
point} of a scalar- or vector-valued \mbox{integrable}
function $\sigma$ defined on
an interval $[a,b]$ is a point \mbox{$t\in\,]\,a,b\,[\,$} such that 
\mbox{$\lim_{h\downarrow 0}\frac{1}{h}
\int_{t-h}^{t+h}\|\sigma(s)-\sigma(t)\|\,ds=0$}.)
Let $t\in G''$,
and fix an $h$ such that \mbox{$0<h<T-t$}. Let 
\mbox{$E_{t,h}=[t,t+h]\cap G'$}, so the Lebesgue
measure $|E_{t,h}|$ of $E_{t,h}$ satisfies
$\lim_{h\downarrow 0}h^{-1}|E_{t,h}|=1$.
\mbox{Using} the \mbox{facts} that the Borel measure
$\mu^{\infty,at}+\mu^{\infty,sing}$ is \mbox{regular}
and $(\mu^{\infty,at}+\mu^{\infty,sing})(E_{t,h})=0$,
we can find a \mbox{relatively}  open subset $\tilde U_{t,h}$ 
of $[0,T]$ such that \mbox{$E_{t,h}\subseteq\tilde U_{t,h}$}
and 
\mbox{$(\mu^{\infty,at}+\mu^{\infty,sing})(\tilde U_{t,h})\le h^2$}.
We then let \mbox{$U_{t,h}=\tilde U_{t,h}\cap\,]\,t,t+h\,[\,$}, so $U_{t,h}$ 
is an open subset of $\R$, $U_{t,h}\subseteq\,]\,t,t+h\,[\,$, and 
\mbox{$(\mu^{\infty,at}+\mu^{\infty,sing})(U_{t,h})\le h^2$}.
Using the regularity of Lebesgue measure we can find a compact subset
$K_{t,h}$ of $E_{t,h}\backslash\{t,t+h\}$ such that 
\mbox{$|K_{t,h}|\ge|E_{t,h}|-h^2$}. Then of course 
\mbox{$\lim_{h\downarrow 0}h^{-1}|K_{t,h}|=1$}, and 
$K_{t,h}\subseteq U_{t,h}$ for each $h$.
Let \mbox{$\tilde\varphi_{t,h}(s)={\rm dist}(s,\R\backslash U_{t,h})$}, so
$\tilde\varphi_{t,h}:\R\mapsto\R$ is \mbox{continuous}, 
$\tilde\varphi_{t,h}(s)=0$ whenever $s\notin U_{t,h}$, 
and $\tilde\varphi_{t,h}(s)>0$ \mbox{whenever} $s\in K_{t,h}$. If we let
\begin{eqnarray*}
\beta_{t,h}&=&\min\{\tilde\varphi_{t,h}(s):s\in K_{t,h}\}\,,\\
\hat\varphi_{t,h}(s)&=&\min(\tilde\varphi_{t,h}(s),\beta_{t,h})\,,\\
\varphi_{t,h}(s)&=&\beta_{t,h}^{-1}\hat\varphi_{t,h}(s)\,,
\end{eqnarray*}
then $\varphi_{t,h}$  is  a continuous real-valued function
on $\R$ such that $0\le\varphi_{t,h}(s)\le 1$ for all $s$,
$\varphi_{t,h}(s)=1$ for all \mbox{$s\in K_{t,h}$}, and
$\varphi_{t,h}(s)=0$ for all $s\in\R\backslash U_{t,h}$.
In \mbox{particular}, $\varphi_{t,h}(s)=0$ whenever 
$s\notin\,]\,t,t+h\,[\,$.


Let $a_{t,h}=\int_t^{t+h}\varphi_{t,h}(s)\,ds=
\int_{-\infty}^{+\infty}\varphi_{t,h}(s)\,ds$. Then
$$
|E_{t,h}|-h^2\le a_{t,h}\le h\,,
$$
from which it follows that
$$
\lim_{h\downarrow 0}\frac{a_{t,h}}{h}=1\,.
$$
Let $\psi_{t,h}=a_{t,h}^{-1}\varphi_{t,h}$. Then
$$
\int_t^{t+h}\psi_{t,h}(s)\,ds=1\,.
$$
If $h>0$, write 
$$
\delta_j(h)=2^{-j}+h^{1/\rho}(2^r|k|+C'T)^{1/r}\,.
$$ 
Then, if $s\in[t,t+h]$, we have
\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!
\!\!\!\!\|\xi^j(s)-\xi^\infty(t)\|\\
&\le&\|\xi^j(t)-\xi^\infty(t)\|+\|\xi^j(s)-\xi^j(t)\|\\
&\le&2^{-j}+\int_t^s\|\dot\xi^j(\tau)\|\,dt\\
&\le&2^{-j}+
(s-t)^{1/\rho}\Big(\int_t^s\|\dot\xi^j(\tau)\|^r\,dt\Big)^{1/r}\\
&\le&2^{-j}+
h^{1/\rho}\Big(\int_0^T\|\dot\xi^j(\tau)\|^r\,dt\Big)^{1/r}\\
&\le&2^{-j}+h^{1/\rho}\Big(\int_0^T(2^rL(\xi^j(\tau),\eta^j(\tau))
+C')\,d\tau\Big)^{1/r}\\
&\le&2^{-j}+h^{1/\rho}(2^r|k|+C'T)^{1/r}\\
&=&\delta_j(h)\,.
\end{eqnarray*}
For almost all $s\in[t,t+h]$, the derivative 
$\dot\Xi^j(s)$ \mbox{exists} and is equal to 
$\fsh(\xi^j(s),\eta^j(s))+(v^j(s),0)$. Hence
$$
\|\,\dot\Xi^j(s)-\fsh(\xi^\infty(t),\eta^j(s))\|
\le\omega(\delta_j(h))+\eps_j\,,
$$ 
from which it follows that
$$
{\rm dist}(\dot\Xi^j(s),\fsh(\xi^\infty(t),U)
\le\omega(\delta_j(h))+\eps_j\,.
$$ 
Therefore the average 
$$
A_{t,h}^j=\int_t^{t+h}\psi_{t,h}(s)\dot\Xi^j(s)\,ds
=\int_{0}^{T}\psi_{t,h}(s)\dot\Xi^j(s)\,ds
$$ 
also satisfies 
\begin{equation}\label{avesat}
{\rm dist}(A_{t,h}^j,\fsh(\xi^\infty(t),U)
\le\omega(\delta_j(h))+\eps_j\,,
\end{equation}
because $\fsh(\xi^\infty(t),U)$ is closed and convex.
As $j\to\infty$, the vector functions $\dot\xi^j$ converge weakly in $L^r$ 
to $\zeta$, so 
\begin{eqnarray*}
\int_t^{t+h}\psi_{t,h}(s)\dot\xi^j(s)\,ds
&\to&\int_t^{t+h}\psi_{t,h}(s)\zeta(s)\,ds\\
&=&\int_t^{t+h}\psi_{t,h}(s)\dot\xi^\infty(s)\,ds
\end{eqnarray*}
as $j\to\infty$, $j\in J$. The integral 
$\int_t^{t+h}\psi_{t,h}(s)\dot\xi^\infty(s)\,ds$ satisfies
\begin{eqnarray*}
\int_{t}^{t+h}\psi_{t,h}(s)\dot\xi^\infty(s)\,ds&=&
\int_{t}^{t+h}\psi_{t,h}(s)\dot\xi^\infty(t)\,ds+{\cal E}_{t,h}\\
&=&\dot\xi^\infty(t)+{\cal E}_{t,h}\\
\end{eqnarray*}
where 
$$
{\cal E}_{t,h}=\int_{t}^{t+h}\psi_{t,h}(s)(\dot\xi^\infty(s)-
\dot\xi^\infty(t))\,ds\,,
$$
and we have used the fact that $\int_{t}^{t+h}\psi_{t,h}(s)\,ds=1$.

The error term ${\cal E}_{t,h}$ satisfies
\begin{eqnarray*}
|{\cal E}_{t,h}|&\le&\max\big\{\psi_{t,h}(s):s\in[0,T]\big\}\\
&\,&\,\,\,\times 
\int_{t}^{t+h}\|\dot\xi^\infty(s)-\dot\xi^\infty(t)\|\,ds\\
&=&ha_{t,h}^{-1}\hat{\cal E}_{t,h}\\
&=&\alpha_{t,h}\hat{\cal E}_{t,h}\,,
\end{eqnarray*}
where $\alpha_{t,h}=ha_{t,h}^{-1}$, so $\alpha_{t,h}\to 1$ as
$h\downarrow 0$, and
$$
\hat{\cal E}_{t,h}=\frac{1}{h}\int_{t}^{t+h}\|\dot\xi^\infty(s)-
\dot\xi^\infty(t)\|\,ds\,.
$$
Then 
\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\lim\sup_{j\to\infty,j\in J}
\Big\|\int_t^{t+h}\psi_{t,h}(s)\dot\xi^j(s)\,ds
-\dot\xi^\infty(t)\Big\|\nonumber\\
&\le&\alpha_{t,h}\hat{\cal E}_{t,h}\,.\label{gosto-x}
\end{eqnarray}
%\lim\sup_{h\downarrow 0}\alpha_{t,h}(h+\hat E_{t,h})\,.

To analyze the behavior of the integrals 
$$
I_{t,h}^j\equaldef\int_t^{t+h}\psi_{t,h}(s)\dot\xi_0^j(s)\,ds\,,
$$
we write $\dot\xi_0^j(s)=\theta^j(s)-C$, so 
\begin{eqnarray*}
I_{t,h}^j&=&-Ch+\int_t^{t+h}\psi_{t,h}(s)\theta^j(s)\,ds\\
&=&-Ch+\int_0^{T}\psi_{t,h}(s)\theta^j(s)\,ds\\
&=&-Ch+\int_{[0,T]}\psi_{t,h}d\mu^j\,.
\end{eqnarray*} 
Hence 
\begin{equation}\label{gosto}
I_{t,h}^j\to-Ch+\int_{[0,T]}\psi_{t,h}(s)\,d\mu^\infty(s)
\,\,{\rm as}\,\,j\to\infty\,.
\end{equation}
Write 
\begin{eqnarray*}
\int_{[0,T]}\psi_{t,h}(s)\,d\mu^\infty(s)&=&
\int_0^T\psi_{t,h}(s)\theta^\infty(s)\,ds\\
&\,&\,\,\,+\int_{[0,T]}\psi_{t,h}(s)\,d\hat\mu^\infty(s)\,,
\end{eqnarray*}
where $\hat\mu^\infty=\mu^{\infty,at}+\mu^{\infty,sing}$. Since
$\dot\xi_0^\infty(s)=\theta^j(s)-C$, (\ref{gosto}) implies
\begin{eqnarray}
I_{t,h}^j&\to&\int_{[0,T]}\psi_{t,h}(s)\dot\xi_0^\infty(s)\,ds\nonumber\\
&\,&\,\,\,\,+\int_{[0,T]}\psi_{t,h}(s)\,d\hat\mu^\infty(s)
\,\,{\rm as}\,\,j\to\infty\,.\label{gosto-a}
\end{eqnarray}
The integral $\int_{[0,T]}\psi_{t,h}(s)\,d\hat\mu^\infty(s)$
that occcurs in (\ref{gosto-a}) 
is a \mbox{nonnegative} \mbox{number}, and is bounded above
by \mbox{$\max\{\psi_{t,h}(s):s\in[0,T]\}$} times
$\hat\mu^\infty(U_{t,h})$, since $\psi_{t,h}$ \mbox{vanishes} outside
$U_{t,h}$. Therefore
\begin{equation}\label{gosto-b}
0\le\int_{[0,T]}\psi_{t,h}(s)\,d\hat\mu^\infty(s)
\le h^{2}a_{t,h}^{-1}=h\alpha_{t,h}\,,
\end{equation}
On the other hand, the integral 
$\int_{[0,T]}\psi_{t,h}(s)\dot\xi_0^\infty(s)\,ds$ satisfies
\begin{eqnarray}
\int_{[0,T]}\psi_{t,h}(s)\dot\xi_0^\infty(s)\,ds&=&
\int_{t}^{t+h}\psi_{t,h}(s)\dot\xi_0^\infty(s)\,ds\nonumber\\
&=&\dot\xi_0^\infty(t))\,ds+E_{t,h}\nonumber\\
&=&\dot\xi_0^\infty(t)+E_{t,h}\,,\label{ontheot}
\end{eqnarray}
where 
$$
E_{t,h}=\int_{t}^{t+h}\psi_{t,h}(s)(\dot\xi_0^\infty(s)-
\dot\xi_0^\infty(t))\,ds\,,
$$
and we have used the fact that $\int_{t}^{t+h}\psi_{t,h}(s)\,ds=1$.

The error term $E_{t,h}$ satisfies
\begin{eqnarray*}
|E_{t,h}|&\le&\max\big\{\psi_{t,h}(s):s\in[0,T]\big\}\\
&\,&\,\,\,\times 
\int_{t}^{t+h}|\dot\xi_0^\infty(s)-\dot\xi_0^\infty(t)|\,ds\\
&=&ha_{t,h}^{-1}\hat E_{t,h}\\
&=&\alpha_{t,h}\hat E_{t,h}\,,
\end{eqnarray*}
where 
$$
\hat E_{t,h}=\frac{1}{h}\int_{t}^{t+h}|\dot\xi_0^\infty(s)-
\dot\xi_0^\infty(t)|\,ds\,.
$$


It follows from (\ref{gosto-a}), (\ref{gosto-b}), (\ref{ontheot}), 
and the bound
\mbox{$|E_{t,h}|\le\alpha_{t,h}\hat E_{t,h}$}, that
\begin{equation}\label{gosto-y}
\lim\sup_{j\to\infty,j\in J}\Big|I_{t,h}^j-\dot\xi_0^\infty(t)\Big|
\le\alpha_{t,h}(h+\hat E_{t,h})\,.
\end{equation}
If we now combine (\ref{gosto-x}) and (\ref{gosto-y}), we find that
\begin{equation}
\limsup_{j\to\infty,j\in J}\Big\|A_{t,h}^j\!-\!\dot\Xi^\infty(t)\Big\|
\!\le\!\alpha_{t,h}(h\!+\!\hat E_{t,h}\!+\!{\cal E}_{t,h}).\label{gosto-z}
\end{equation}
Then (\ref{avesat}) implies that
\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
{\rm dist}(\dot\Xi^\infty(t),\fsh(\xi^\infty(t)))\\
&\,&\!\!\!\le\!\alpha_{t,h}(h\!+\!\hat E_{t,h}\!+\!{\cal E}_{t,h})\!+\!
\limsup_{j\to\infty}(\omega(\delta_j(h))\!+\!\eps_j)\,.
\end{eqnarray*}
Hence, given any $j_*$, we have 

\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!
{\rm dist}(\dot\Xi^\infty(t),\fsh(\xi^\infty(t)))\\
&\,&\!\!\!\le\!\alpha_{t,h}(h\!+\!\hat E_{t,h}\!+\!{\cal E}_{t,h})\!+\!
\sup\{\omega(\delta_j(h))\!+\!\eps_j:j\ge j_*\}\,.
\end{eqnarray*}

\medbreak\noindent Given any positive number $\beta$, we can find a positive 
$\gamma$ such that $\omega(s)<\beta$ whenever $0<s\le\gamma$, and
then find $j_*$, $h_*$ such that $\delta_j(h)\le\gamma$ whenever 
$j\ge j_*$ and $0<h\le h_*$, and $\eps_j<\beta$ whenever 
$j\ge j_*$. Then we can pick $h$ such that  $0<h\le h_*$, 
and $\alpha_{t,h}(h\!+\!\hat E_{t,h}\!+\!{\cal E}_{t,h})<\beta$.
Then 
$$
{\rm dist}(\dot\Xi^\infty(t),\fsh(\xi^\infty(t)))<3\beta\,.
$$
Since $\beta$ was arbitrary, we conclude that 
${\rm dist}(\dot\Xi^\infty(t),\fsh(\xi^\infty(t)))=0$, 
so $\dot\Xi^\infty(t)\in\fsh(\xi^\infty(t))$, because 
$\fsh(\xi^\infty(t))$ is closed. Since this is true 
for almost all $t\in[0,T]$, and $\Xi^\infty$ is absolutely
continuous, we have shown that $\Xi^\infty$ is an augmented
trajectory of $\hat\Sigma$.

By construction, the $\xi^j$ converge uniformly to $\xi^\infty$.
\mbox{Also}, it is clear from (\ref{defdef})  that $\xi_0^\infty(0)=0$.
To \mbox{conclude} our proof, we have to show that

\begin{equation}\label{conclu}
\liminf\,_{j\to\infty}\inf\big\{\Delta^{j,\infty}(t,s)\!:\!
0\!\le\!s\!\le\!t\!\le\!T\big\}\!\ge\!0\,,
\end{equation}

\medbreak\noindent where $\Delta^{j,\infty}(t,s)$ is the quantity defined
in (\ref{defodel}). 
Suppose that
\mbox{$\liminf_{j\to\infty}\inf\big\{\Delta^{j,\infty}(t,s)\!:\!
0\!\le\!s\!\le\!t\!\le\!T\big\}\!<\!0$}. 
Pick  a number  $\beta$ such that $\beta>0$ and

$$
\liminf_{j\to\infty}
\inf\big\{\Delta^{j,\infty}(t,s)\!:\!
0\!\le\!s\!\le\!t\!\le\!T\big\}\le-3\beta\,.
$$ 

\medbreak\noindent Then there exists a subsequence
$\{\Xi^{j(k)}\}_{k=1}^\infty$ of $\{\Xi^{j}\}_{j=1}^\infty$ such that
$\inf\big\{\Delta^{j,\infty}(t,s)\!:\!
0\!\le\!s\!\le\!t\!\le\!T\big\}\le-2\beta$
for all $k$.  We can then choose, for each $k$, members $s_k$ and $t_k$
of $[0,T]$ such that $s_k\le t_k$ and 
\begin{equation}\label{conclu-x}
\big(\xi_0^{j(k)}(t_k)-\xi_{0}^{j(k)}(s_k)\big)-
\big(\xi_0^{\infty}(t_k)-\xi_{0}^{\infty}(s_k)\big)\le-\beta\,.
\end{equation}
By passing to a subsequence, if necessary, we may \mbox{assume}
that the $s_k$ and the $t_k$ converge to limits $s$, $t$. Clearly, 
then, $s\le t$.

If $s=t$, then 
\begin{eqnarray*}
\xi_0^{j(k)}(t_k)-\xi_0^{j(k)}(s_k)&=&\int_{s_k}^{t_k}
\dot\xi_0^{j(k)}(v)\,dv\\
&=&\int_{s_k}^{t_k}\big(\theta^{j(k)}(v)-C)\,dv\\
&\ge&-C(t_k-s_k)\,,
\end{eqnarray*}
so the fact that 
$\lim_{k\to\infty}(\xi_0^{\infty}(t_k)-\xi_{0}^{\infty}(s_k))=0$ implies 
the inequalities
\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\liminf_{k\to\infty}\Big(\big(\xi_0^{j(k)}(t_k)-\xi_{0}^{j(k)}(s_k)\big)-
\big(\xi_0^{\infty}(t_k)-\xi_{0}^{\infty}(s_k)\big)\Big)\\
\!\!\!&\!\!\!=&\!\!\!\!\liminf_{k\to\infty}\big(\xi_0^{j(k)}(t_k)\!
-\!\xi_{0}^{j(k)}(s_k)\big)
\!-\!\lim_{k\to\infty}\big(\xi_0^{\infty}(t_k)\!-\!\xi_{0}^{\infty}(s_k)\big)\\
\!\!\!&\!\!\!=&\!\!\!\!\liminf_{k\to\infty}\big(\xi_0^{j(k)}(t_k)-
\xi_{0}^{j(k)}(s_k)\big)\\
\!\!\!&\!\!\!\ge&\!\!\!\!\liminf_{k\to\infty}(-C(t_k-s_k))\\
\!\!\!&\!\!\!=&\!\!\!\!\lim_{k\to\infty}(-C(t_k-s_k))\\
\!\!\!&\!\!\!=&\!\!\!\!0\,,
\end{eqnarray*}
which clearly contradict (\ref{conclu-x}).

Now assume that $s<t$. Fix a positive number $\gamma$ such that
$2\gamma<t-s$,  and let $\Phi_{s,t,\gamma}$ be the set of all continuous 
nonnegative 
functions $\varphi:\R\mapsto\R$  that vanish outside the interval
$[s+\gamma,t-\gamma]$ and 
are such that $\varphi(v)\le 1$ for all $v\in\R$. Then

\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\!\!\!\!\liminf_{k\to\infty}\Big(\xi_0^{j(k)}(t_k)-\xi_0^{j(k)}(s_k)\Big)\\
&=&\liminf_{k\to\infty}\int_{s_k}^{t_k}\dot\xi_0^{j(k)}(v)\,dv\\
&=&\liminf_{k\to\infty}\int_{s_k}^{t_k}(\theta^{j(k)}(v)-C)\,dv\\
&=&\liminf_{k\to\infty}\Big(-C(t_k-s_k)+\int_{s_k}^{t_k}
\theta^{j(k)}(v)\,dv\Big)\\
&=&-C(t-s)+\liminf_{k\to\infty}\int_{s_k}^{t_k}\theta^{j(k)}(v)\,dv\\
&\ge&-C(t-s)+\liminf_{k\to\infty}\int_{s_k}^{t_k}
\varphi(v)\theta^{j(k)}(v)\,dv\\
&=&-C(t-s)+\liminf_{k\to\infty}\int_{[0,T]}\varphi(v)d\mu^{j(k)}(v)\\
&=&-C(t-s)+\lim_{k\to\infty}\int_{[0,T]}\varphi(v)d\mu^{j(k)}(v)\\
&=&-C(t-s)+\int_{[0,T]}\varphi(v)d\mu^\infty(v)\\
&\ge&-C(t-s)+\int_{[0,T]}\varphi(v)d\mu^{\infty,ac}(v)\\
&=&-C(t-s)+\int_{s}^t\varphi(v)\theta^\infty(v)\,dv\\
&=&\int_{s}^t\varphi(v)(\theta^\infty(v)-C)\,dv\\
&=&\int_{s}^t\varphi(v)\dot\xi_0^\infty(v)\,dv\,,
\end{eqnarray*}
where, for the first inequality, we have used the 
\mbox{fact} that $[s+\gamma,t-\gamma]\subseteq[s_k,t_k]$ 
when $k$ is large enough. 
\mbox{Therefore} 

$$
\liminf_{k\to\infty}\Big(\xi_0^{j(k)}(t_k)-\xi_0^{j(k)}(s_k)\Big)
\ge\int_{s}^t\varphi(v)\dot\xi_0^\infty(v)\,dv
$$

\medbreak\noindent for \mbox{every} \mbox{$\varphi\in\Phi_{s,t,\gamma}$}. 
Hence

\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\liminf_{k\to\infty}\Big(\xi_0^{j(k)}(t_k)-\xi_0^{j(k)}(s_k)\Big)\\
&\ge&\sup\Big\{\int_{s}^t\varphi(v)\dot\xi_0^\infty(v)\,dv:
\varphi\in\Phi_{s,t,\gamma}\Big\}\\
&=&\int_{s+\gamma}^{t-\gamma}\dot\xi_0^\infty(v)\,dv\\
&=&\xi_0^\infty(t-\gamma)-\xi_0^\infty(s+\gamma)\,.
\end{eqnarray*}

\medbreak\noindent Since $\gamma$ is arbitrary, we can let 
$\gamma\downarrow 0$, and conclude that 

\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\!\!\!\!\!\!\!
\liminf_{k\to\infty}\Big(\xi_0^{j(k)}(t_k)-\xi_0^{j(k)}(s_k)\Big)\\
&\ge&\xi_0^\infty(t)-\xi_0^\infty(s)\\
&=&\lim_{k\to\infty}\Big(\xi_0^\infty(t_k)-\xi_0^\infty(s_k)\Big)\,,
\end{eqnarray*}
so 

$$
\liminf_{k\to\infty}\Big(\big(\xi_0^{j(k)}(t_k)-\xi_0^{j(k)}(s_k)\big)-
\big(\xi_0^\infty(t_k)-\xi_0^\infty(s_k)\big)\Big)\ge 0\,,
$$
contradicting (\ref{conclu-x}). This completes the proof of (\ref{conclu}). 
\end{proof}

\ssection{Trajectories of steepest descent.}

\noindent We recall from \S\ref{sec-standard} that $\msdsvx$ denotes 
the set of all maximal augmented trajectories of $\hat\Sigma$ from $x$
of \mbox{steepest} descent with respect to $V$ and that, as \mbox{explained}
in Proposition \ref{trivialprop} and the remarks preceding its 
\mbox{statement}, $\msdsvx$ is always nonempty for 
\mbox{trivial} reasons, because 
the \mbox{trivial} trajectory $\Xi_x^{triv}$ always 
\mbox{belongs} to $\sdsvx$,
and once we know that $\sdsvx\neq\emptyset$ it \mbox{follows} immediately
from Zorn's Lemma that $\sdsvx$ must have a maximal \mbox{element}.

The truly nontrivial and useful result is the \mbox{statement} that
maximal steepest descent trajectories not only \mbox{exist} but are
``large,'' in the sense that they are ``right-unbounded.'' 
Precisely, if $\xi:I\mapsto\Omega$ is a curve, we say that $\xi$ is 
{\em right-unbounded} if (i)~the interval $I$ is open on the right
(that is, if $\tau=\sup\,I$, then either 
(a)~$\tau=+\infty$ or (b)~$\tau$ is finite and does not belong to 
$I$), and (ii)~if $\tau$ is finite, then for every compact subset $K$
of $\Omega$ there exists a $\tau_K$ such that $0\le\tau_K<\tau$ and
$\xi(t)\notin K$ whenever $\tau_K<t<\tau$. (Equivalently, condition
(ii) asserts that 
$\lim_{t\uparrow\tau}\xi(t)=\infty_\Omega$, where $\infty_\Omega$ is
the point at infinity of the one-point compactification of $\Omega$.)

\begin{theorem}\label{maintech}
Let $\hat\Sigma=(\Omega,U,f,L)$ be a
locally \mbox{coercive}, locally uniformly continuous augmented control system
such that $\fsh(x,U)$ is closed and convex for 
\mbox{every} $x\in\Omega$. 
Let $V:\Omega\mapsto\R$ be a \mbox{continuous} function that  
\mbox{satisfies} (\ref{loweq}) on $\Omega$ in the
\mbox{viscosity} sense. Let $x_*\in\Omega$, and let 
$\Xi=(\xi,\xi_0)$ 
be a maximal augmented trajectory of $\hat\Sigma$  from $x_*$ 
of \mbox{steepest} 
\mbox{descent} with respect to $V$, defined on an
interval $I$. Then $\xi$ is right-unbounded. 
\end{theorem}
\begin{proof}
We assume that the conclusion is not true and derive a contradiction.
Pick a $\Xi=(\xi,\xi_0)\in{\cal MSD}_{\hat\Sigma,V,x_*}$ that violates 
the conclusion.
This means, to begin with, that $\Xi$ is defined on a bounded interval 
$I$, and, in \mbox{addition}, this interval is either of the form
$[0,\tau]$, with $0\le\tau<+\infty$ (the ``right-closed case''), 
or of the \mbox{form} $[0,\tau\,[\,$, with $0<\tau<+\infty$
(the ``right-open case''). \mbox{Furthermore}, in the right-open case
there exist a \mbox{compact} subset $K$ of $\Omega$ and a sequence
$\{t_j\}_{j=1}^\infty$ such that $t_j\in I$ and 
$\xi(t_j)\in K$ for all $j$, and $\lim_{j\to\infty}t_j=\tau$. 
In order to treat the right-open and right-closed cases together, we also 
choose a compact subset $K$ of $\Omega$ and a sequence $\{t_j\}_{j=1}^\infty$ 
in $I$  in the right-closed case, subject 
to the only  requirements that $\xi(t)\in K$ for all $t\in[0,\tau]$
and $t_j\to\tau$ as $j\to\infty$. (For example, we could just 
choose $K=\{\xi(t):0\le t\le\tau\}$, $t_j=\tau$ for all $j$.)
Then in both cases
the $t_j$ belong to $I$ and converge to $\tau$, and 
the $\xi(t_j)$ belong to $K$.
In addition, we pick a function $\eta:I\mapsto U$ such that
$\dot\Xi(t)=F(\xi(t),\eta(t))$ for almost all $t\in I$.

Fix a positive number $\delta$ such that the compact set
$$
K_\delta=\{x\in\R^n:{\rm dist}(x,K)\|\le\delta\}
$$
is contained in $\Omega$. Then use the fact that 
$\hat\Sigma$ is locally coercive to choose $r,C$ such that 
$r>1$, $C>0$, and $\|f(x,u)\|^{r}\le L(x,u)+\tilde C$ 
for all $x\in K_\delta$, $u\in U$. Let 

$$
\bar V=\max\{|V(x)|:x\in  K_\delta\}\,.
$$

\medbreak\noindent
If $\rho=\frac{r}{r-1}$, so that $\frac{1}{r}+\frac{1}{\rho}=1$,	
and we define 
$$
\bar\delta(\sigma)\equaldef
2\Big(\sigma^{1/\rho}(2|\bar V|+(C+1)\sigma)^{1/r}+\sigma\Big)\,,
$$ 
then $\bar\delta(\sigma)$ goes to zero
as $\sigma\downarrow 0$. Therefore we can pick $\sigma$ 
such that $\bar\delta(\sigma)<\delta$.

Now suppose that we are in the right-open case. \mbox{Given}  any $j$,
the point $x_j=\xi(t_j)$ belongs to $K$. Let 

$$
S_j=\Big\{t\in I:t\ge t_j\wedge(\forall s)\big((t_j\le s\le t)\Rightarrow
\xi(s)\in K_\delta\big)\Big\}\,.
$$

\medbreak\noindent Then $S_j$ is a subinterval of $I$, whose left 
endpoint is $t_j$.

\medbreak
If $t,t'\in S_j$, and $t'\le t$, then the steepest descent property 
of $\Xi$ implies that 

$$
\xi_0(t)+V(\xi(t))\le\xi_0(t')+V(\xi(t'))\,,
$$
so
$$
\xi_0(t)-\xi_0(t')\le V(\xi(t))-V(\xi(t'))\le 2|\bar V|\,,
$$

\medbreak\noindent since both $\xi(t)$ and $\xi(t')$ belong to $K_\delta$.
Furthermore, the inequality 

$$
\|f(\xi(s),\eta(s))\|^r\le L(\xi(s),\eta(s))+C
$$

\medbreak\noindent is true for all $s\in S_j$, since 
$\xi(s)\in K_\delta$ for all such $s$. Therefore

\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\|\xi(t)-\xi(t')\|\\
&=&\Big\|\int_{t'}^t\dot\xi(s)\,ds\Big\|\\
&\le&\int_{t'}^t\|\dot\xi(s)\|\,ds\\
&=&\int_{t'}^t\|f(\xi(s),\eta(s))\|\,ds\\
&\le&(t-t')^{1/\rho}
\Big(\int_{t'}^t\|f(\xi(s),\eta(s))\|^r\,ds\Big)^{1/r}\\
&\le& (t-t')^{1/\rho}
\Big(\int_{t'}^{t}(L(\xi(s),\eta(s))+C)\,ds\Big)^{1/r}\\
&=& (t-t')^{1/\rho}(C(t-t')+\xi_0(t)-\xi_0(t'))^{1/r}\\
&=& (t-t')^{1/\rho}(C(t-t')+2|\bar V|))^{1/r}\\
&\le&\frac{1}{2}\bar\delta(t-t')\,.
\end{eqnarray*}
Now pick $j$ so large that $\tau-t_j<\sigma$, and apply 
the above inequality with $t'=t_j$. Then 
\begin{equation}\label{impli-x}
\|\xi(t)-\xi(t_j)\|\le\frac{1}{2}\bar\delta(\sigma)
\le\frac{1}{2}\delta\,\,\,\,{\rm for\,\,\,all}\,\,\,\,t\in S_j\,. 
\end{equation}
Hence, if we let $\bar t=\sup\,S_j$, it is impossible
that \mbox{$\bar t<\tau$}, because if $\bar t<\tau$ then 
$\bar t\in I$,  and  (\ref{impli-x}) implies that
\mbox{$\|\xi(\bar t)-\xi(t_j)\|\le\frac{1}{2}\delta$}, from which it follows,
by \mbox{continuity}, that there exists a positive $\alpha$ such that
$\|\xi(t)-\xi(t_j)\|<\delta$ for $\bar t\le t\le\bar t+\alpha$;
but then $\xi(t)\in K_\delta$ if $\bar t\le t\le\bar t+\alpha$,
since $\xi(t)\in K$; therefore $\bar t+\alpha\in S_j$, contradicting 
the definition of $\bar t$. 

Therefore $\bar t=\tau$, and this implies
that $S_j=[t_j,\tau\,[\,$, so $\xi(t)\in K_\delta$ 
for all $t\in[t_j,\tau\,[\,$. Then the bound 
$L(\xi(s),\eta(s))+C\ge\|f(\xi(s),\eta(s))\|^r$ 
holds for almost all $s\in[t_j,\tau\,[\,$, showing in particular that
the function 
$$
[t_j,\tau\,[\,\ni s\mapsto\psi(s)\equaldef L(\xi(s),\eta(s))+C
$$
which is measurable because $L(\xi(s),\eta(s))=\dot\xi_0(s)$,
is nonnegative. Hence, to prove that 
$\psi$ is Lebesgue-integrable on $[t_j,\tau\,[\,$, 
it suffices to show that
the integrals $\int_{t_j}^t (L(\xi(s),\eta(s))+C)\,ds$,
for $t_j\le t<\tau$, are bounded above by a fixed constant. 
But, if $t_j\le t<\tau$, then

\begin{eqnarray*}
\int_{t_j}^t (L(\xi(s),\eta(s))+C)\,ds&=&\xi_0(t)-\xi_0(t_j)+C(t-t_j)\\
&\le&2|V|+C\sigma\,.
\end{eqnarray*}

\medbreak\noindent Therefore $\psi$ is Lebesgue-integrable on 
$[t_j,\tau\,[\,$,  and then $\dot\xi_0$  is also Lebesgue-integrable 
on $[t_j,\tau\,[\,$, because 
\mbox{$\dot\xi_0=\psi-C$}.  Since 
$\dot\xi_0$  is  Lebesgue-integrable on $[0,t_j]$, we conclude that
$\dot\xi_0$  is  Lebesgue-integrable on $[0,\tau\,[\,$. Hence
the limit $x_0^{\#}=\lim_{t\uparrow\tau}\xi_0(t)$ exists,
and the \mbox{extended} function $\xi_0^{\#}:[0,\tau]\mapsto\R$
defined by $\xi_0^{\#}(t)=\xi_0(t)$ if $0\le t<\tau$, 
$\xi_0^{\#}(\tau)=x_0^{\#}$, is absolutely continuous.

In addition, the bound

$$
\|\dot\xi(s)\|^r=\|f(\xi(s),\eta(s)\|^r\le L(\xi(s),\eta(s))+C=\psi(s)\,,
$$

\medbreak\noindent valid on $[t_j,\tau\,[\,$, shows that
$\dot\xi$ in $L^r$ on $[s_j,\tau\,[\,$, so {\em a \mbox{fortiori}}
$\dot\xi$ is Lebesgue-integrable on $[s_j,\tau\,[\,$, and
then the limit $x^{\#}=\lim_{t\uparrow\tau}\xi(t)$ exists
and belongs to $K_\delta$ (\mbox{because} $x^{\#}=
\lim_{\ell\to\infty}\xi(t_\ell)$ and the $\xi(t_\ell)$ belong to 
$K_\delta$), and 
the extended function $\xi^{\#}:[0,\tau]\mapsto\Omega$
defined by $\xi^{\#}(t)=\xi(t)$ if $0\le t<\tau$, 
$\xi^{\#}(\tau)=x_0^{\#}$, is absolutely continuous.  

Hence we have constructed an absolutely continuous
extension $\Xi^{\#}=(\xi^{\#},\xi_0^{\#})$ of
$\Xi$ to the closed interval $[0,\tau]$. Clearly, 
$\Xi^{\#}$ is also an augmented trajectory of $\hat\Sigma$
\mbox{starting} at $x_*$ at time $0$, and 
the fact that $\Xi$ is of steepest descent with respect to $V$
and $V$ is continuous implies that 
$\Xi^{\#}$ is of steepest descent with respect to $V$ as well. 
\mbox{Therefore} $\Xi$ is not maximal, because we have constructed 
an \mbox{extension} to a strictly larger interval. It follows that the
``right-open case'' cannot arise at all. 
 
\medbreak
We now analyze the right-closed case, and show that it cannot arise either.
We do this by constructing an extension $\Xi^{\#}$ of $\Xi$ to 
a $\Xi^{\#}\in{\cal MSD}_{\hat\Sigma,V,x_*}$ defined on the
interval $[0,\tau+\sigma]$. This will, of course, contradict 
the assumed maximality of $\Xi$, and conclude our proof.

To construct $\Xi^{\#}$, we construct $\eps$-approximate 
\mbox{augmented}
trajectories $\Xi^\eps:[\tau,\tau+\sigma]\mapsto\Omega\times\R$
of $\hat\Sigma$ 
that are ``$\eps$-approximately of steepest descent,''
and such that $\Xi^\eps(\tau)=\Xi(\tau)$. We then pass to the limit 
as $\eps\downarrow 0$, using the compactness theorem (\ref{compacttheo}),
and get an exact  augmented
trajectory $\tilde\Xi:[\tau,\tau+\sigma]\mapsto\Omega\times\R$
of $\hat\Sigma$ such that $\tilde\Xi(\tau)=\Xi(\tau)$, which is 
exactly of steepest descent. 

Write $x^{\#}=\xi(\tau)$, $x_0^{\#}=\xi_0(\tau)$. 
Fix an $\eps$ such that $0<\eps<1$.
Let ${\cal Z}_\eps$ be the set of all
triples $(I',\Xi',S')$ such that 

\begin{itemize}
\item[(1)] $I'$ is a \mbox{subinterval} of $[\tau,\tau+\sigma]$
such that $\tau\in I'$, 
\item[(2)] \mbox{$\Xi'=(\xi',\xi'_0):I'\mapsto K_\delta\times\R$} 
is an $\eps$-approximate augmented trajectory of $\hat\Sigma$, 
\item[(3)] $\Xi'(\tau)=(x^{\#},x_0^{\#})$, 
\item[(4)] $S'$ is a strongly $\eps$-dense subset of $I'$ 
such that \mbox{$\xi'_0(s)+V(\xi'(s))+\eps(t-s)\ge\xi'_0(t)+V(\xi'(t))$} 
for all $s,t\in S'$ such that $s\le t$.
\end{itemize}

\noindent (We say that a subset $S$ of an interval $J$ is {\em
strongly \mbox{$\eps$-dense}} if for every $t\in J$ there exist $s,s'\in S$
such that $s\le t\le s'$ and $s'-s\le\eps$.)

\medbreak
We partially order ${\cal Z}_\eps$ by stipulating that, if 
\mbox{$(I_i,\Xi_i,S_i)\in{\cal Z}_\eps$} for $i=1,2$, then 
\mbox{$(I_1,\Xi_1,S_1)\preceq(I_2,\Xi_2,S_2)$} iff $I_1\subseteq I_2$,
$\Xi_1$ is the \mbox{restriction} of $\Xi_2$ to $I_1$, and 
$S_1=S_2\cap I_1$. 

It is clear that ${\cal Z}_\eps\neq\emptyset$, because the triple 
$(\{\tau\},\Xi_{x^{\#},\tau},\{\tau\})$\mbox{---where} $\Xi_{x^{\#},\tau}$
is the map \mbox{$\Xi:\{\tau\}\mapsto\Omega\times\R$} such that 
$\Xi(\tau)=(x^{\#},\xi_0(\tau))$---belongs to 
${\cal Z}_\eps$.  If $Z$ is a totally ordered subset 
of ${\cal Z}_\eps$, we show that $Z$ has an upper bound 
$(I'_*,\Xi'_*,S'_*)$ in ${\cal Z}_\eps$. 
This conclusion is \mbox{trivial} if $Z=\emptyset$, for in that 
case we can take 
\mbox{$(I'_*,\Xi'_*,S'_*)=(\{\tau\},\Xi_{x^{\#},\tau},\{\tau\})$}. 
Assume that 
$Z\neq\emptyset$. Let $I'_*$ be the union of the intervals $I'$ for all
the \mbox{members} $(I',\Xi',S')$ of $Z$. Then $I'_*$ is clearly a subinterval
of $[\tau,\tau+\sigma]$, and $\tau\in I'$. 
If $t\in I'_*$, we define $\Xi'_*(t)=\Xi'(t)$,
where $(I',\Xi',S')$ is any member of $Z$ such that $t\in I$.
Write $\Xi'_*=(\xi'_*,\xi'_{0,*})$.  Then $\Xi'_*$ is
obviously well \mbox{defined}, and is an 
$\eps$-approximate augmented \mbox{trajectory} of $\hat\Sigma$
such that $\xi'_*(\tau)=x^{\#}$ and
$\xi'_{0,*}(\tau)=x_0^{\#}$. We then let 

\begin{equation}\label{showing}
S'_*=\bigcup_{(I',\Xi',S')\in Z}S'\,.
\end{equation}

We want to prove that the triple $(I'_*,\Xi'_*,S'_*)$ is 
an upper bound for $Z$ in ${\cal Z}_\eps$. If we show that 

\begin{equation}\label{showing-x}
(I'_*,\Xi'_*,S'_*)\in {\cal Z}_\eps\,,
\end{equation}

\medbreak\noindent then the fact that
$(I'_*,\Xi'_*,S'_*)$ is an upper bound for $Z$  is immediate,
so all we really need is to prove (\ref{showing-x}).

It is evident that the triple 
$(I'_*,\Xi'_*,S'_*)$ satisfies the first three of the four
conditions in the definition of ${\cal Z}_\eps$. 
Let us show that it satisfies the fourth one as well. 
If $t\in I'_*$,  we can find  $(I'',\Xi'',S'')\in Z$ such that
$t\in I''$. Since $S''$ is a strongly $\eps$-dense subset
of $I''$, there exist $s_1,s_2\in S''$ such that
$s_1\le t\le s_2$ and $s_2-s_1\le\eps$. Then 
$s_1$ and $s_2$ belong to $ S'_*$, and this establishes that
$S'_*$ is a strongly $\eps$-dense subset of $I'_*$. 
Now, if $s_1$, $s_2$ are members of $S'_*$ such that
$s_1\le s_2$, then we can find (using the fact that 
$Z$ is totally ordered) a member $(I',\Xi',S')$ 
of $Z$ such that $s_1$ and $s_2$ belong to $I'$. It then follows
easily that $s_1$ and $s_2$ must  belong to $S'$. Then,
if we write $\Xi'=(\xi',\xi'_0)$, the fact that 
$s_1\in S'$, $s_2\in S'$, $s_1\le s_2$, and 
\mbox{$(I',\Xi',S')\in{\cal Z}_\eps$},
imply that 

$$
\xi'_0(s_1)+V(\xi'(s_1))+\eps(s_2-s_1)\ge\xi'_0(s_2)+V(\xi'(s_2))\,.
$$
Hence 

$$
\xi'_{0,*}(s_1)+V(\xi'_*(s_1))+\eps(s_2-s_1)\ge
\xi'_{0,*}(s_2)+V(\xi'_*(s_2))\,.
$$
Since this is true for any two members 
$s_1$, $s_2$ of $S'_*$ such that $s_1\le s_2$, we conclude that
the fourth condition holds as well, and the proof of 
(\ref{showing-x}) is complete.

We have shown that every totally ordered subset of ${\cal Z}_\eps$ has
an upper bound in ${\cal Z}_\eps$. Therefore Zorn's
Lemma implies that ${\cal Z}_\eps$ has a maximal element 
$(I',\Xi',S')$.  We claim that $I'=[\tau,\tau+\sigma]$. 
Suppose this was not true.
Then either 
\begin{itemize}
\item[(A)] $I'=[\tau,\zeta\,[\,$ 
for some $\zeta$ such that $\tau<\zeta\le\tau+\sigma$, 
\end{itemize}
or
\begin{itemize}
\item[(B)] $I'=[\tau,\zeta]$ for some $\zeta$ such that 
$\tau\le\zeta<\tau+\sigma$.
\end{itemize}

\medbreak\noindent We shall exclude both possibilities. 

\medbreak Write $\Xi'=(\xi',\xi'_0)$. 
Let \mbox{${\cal B}_\eps=\{v\in\R^n:\|v\|\le\eps\}$.} 
Let 
$I'\ni t\mapsto(\eta'(t),v(t))\in U\times{\cal B}_\eps$ 
be a function such that the function $v(\cdot)$ is measurable,
and the equalities 
$\dot\xi'(t)=f(\xi'(t),\eta'(t))+v(t)$
and $\dot\xi'_0(t)=L(\xi'(t),\eta'(t))$ hold for almost all $t\in I'$. 

Let $t\in I'$. Then there must exist a $\bar t\in S'$ such that
$\bar t\ge t$. Also, $\tau$ must belong to $S'$, because
$\tau$ is the leftmost point of $I'$. Then 

$$
\xi'_0(\tau)+V(\xi'(\tau))+\eps(t-\tau)\ge \xi'_0(\bar t)+V(\xi'(\bar t))\,,
$$
so (since $\eps<1$) 

\begin{eqnarray*}
\xi'_0(\bar t)-\xi'_0(\tau)&\le&\eps(t-\tau)+
V(\xi'(\tau))-V(\xi'(\bar t))\\
&\le&2|\bar V|+\sigma\,.
\end{eqnarray*}

\medbreak\noindent We then have (using the fact that
$\|v(s)\|<1$, which is true because $\eps<1$), 

\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\int_\tau^{t}\|\dot\xi'(s)\|ds\\
&=&\int_\tau^{t}\|f(\xi'(s),\eta'(s))+v(s)\|ds\\
&\le&\int_\tau^{\bar t}\|f(\xi'(s),\eta'(s))+v(s)\|ds\\
&\le&\int_\tau^{\bar t}\|f(\xi'(s),\eta'(s))\|ds+\int_\tau^{\bar t}
\|v(s))\|ds\\
&\le&(\bar t-\tau)^{1/\rho}
\Big(\int_\tau^{\bar t}\|f(\xi'(s),\eta'(s))\|^rds\Big)^{1/r}
+\bar t-\tau\\
&\le& \sigma^{1/\rho}
\Big(\int_\tau^{\bar t}\big(L(\xi'(s),\eta'(s))+C\big)ds\Big)^{1/r}
+\sigma\\
&\le& \sigma^{1/\rho}
\Big(\xi'_0(\bar t)-\xi'_0(\tau)+C(\bar t-\tau)\Big)^{1/r}
+\sigma\\
&\le& \sigma^{1/\rho}\big(2|\bar V|+\sigma+C\sigma\big)^{1/r}
+\sigma\\
&=& \frac{1}{2}\bar\delta(\sigma)\\
&\le&\frac{1}{2}\delta\,.
\end{eqnarray*}

Also, if we let $\theta(s)=\dot\xi'_0(s)+C=
L(\xi'(s),\eta'(s))+C$, then $\theta$ is nonnegative 
and 
\begin{eqnarray*}
\int_\tau^{t}\theta(s)ds&\le&\int_\tau^{\bar t}\theta(s)ds\\
&=&\xi'_0(\bar t)-\xi'_0(\tau)+C(\bar t-\tau)\\
&\le& 2|\bar V|+(C+1)\sigma\,.
\end{eqnarray*}

Since $t$ is an arbitrary member of $I'$, the above
\mbox{inequalities} imply that the functions $\dot\xi'$ and $\theta$
are \mbox{integrable} on $I'$. Since $I'$ is bounded, $\dot\xi'$ is
\mbox{integrable} as well.  This implies, in particular, that the
\mbox{limits} $\lim_{t\uparrow\tau+\zeta}\xi'(t)$ and
$\lim_{t\uparrow\tau+\zeta} \xi'_0(t)$ exist. Hence, if (A) holds, we
can extend $\Xi'$ to the closed interval 
\mbox{$\tilde I'\equaldef[\tau,\tau+\zeta]$}, 
and the result is a curve $\tilde\Xi'$
in $\R^{n+1}$ which is obviously an $\eps$-approximate augmented
\mbox{trajectory} of $\hat\Sigma$. If we then define $\tilde
S'=S'\cup\{\tau+\zeta\}$, then $\tilde S'$ is a strongly $\eps$-dense
subset of $\tilde I'$. Now, if $s_1,s_2$ are \mbox{members} of $\tilde
S'$ such that $s_1\le s_2$ we have to prove that
$\tilde\xi'_0(s_1)+V(\tilde\xi'(s_1))+\eps(s_2-s_1)\ge
\tilde\xi'_0(s_2)+V(\tilde\xi'(s_2))$. This is clearly true if
$s_2<\tau+\zeta$ or if $s_1=s_2$. So the only 
\mbox{remaining} case for us to consider
is when $s_1<s_2=\tau+\zeta$. But in that case we can take a \mbox{sequence}
$\{v_\ell\}_{\ell=1}^\infty$ of points of $I'$ such that
$v_\ell\uparrow\tau+\zeta$ as $\ell\to\infty$, and $v_\ell>s_1$ for
all $\ell$. Then we can pick, for each $\ell$, a $w_\ell\in S'$ such
that $v_\ell\le w_\ell$. On the other hand, $s_1\in S'$, since
$s_1<\tau+\zeta$. Therefore $\tilde\xi'_0(s_1)+V(\tilde\xi'(s_1))
+\eps(w_\ell-s_1)\ge
\tilde\xi'_0(w_\ell)+V(\tilde\xi'(w_\ell))$ for all
$\ell$. If we let $\ell\to\infty$ and use the continuity 
of $\tilde\xi'_0$ and $V$, we find that 

$$
\tilde\xi'_0(s_1)+V(\tilde\xi'(s_1))+\eps(s_2-s_1)
\ge\tilde\xi'_0(s_2)+V(\tilde\xi'(s_2))\,,
$$

\medbreak\noindent as desired. This completes the proof that the 
\mbox{extension} $(\tilde I',\tilde\Xi',\tilde S')$ of 
$(I',\Xi',S')$ is also in ${\cal Z}_\eps$, a fact that 
of course \mbox{contradicts} the maximality of 
$(I',\Xi',S')$ if (A) holds. We have thus derived a contradiction 
from the \mbox{assumption} that (A) is true. Hence (A) is excluded. 

\medbreak We are thus left with Case (B), that is, the possibility that
$I'=[\tau,\tau+\zeta]$ and $\zeta<\sigma$.
We now \mbox{proceed} to exclude this case.
The integral calculation done above shows that 

$$
\int_\tau^{\tau+\zeta}\|\dot\xi'(t)\|dt\le\frac{\delta}{2}\,,
$$
so 
\begin{equation}\label{bbc}
\|\xi'(\tau+\zeta)-\xi'(\tau)\|\le\frac{\delta}{2}\,.
\end{equation}

\medbreak\noindent Since $\xi'(\tau)\in K$, (\ref{bbc}) implies that,
if $q=\xi'(\tau+\zeta)$, then ${\rm dist}(q,K)\le\frac{\delta}{2}$.
In particular, $q$  is an interior point of $K_\delta$. Using Theorem 
\ref{satinfi},  we construct  sequences 
$\{x_j\}_{j=1}^\infty$, $\{v_j\}_{j=1}^\infty$,
$\{\lambda_j\}_{j=1}^\infty$, $\{h_j\}_{j=1}^\infty$,
$\{\gamma_j\}_{j=1}^\infty$
in $\Omega$, $\R^n$, $\R$, $\R$, and $\R$, respectively, such that 
\begin{itemize}
\item[(1)] the inequalities
\begin{eqnarray*}
h_j&>&0\,,\\
\gamma_j&>&0\,,\\
\|x_j-q-h_jv_j\|&\le&h_j\gamma_j\,,\\
V(x_j)&\le&V(q)-h_j\lambda_j+h_j\gamma_j\,,
\end{eqnarray*}
hold for all $j$, 
\item[(2)] $(v_j,\lambda_j)\in\fsh(q,U)$ for all $j$, 
\item[(3)] $h_j\downarrow 0$, $\gamma_j\downarrow 0$, and
$x_j\to q$  as $j\to\infty$.
\end{itemize}
We let $(v_j,\lambda_j)=\fsh(q,u_j)$, $u_j\in U$, and write
\mbox{$\tilde v_j=h_j^{-1}(x_j-q)$}, so that 
\begin{equation}\label{cypr}
\|\tilde v_j-v_j\|\le\gamma_j\,.
\end{equation}

Choose $j$ so large that the following conditions are fulfilled:
\begin{eqnarray}
h_j&<&\eps\,,\\
\gamma_j&<&\frac{\eps}{2}\,,\\
\|x_j-q\|&<&\beta\,,
\end{eqnarray}
where $\beta$ is a positive number such that $\beta<\frac{\delta}{2}$ and
\mbox{$\omega(\beta)<\frac{\eps}{2}$}.


Then 
$$
h_j\|\tilde v_j\|\le\|x_j-q\|\le\beta\le\frac{\delta}{2}\,.
$$

\medbreak\noindent Define a new trajectory 
$\xi^{\#}:I^{\#}\mapsto\R^n$, where
$$
I^{\#}\equaldef[\tau,\tau+\zeta+h_j]\,,
$$ 
by letting 
$$
\xi^{\#}(s)=\left\{\begin{array}{lll}
\xi'(s)&{\rm if}&\tau\le s\le\tau+\zeta\,,\\
q\!+\!(s\!-\!\tau\!-\!\zeta)\tilde v_j
&{\rm if}&\tau+\zeta\le s\le\tau+\zeta+h_j\,,
\end{array}\right.
$$
and augment it by defining
\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!
\xi_0^{\#}(s)=\\
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\left\{\begin{array}{lll}
\xi'_0(s)&{\rm if}&\tau\!\le\!s\!\le\!\tau\!+\!\zeta\,,\\
q_0\!+\!\int_{\tau+\zeta}^sL(\xi^{\#}(v),u_j)\,dv
&{\rm if}&\tau\!+\!\zeta\!\le\!s\!\le\!\tau\!+\!\zeta\!+\!h_j
,\end{array}\right.
\end{eqnarray*}

\medbreak\noindent where $q_0=\xi'_0(\tau+\zeta)$. 
Let $\Xi^{\#}=(\xi^{\#},\xi_0^{\#})$. Then $\Xi^{\#}$ is 
clearly \mbox{absolutely} continuous, and its restriction to 
$[\tau,\tau+\zeta]$
is an \mbox{$\eps$-approximate} augmented trajectory of $\hat\Sigma$. If 
$s$ belongs to \mbox{$[\tau+\zeta,\tau+\zeta+h_j]$}, then

$$
\|\xi^{\#}(s)-q\|\le h_j\|\tilde v_j\|\le\frac{\delta}{2}\,.
$$

\medbreak\noindent Therefore $\xi^{\#}(s)\in K_\delta$. Then

\begin{eqnarray*}
\dot\Xi^{\#}(s)&=&(v_j,L(\xi^{\#}(s),u_j))\\
&=&(f(q,u_j),L(\xi^{\#}(s),u_j))\\
&=&\fsh(\xi^{\#}(s),u_j)+(w_j(s),0)\,,
\end{eqnarray*}

\medbreak\noindent where 

$$
w_j(s)=f(q,u_j)-f(\xi^{\#}(s),u_j)\,.
$$

\medbreak\noindent Since

$$
\|\xi^{\#}(s)-q\|\le h_j\|v_j\|\le\beta\,,
$$

\medbreak\noindent we have 

$$\
\|w_j(s)\|\le\omega(\beta)\le\eps\,.
$$

\medbreak\noindent Therefore $\Xi^{\#}$ is an 
\mbox{$\eps$-approximate} augmented trajectory of $\hat\Sigma$. 

Clearly, $\xi^{\#}(\tau+\zeta+h_j)=x_j$. Therefore,

\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
V(\xi^{\#}(\tau+\zeta+h_j))\\
&=&V(x_j)\\
&\le&V(q)-h_j\lambda_j+h_j\gamma_j\\
&=&V(\xi^{\#}(\tau+\zeta))-h_jL(q,u_j)+h_j\gamma_j\\
&=&V(\xi^{\#}(\tau+\zeta))
-\int_{\tau+\zeta}^{\tau+\zeta+h_j}L(\xi^{\#}(v),u_j)\,dv\\
&\,&\,\,
-\int_{\tau+\zeta}^{\tau+\zeta+h_j}\Big(L(q,u_j)-L(\xi^{\#}(v),u_j)\Big)\,dv
+h_j\gamma_j\\
&=&V(\xi^{\#}(\tau+\zeta))+\xi_0^{\#}(\tau+\zeta)
-\xi_0^{\#}(\tau+\zeta+h_j)+E\,,
\end{eqnarray*}
where 
$$
E=
-\int_{\tau+\zeta}^{\tau+\zeta+h_j}\Big(L(q,u_j)-L(\xi^{\#}(v),u_j)\Big)\,dv
+h_j\gamma_j\,.
$$

\medbreak\noindent Then
$$
E\le h_j\omega(\beta)+h_j\gamma_j\le\eps h_j\,.
$$

\medbreak\noindent Therefore

\begin{eqnarray*}
&\,&\!\!\!\!\!\!\!\!\!V(\xi^{\#}(\tau+\zeta+h_j))\le\\
&\,&\!\!\!\!\!V(\xi^{\#}(\tau+\zeta))+\xi_0^{\#}(\tau+\zeta)
-\xi_0^{\#}(\tau+\zeta+h_j)+\eps h_jE\,,
\end{eqnarray*}
that is, 
\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
V(\xi^{\#}(\tau+\zeta))+\xi_0^{\#}(\tau+\zeta)
+\eps h_j\nonumber\\
&\ge&V(\xi^{\#}(\tau+\zeta+h_j))+\xi_0^{\#}(\tau+\zeta+h_j)\,.\label{labla-y}
\end{eqnarray}

\medbreak\noindent This last inequality implies that, if we define

$$
S^{\#}=S'\cup\{\tau+\zeta+h_j\}\,,
$$

\medbreak\noindent then we can easily show that
$S^{\#}$ is a strongly $\eps$-dense subset
of $I^{\#}$ such that 

\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
V(\xi^{\#}(s_1))+\xi_0^{\#}(s_1)+\eps(s_2-s_1)\nonumber\\
&\ge& V(\xi^{\#}(s_2))+\xi_0^{\#}(s_2)\label{labla}
\end{eqnarray}

\medbreak\noindent whenever $s_1,s_2\in S^{\#}$ and
$s_2\ge s_1$.  The strong \mbox{$\eps$-density} follows because 
$S'$ is a strongly $\eps$-dense subset of $I'$ and $h_j<\eps$,
since $\tau+\zeta$ necessarily belongs to $S'$. Inequality (\ref{labla}) 
is clearly true if $s_1=s_2$ or both $s_1$ and $s_2$ belong to 
$S'$. To verify that it holds in the remaining case, that is, when
$s_1\in S'$ and $s_2=\tau+\zeta+h_j$, if suffices to use once again
the fact that $\tau+\zeta\in S'$, so 


\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
V(\xi^{\#}(s_1))+\xi_0^{\#}(s_1)+\eps(\tau+\zeta-s_1)\nonumber\\
&\ge&  V(\xi^{\#}(\tau+\zeta)+\xi_0^{\#}(\tau+\zeta)\label{labla-x}\,.
\end{eqnarray}

\medbreak\noindent If we add (\ref{labla-y}) and (\ref{labla-x}), and
cancel the sum 
\mbox{$V(\xi^{\#}(\tau+\zeta))+\xi_0^{\#}(\tau+\zeta)$} that
appears on both sides of the result, we get 

\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\!\!\!\!\!\!\!
V(\xi^{\#}(s_1))+\xi_0^{\#}(s_1)+\eps(\tau+\zeta+h_j-s_1)\nonumber\\
&\ge&  V(\xi^{\#}(\tau+\zeta+h_j))
+\xi_0^{\#}(\tau+\zeta+h_j)\label{labla-uu}\,,
\end{eqnarray}

\medbreak\noindent that is, 

\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
V(\xi^{\#}(s_1))+\xi_0^{\#}(s_1)+\eps(s_2-s_1)\nonumber\\
&\ge&  V(\xi^{\#}(s_2))+\xi_0^{\#}(s_2)\label{labla-vv}\,.
\end{eqnarray}


\medbreak\noindent It then follows that $(I^{\#},\Xi^{\#},S^{\#})$
belongs to ${\cal Z}_\eps$. \mbox{Since}
$(I',\Xi',S')\preceq(I^{\#},\Xi^{\#},S^{\#})$ but
\mbox{$(I',\Xi',S')\neq(I^{\#},\Xi^{\#},S^{\#})$}, we have arrived at a
\mbox{contradiction}, which this time has arisen from the \mbox{assumption}
that (B) holds. Hence (B) is excluded as well.

It now follows that $I'=[\tau,\tau+\sigma]$. In other \mbox{words}, we
have shown that there exists an $\eps$-approximate \mbox{augmented}
trajectory $\Xi_\eps=(\xi_\eps,\xi_{0,\eps})$ of $\hat\Sigma$ which
is \mbox{defined} on $[\tau,\tau+\sigma]$, and is such that there is a
strongly $\eps$-dense subset $S_\eps$ of $[\tau,\tau+\sigma]$ having
the property that 

\begin{equation}
V(\xi_\eps(s_1))\!+\!\xi_{0,\eps}(s_1)\!+\!\eps(s_2\!-\!s_1)
\!\ge\! V(\xi_\eps(s_2))\!+\!\xi_{0,\eps}(s_2)\label{labla-aa}
\end{equation}


\medbreak\noindent whenever $s_1,s_2\in S_\eps$ and $s_1\le s_2$.

Clearly, the points $\tau$, $\tau+\sigma$ must belong to $S_\eps$.
If we aplly (\ref{labla-aa}) with $s_1=\tau$, $s_2=\tau+\sigma$, we find
the bound 

$$
\xi_{0,\eps}(\tau+\sigma)-\xi_{0,\eps}(\tau)\le 2|\bar V|+\sigma\,.
$$

\medbreak\noindent Therefore, if $\{\eps_j\}_{j=1}^\infty$ is a sequence
of positive numbers that converges to $0$, we can apply 
Theorem (\ref{compacttheo}) with $T=\sigma$ and
$\Xi^j=(\xi^j,\xi_{0}^{j})=(\check\xi_{\eps_j},\check\xi_{0,\eps_j})$,
where \mbox{$\check\xi_{\eps_j}(s)=\xi_{\eps_j}(s+\tau)$} and 
\mbox{$\check\xi_{0,\eps_j}(s)=\xi_{0,\eps_j}(s+\tau)-
\xi_{0,\eps_j}(\tau)$} for $s\in[0,\sigma$. 
Then, after passing to a \mbox{subsequence},
we may assume that there exists an augmented
trajectory $\Xi^\infty=(\xi^\infty,\xi_{0}^\infty)$ of 
$\hat\Sigma$ such that
\begin{itemize}
\item[(i)] $\xi_0^\infty(\tau)=0$, 
\item[(ii)] the sequence $\{\xi^{j}\}_{j=1}^\infty$
converges uniformly to $\xi^\infty$, and 
\item[(iii)] 
$\liminf_{j\to\infty}\inf\big\{\Delta^{j,\infty}(t,s)\!:\!
0\!\le\!s\!\le\!t\!\le\!T\big\}\!\ge\!0$,
where, if $j,j'\in\nn\cup\{+\infty\}$, we define
$\Delta^{j,j'}$ as in (\ref{defodel}). 
\end{itemize}

\medbreak\noindent We now show that $\Xi^\infty$ 
is an augmented  trajectory of steepest descent of
$\hat\Sigma$ from $x^{\#}$ with respect to $V$.  For this purpose,
we pick $s_1,s_2\in[0,\sigma]$ such that $s_1\le s_2$, and prove that

\begin{equation}
V(\xi^\infty(s_1))+\xi_{0}^\infty(s_1)
\ge V(\xi^\infty(s_2))+\xi_{0}^\infty(s_2)\,.\label{labla-bb}
\end{equation}

\medbreak\noindent For this purpose we pick, for each sufficiently large
$j$, points $s_1^j,s_2^j$ in $S_{\eps_j}$ such that 


$$
s_1^j\le s_1+\tau\le s_1+\eps_j<s_2^j-\eps_j\le
s_2+\tau\le s_2^j\,.
$$

\medbreak\noindent (We assume that $s_1<s_2$, because 
(\ref{labla-bb}) is trivially true if $s_1=s_2$.) 

Then 

\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
V(\xi_{\eps_j}(s_1^j))-V(\xi_{\eps_j}(s_2^j))+\eps_j(s_2^j-s_1^j)
\nonumber\\
&\ge&\xi_{0,\eps_j}(s_2^j)-\xi_{0,\eps_j}(s_1^j)
\,,\label{labla-cc}
\end{eqnarray}

\medbreak\noindent that is, 

\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
V(\xi^j(s_1^j-\tau))-V(\xi^j(s_2^j-\tau))+\eps_j(s_2^j-s_1^j)
\nonumber\\
&\ge&\xi_{0}^j(s_2^j-\tau)-\xi_{0}^j(s_1^j-\tau)\nonumber\\
&=&\xi_{0}^\infty(s_2^j-\tau)-\xi_{0}^\infty(s_1^j-\tau)
+\Delta^{j,\infty}(s_2^j-\tau,s_1^j-\tau)\nonumber\\
&\ge&\xi_{0}^\infty(s_2^j-\tau)-\xi_{0}^\infty(s_1^j-\tau)\nonumber\\
&\,&\,\,+\inf\{\Delta^{j,\infty}(t,s):0\le s\le t\le\sigma\}\nonumber
\end{eqnarray}

\medbreak\noindent Then 

\begin{eqnarray}
&\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!V(\xi^\infty(s_1))-V(\xi^\infty(s_2))
\nonumber\\
&=&\lim_{j\to\infty}\Big(
V(\xi^j(s_1^j\!-\!\tau))\!-\!V(\xi^j(s_2^j\!-\!\tau))\!+\!
\eps_j(s_2^j\!-\!s_1^j)\Big)\nonumber\\
&\ge&\liminf_{j\to\infty}\Big(
\xi_{0}^j(s_2^j-\tau)-\xi_{0}^j(s_1^j-\tau)\Big)\nonumber\\
&=&\liminf_{j\to\infty}\Big(
\xi_{0}^\infty(s_2^j-\tau)-\xi_{0}^\infty(s_1^j-\tau)\nonumber\\
&\,&\,\,\,+\Delta^{j,\infty}(s_2^j-\tau,s_1^j-\tau)\Big)\nonumber\\
&\ge&\liminf_{j\to\infty}\Big(
\xi_{0}^\infty(s_2^j-\tau)-\xi_{0}^\infty(s_1^j-\tau)\nonumber\\
&\,&\,\,+\inf\{\Delta^{j,\infty}(t,s):0\le s\le t\le\sigma\}\Big)\nonumber\\
&=&\lim_{j\to\infty}\Big(
\xi_{0}^\infty(s_2^j-\tau)-\xi_{0}^\infty(s_1^j-\tau)\Big)\nonumber\\
&\,&\,\,+\liminf_{j\to\infty}\Big(
\inf\{\Delta^{j,\infty}(t,s):0\le s\le t\le\sigma\}\Big)\nonumber\\
&=&\xi_{0}^\infty(s_2)-\xi_{0}^\infty(s_1)\nonumber\\
&\,&\,\,+\liminf_{j\to\infty}\Big(
\inf\{\Delta^{j,\infty}(t,s):0\le s\le t\le\sigma\}\Big)\nonumber\\
&\ge&\xi_{0}^\infty(s_2)-\xi_{0}^\infty(s_1)\,.\nonumber
\end{eqnarray}

\medbreak\noindent We have thus proved (\ref{labla-bb}), thereby 
establishing that $\Xi^\infty$ is an augmented trajectory of
$\hat\Sigma$ from $x^{\#}$ of steepest descent 
with respect to $V$. If we now concatenate
$\Xi$ and $\Xi^\infty$ in the obvious way, by defining

$$
\xi^{\#}(s)=\left\{\begin{array}{lll}
\xi(s)&{\rm if}&0\le s\le\tau\,,\\
\xi^\infty(s-\tau)&{\rm if}&\tau\le s\le\tau+\sigma\,,
\end{array}\right.
$$

$$
\xi_0^{\#}(s)=\left\{\begin{array}{lll}
\xi_0(s)&{\rm if}&0\le s\le\tau\,,\\
\xi_0^\infty(s-\tau)+\xi_0(\tau)&{\rm if}&\tau\le s\le\tau+\sigma\,,
\end{array}\right.
$$

\medbreak\noindent then $\Xi^{\#}$ is an augmented
trajectory of $\hat\Sigma$ from $x_*$ of steepest descent
with respect to $V$, defined on $[0,\tau+\sigma]$. This contradicts
the maximality of $\Xi$, and concludes our proof. 
\end{proof}

\vbox{

\ssection{The dynamic programming \mbox{inequality}.}

\begin{theorem}\label{dynineqtheo}
Let $\Omega$ be an open subset of $\R^n$,  and let 
$f:\Omega\mapsto\R^n$, \mbox{$L:\Omega\mapsto\R$}
be continuous maps. 
Let \mbox{$V:\Omega\mapsto\R$} be a \mbox{continuous} function that  
satisfies 

\begin{equation}\label{sativis}
-\nabla V(x)\cdot f(x)-L(x)\le 0
\end{equation}
	
\medbreak\noindent on $\Omega$ in the \mbox{viscosity} sense. Then 
for every $x_*\in\Omega$ there exists a curve $\xi$ in
$\Omega$, defined on an interval $I$ that contains $0$, such that 
\begin{itemize}
\item[(1)] $\xi$ is an integral curve of $f$ (that is, 
$\xi$ is \mbox{locally} \mbox{absolutely} continuous and $\dot\xi(t)=f(\xi(t))$
for \mbox{almost} every $t\in I$, from which it follows that 
$\xi$ is \mbox{continuously} differentiable and
$\dot\xi(t)=f(\xi(t))$ for every $t\in I$), 
\item[(2)] $\xi(0)=x_*$,
\item[(3)] $V(\xi(s))\le V(\xi(t))+\int_s^tL(\xi(v))\,dv$ 
whenever $s,t\in I$ and $s\le t$, 
\item[(4)] $\xi$  is right-unbounded.
\end{itemize}
\end{theorem}
}

\begin{proof}
Let $U$ be a set consisting of a single point $\bar u$. Let $\tilde\Sigma$ 
be the augmented control system $(\Omega,U,\tilde f,\tilde L)$,
where $\tilde f(x,\bar u)=f(x)$, $\tilde L(x,\bar u)=-L(x)$. Then 
$\tilde\Sigma$ satisfies all the hypotheses of Theorem \ref{maintech}. 

Let ${\cal V}=-V$. We claim that ${\cal V}$ satisfies 
$$
\sup\{-\nabla{\cal V}(x)\cdot\tilde f(x,u)-\tilde L(x,u):u\in U\}\ge 0
$$
in the viscosity sense. To prove this, we have to pick a
point $x\in\Omega$ and a subdifferential $p$ of ${\cal V}$ at
$x$, and show that 
$$
\sup\{-p\cdot\tilde f(x,u)-\tilde L(x,u):u\in U\}\ge 0\,,
$$
i.e., that 
\begin{equation}\label{sativis-x}
-p\cdot f(x)+L(x)\ge 0\,.
\end{equation}
But, if $p$ is a subdifferential of $-V$ at $x$, and we let 
\mbox{$\pi=-p$}, then it follows that
$\pi$ is a superdifferential of $V$ at $x$. Since $V$ satisfies 
(\ref{sativis}) in the viscosity sense, this implies that
$-\pi\cdot f(x)-L(x)\le 0$. But then 
\mbox{$p\cdot f(x)-L(x)\le 0$}, so $-p\cdot f(x)+L(x)\ge 0$, 
and (\ref{sativis-x}) has been proved.
		
We can therefore apply the trivial Proposition \ref{trivialprop}
to the augmented system $\tilde\Sigma$ and the function ${\cal V}$, 
and \mbox{conclude} 
that there exists a maximal augmented \mbox{trajectory}
$\Xi=(\xi,\xi_0)$ of $\tilde\Sigma$ from $x_*$ of steepest descent 
with respect to ${\cal V}$, 
and then use the nontrivial \mbox{Theorem} \ref{maintech} to conclude that
$\xi$ is right-unbounded. 
The fact that $\xi$ is a trajectory of $\tilde\Sigma$ 
means, of course, that $\xi$ is an \mbox{integral} curve of $f$. 
The steepest descent condition says that
$$
{\cal V}(\xi(s))\ge{\cal V}(\xi(t))+\int_s^t(-L(\xi(v)))\,dv
$$
whenever $0\le s\le t<\tau$. But this says precisely that 
$$
V(\xi(s))\le V(\xi(t))+\int_s^tL(\xi(v))\,dv
$$
whenever $0\le s\le t<\tau$. Hence $\xi$ satisfies all the desired
properties, and our proof is complete. 
\end{proof}

The following result is then a trivial corollary of 
\mbox{Theorem} \ref{dynineqtheo}.

\begin{theorem}\label{dynineqtheo-a}
Let $\hat\Sigma=(\Omega,U,f,L)$ be an 
\mbox{$n$-dimensional} augmented system
such that the map \mbox{$\Omega\ni x\mapsto (f(x,u),L(x,u)$} is continuous
for each $u\in U$. 
Let \mbox{$V:\Omega\mapsto\R$} be a \mbox{continuous} function that  
satisfies (\ref{upeq}) on $\Omega$ in the \mbox{viscosity} sense. Then 
for every $x_*\in\Omega$ and every piecewise constant
function $\eta:[0,\infty\,[\,\mapsto U$ there exists a curve 
$\xi:I\mapsto\Omega$, defined on a subinterval $I$ of $[0,\infty\,[\,$,
such that 
\begin{itemize}
\item[(1)] $0\in I$ and $\xi(0)=x_*$,
\item[(2)] $\xi$ is a trajectory for the control $\eta$ (that is, 
$\xi$ is \mbox{locally} \mbox{absolutely} continuous and 
$\dot\xi(t)=f(\xi(t),\eta(t))$ for \mbox{almost} every $t\in I$)
\item[(3)] $V(\xi(s))\le V(\xi(t))+\int_s^tL(\xi(v),\eta(v))\,dv$ 
whenever $s,t\in I$ and $s\le t$, 
\item[(4)] $\xi$ is right-unbounded. \proofend
\end{itemize}
\end{theorem}

Theorem  \ref{dynineqtheo-a} has the following immediate 
\mbox{consequence}.

\begin{theorem}\label{dynineqtheo-b}
Let $\hat\Sigma=(\Omega,U,f,L)$ be an 
\mbox{$n$-dimensional} augmented system
such that the map \mbox{$\Omega\ni x\mapsto (f(x,u),L(x,u)$} is continuous
for each $u\in U$. 
Let \mbox{$V:\Omega\mapsto\R$} be a \mbox{continuous} function that  
satisfies (\ref{upeq}) on $\Omega$ in the \mbox{viscosity} sense.
Let $\Xi(\xi,\xi_0):I\mapsto\Omega\times\R$ be a locally uniquely 
limiting augmented trajectory of $\hat\Sigma$. Then
the dynamic programming inequality 
$$
V(\xi(s))\le V(\xi(t))+\xi_0(t)-\xi_0(s)
$$
holds for all $s,t\in I$ such that $s\le t$. \proofend
\end{theorem}

\ssection{Proof of Theorem \ref{mainres}}

\noindent According to Theorem \ref{dynineqtheo-b}, the \mbox{dynamic} 
\mbox{programming} inequality holds along every almost locally
uniquely limiting augmented trajectory. The hypothesis 
that every augmented arc has an almost locally uniquely limiting improvement then implies that 
the dynamic \mbox{programming} inequality
holds along every augmented trajectory. If we apply 
the inequality to an augmented arc $\Xi$ that starts at a point 
$x$ and ends at the target,
and use the fact that $V=0$ on the target, we find that 
$V(x)\le J(\Xi)$. Hence $V$ is bounded above by the value function
$\vts$.

To prove that $V\ge\vts$, we pick $x\in\Omega$ and
use \mbox{Proposition} \ref{trivialprop} to conclude that
there exists a maximal augmented \mbox{trajectory}
$\Xi=(\xi,\xi_0)$ of $\tilde\Sigma$ from $x$ of \mbox{steepest} descent 
with respect to ${\cal V}$, and then use  
\mbox{Theorem} \ref{maintech} to conclude that $\xi$ is right-unbounded.  
We then invoke our hypotheses to conclude
that $\Xi$ ends at the target. Then $V(x)\ge J(\Xi)$, so
$V$ is bounded below by the value function, and our proof is complete.\proofend





\cchapter[$\quad$Conclusion]{Conclusion}

\noindent We have shown that the results discovered in the 300 years that
followed Johann Bernoulli's challenge yield new and better insights into
his ``splendid problem.'' First, the Euler-Lagrange equations eliminate the
spurious solutions. Second, classical optimal control theory makes it
possible to settle easily the existence question, and to avoid the {\sl
Ansatz} that the optimal arcs are graphs of functions $y=y(x)$.  Third,
modern differential-geometric optimal control---which is heavily indebted
to Lagrange's idea of coordinate invariance---allows the problem to be
formulated and understood in its proper 4-dimensional setting, the only one
that makes Hamilton's principal function meaningful and avoids the
apparently paradoxes pertaining to the correct choice of endpoint
conditions.  Finally, modern methods, using nonsmooth techniques, allow us
to deal with the very natural ``reflected brachistochrone'' problem, and to
honor the ancestors of Bernoulli's own work---Fermat's principle and
\snell{}' law---by including them in a single, unified formulation of the
necessary conditions.

Would Johann Bernoulli and the other five contributors to the May 
1697 {\em Acta Eruditorum} have have appreciated these new ways of
looking at Johann's ``splendid problem? Would Lagrange, Hamilton and
Weierstrass have liked them?  We hope we have presented strong
evidence that they would have, and leave the final verdict up to the 
reader.  


\ifodd \thepage\clearpage\thispagestyle{empty}\hbox{$\,$}\clearpage
\thispagestyle{empty}\else\clearpage\thispagestyle{empty}\fi
\hbox{$\,$}\vskip 3.5cm

\renewcommand{\theequation}{A.\arabic{equation}}

\begin{center}
{\large\bf Appendix: Some formal definitions.} 
\end{center}

\addcontentsline{toc}{chapter}{Appendix: Some formal definitions}

\gnu{This appendix is very weak. It's got to be completely rewritten
and improved.}


\ssubsectionap{Controlled dynamical systems} 
A {\em controlled dynamical law} (CDL) is a triple $\mathcal{ L}=(S,U,f)$
where $S$---the ``state space'' of $\mathcal{ L}$---is a subset of some
Euclidean space $\rr^n$, $U$---the ``control space''---is a set, and
$f$ is a mapping from $S\times U$ to $\rr^n\cup\{\#\}$.  (Here $\#$ is
a fixed object, subject only to the requirement that $\#\notin\rr^n$.
It stands for ``undefined,'' so that $f(x,u)=\#$ is just a way of
saying,  for a particular $(x,u)$, that $f$ is not defined for that $(x,u)$.
This device allows us to
consider CDL's with a partially defined map $f$.)  An {\em open-loop
control} for a CDL $\mathcal{ L}=(S,U,f)$ is a map $\eta$ from a
subinterval $I$ of $\rr$ to $U$. A {\em trajectory-control pair} (TCP)
of $\mathcal{ L}$ is a pair $(\xi,\eta)$ such that $\eta$ is an open-loop
control and $\xi$ is a locally absolutely continuous map from the
domain $D(\eta)$ of $\eta$ to $S$, with the property that
$\dot\xi(t)=f(\xi(t),\eta(t))$ for almost all $t\in D(\eta)$.  We use
$TCP(\mathcal{ L})$ to denote the set of all TCP's of $\mathcal{ L}$.  (More
precisely, we identify two TCP's 
$\gamma_1=(\xi_1,\eta_1)$,
$\gamma_2=(\xi_2,\eta_2)$ if $D(\eta_1)=D(\eta_2)$, 
$\xi_1(t)=\xi_2(t)$ for all $t\in D(\eta_1)$, and $\eta_1(t)=\eta_2(t)$ for
almost all $t\in D(\eta_1)$. So $TCP(\mathcal{ L})$ really is a set of 
equivalence classes of TCP's.) 

It is clear how to define the {\em restrictions}
$\eta\restr I'$, $\gamma\restr I'$ 
of an open-loop control $\eta$ and of a TCP $\gamma=(\xi,\eta)$ 
to a subinterval $I'$ of $D(\eta)$. Also, 
one can define the {\em concatenation} $\eta_1*\eta_2$ of two 
controls $\eta_1$, $\eta_2$, provided that their domains
$I_1$, $I_2$ are such that $I_1\cup I_2$ is an interval,
$I_1\cap I_2$ is a null set (i.e., ${\rm card}(I_1\cap I_2)\le 1$) 
and $I_1$ lies to the left of $I_2$.   If
$\gamma_1=(\xi_1,\eta_1)$, $\gamma_2=(\xi_2,\eta_2)$ are two TCP's
with domains $I_1$, $I_2$ such that $\eta_1*\eta_2$ is defined, 
then the concatenation $\gamma_1*\gamma_2$ is defined---in an obvious 
way---if $\xi_1(t)=\xi_2(t)$ for $t\in I_1\cap I_2$, 
and the curve $\xi$ whose graph is the union of the graphs of
of $\xi_1$ and $\xi_2$ is locally absolutely continuous
on $I_1\cup I_2$.  (For example, if $I_i=[a_i,b_i]$ are compact intervals,
then $\gamma_1*\gamma_2$ is defined iff 
$b_1=a_2$ and $\xi_1(b_1)=\xi_2(a_2)$.) 

A {\em controlled dynamical system} (CDS) is a 4-tuple
$\Sigma=(S,U,f,\mathcal{ A})$ such that $(S,U,f)$ is a CDL and 
$\mathcal{ A}$---the set of ``admissible TCP's of $\Sigma$''---is a subset of
$TCP(M,U,f)$ which is closed under restrictions (that is, if
$\gamma\in\mathcal{ A}$ and $\gamma'=\gamma\restr I'$ for some subinterval
$I'$ of the domain of $\gamma$, then $\gamma'\in\mathcal{ A}$).  Often,
$\mathcal{ A}$ is defined by singling out a ``class of admissible controls.''
That is, we specify a class $\mathcal{ U}$ of open-loop
controls, and then take $\mathcal{ A}=\mathcal{ A}_{\mathcal{ U}}$,
where $\mathcal{ A}_{\mathcal{ U}}$ is the set of all TCP's $(\xi,\eta)$ 
of $(S,U,f)$ such that $\eta\in\mathcal{ U}$. 

\ssubsectionap{Cost functionals, optimal control problems, and the value}
A {\em cost functional} for a CDS
$(S,U,f,\mathcal{ A})$ is a function $J\!:\!\mathcal{ A}\!\mapsto\!
\rr\cup\{+\infty\}$.  The functional $J$ is {\em additive}
if $J(\gamma_1*\gamma_2)=J(\gamma_1)+J(\gamma_2)$ whenever 
$\gamma_1,\gamma_2$ belong to $\mathcal{ A}$ and 
$\gamma_1*\gamma_2$ is defined and belongs to $\mathcal{ A}$. 

A {\em controlled dynamical system with cost} (CDSWC) is a 
five-tuple 
$$
\mathcal{ C}=(S,U,f,\mathcal{ A},J)
$$
such that $(S,U,f,\mathcal{ A})$ is a CDS and $J$ is a cost functional 
for $(S,U,f,\mathcal{ A})$. 

To a CDSWC $\mathcal{ C}=(S,U,f,\mathcal{ A},J)$ we can associate {\em optimal
control problems} in various ways.  Given any subset $\mathcal{ B}$ of
$\mathcal{ A}$ we can consider ``the problem $P_{\mathcal{ C}}(\mathcal{ B})$ of
minimizing $J(\gamma)$ for $\gamma\in\mathcal{ B}$.''  The {\em value}
$V_{\mathcal{ C}}(\mathcal{ B})$ of this problem is the extended real number
\begin{equation}
V_{\mathcal{ C}}(\mathcal{ B})=\inf\{J(\gamma):\gamma\in\mathcal{ B}\}\,.
\end{equation}
Then
$V_{\mathcal{ C}}(\mathcal{ B})=+\infty$ iff $J(\gamma)=+\infty$ for all
$\gamma\in\mathcal{ B}$.  (In particular, $V_{\mathcal{ C}}(\mathcal{ B})=+\infty$
when $\mathcal{ B}$ is empty.)  It is also possible for
$V_{\mathcal{{C}}}(\mathcal{ B})$ to be $-\infty$, which will happen, of
course, iff $J$ is not bounded below on $\mathcal{ B}$.  A $\gamma\in\mathcal{
B}$ will be called a {\em solution} of $P_{\mathcal{ C}}(\mathcal{ B})$ if
$J(\gamma)=V_{\mathcal{ C}}(\mathcal{ B})$. 

\ssubsectionap{Generalized solutions} Naturally, the infimum defining
$V_{\mathcal{C}}(\mathcal{ B})$ need not be attained as a minimum, so a
solution of $P_{\mathcal{ C}}(\mathcal{ B})$ may fail to exist. IN that
case, it is often 
convenient to enlarge the set of TCP's of a system by introducing new,
``generalized trajectory-control pairs,'' so that in the enlarged space the
problem has a true solution.  
\begin{example}{\rm \label{deltafun}
For a trivial example of this situation,
consider the problem of reaching $0\in\rr$ in ``minimum time'' from an
initial point $x\in\rr$, when the dynamics is $\dot x=u$, $u\in U$, with
$U=\rr$.  It is clear that when $x\neq 0$ the value of this problem is
zero, but this value is not attained as a true minimum by any 
trajectory-control pair
arising from an ``ordinary'' control, where ``ordinary''
means, for example, Lebesgue integrable.  On the other hand, if we allow
the ``controls'' to contain impulses, then we can take, for $x\in\rr$,
$\eta_x=-x.\delta$, where ``$\delta$'' is Dirac's Delta function, and 
the control $\eta_x$ obviously steers $x$ to $0$ in time $0$.
}\end{example}
We can define
$\hat{\mathcal{B}}$ to be the set of all sequences
$\Gamma=\{\gamma_j\}_{j=1}^\infty$ of members of $\mathcal{ B}$ for which
the limit $\hat J(\Gamma)=\lim_{j\to\infty}(J(\gamma))$ exists. We can
then think of the members of $\hat{\mathcal{B}}$ as ``generalized TCP's,''
and of the functional $\hat
J:\hat{\mathcal{B}}\to\rr\cup\{-\infty,+\infty\}$ as the extension of $J$
from $\mathcal{ B}$ to $\hat{\mathcal{B}}$.  (There is a an obvious canonical
embedding of $\mathcal{ B}$ in $\hat{\mathcal{B}}$, obtained by regarding each
$\gamma\in\mathcal{ B}$ as a constant sequence.)  It is clear that if
$\mathcal{ B}\neq\emptyset$ then $\hat J$ attains a minimum on
$\hat{\mathcal{B}}$ at some $\Gamma\in\hat{\mathcal{B}}$, and
$\hat{J}(\Gamma)=V_{\mathcal{ C}}(\mathcal{ B})$.  In order words, if
$\mathcal{{B}}\neq\emptyset$ then one can always construct a
{\em{}minimizing sequence} $\Gamma$, i.e., a sequence $\{\gamma_j\}$ in
$\mathcal{ B}$ whose costs $J(\gamma_j)$ converge to $V_{\mathcal{ C}}(\mathcal{
B})$.  If one regards such a $\Gamma$ as a solution of
$P_{\mathcal{{C}}}(\mathcal{ B})$ in a ``generalized sense,'' or as a solution
of the ``extended problem'' in which we seek to minimize $\hat J$ on
$\hat{\mathcal{B}}$ rather than minimizing $J$ on $\mathcal{ B}$, then the
original problem $P_{\mathcal{ C}}(\mathcal{ B})$ always has a generalized
solution, and the entended problem always has a solution, as long as
$\mathcal{ B}\neq\emptyset$. In this abstract setting, the concept of a
generalized solution is rather trivial and uninteresting, but it turns
out that in many cases one can realize generalized solutions as much
more concrete objects, e. g.  by showing that minimizing sequences
converges in some sense, such as that of Schwartz distributions or
relaxed trajectories.
\begin{remark}{\rm 
In the previous example the true solution of the minimization 
problem was a distribution in the sense of L. Schwartz's theory.
The reader should not extrapolate from this example and imagine that for 
general optimal control problems the appropriate class of generalized
controls is that of Schwartz distributions. It turns out that one needs 
a rather different class of generalized objects.   This is discussed 
in detail in papers by A. Bressan, F. Rampazzo, and their coworkers,
cf. \cite{bressra1,bressra2,bressra3,bressra4,bressra5,bressra6}.\proofend
}\end{remark}


\ssubsectionap{The value function (i.e., Hamilton's principal function)}
A particularly important class of optimal control problems is that of
{\em fixed endpoint problems}. Given $x,x'\in S$, we can consider the
set $\mathcal{ B}(x,x')$ of all TCP's $\gamma=(\xi,\eta)\in\mathcal{ A }$ such
that $\gamma$ goes from $x$ to $x'$. (Precisely,
$\gamma\in\mathcal{{B}}(x,x')$ iff $\gamma\in\mathcal{ A}$, $D(\eta)$ is a
compact interval $[a,b]$, $\xi(a)=x$, and $\xi(b)=x'$.)  We will use
$P_{\mathcal{ C}}(x,x')$---rather than 
$P_{\mathcal{ C}}(\mathcal{ B}(x,x'))$---to denote this problem, and 
will refer to it by means of
self-explanatory expressions such as ``the problem of driving $x$ to
$x'$ with minimum cost $J(\gamma)$, subject to the dynamical
constraint $\dot x=f(x,u)$ and the requirement that $\gamma\in\mathcal{
A}$.''  The value of $P_{\mathcal{ C}}(x,x')$ will be written
$V_{\mathcal{{C}}}(x,x')$, and
$V_{\mathcal{{C}}}:S\times{}S\mapsto(\rr\cup\{-\infty,+\infty\})
\times(\rr\cup\{-\infty,+\infty\})$ is the {\em value function} of the
CDSWCF $(S,U,f,\mathcal{ A},J)$.  

The value function, as defined here, is essentially the same as {\bem
Hamilton's principal (or characteristic) function}, except that the modern
definition 
\bi
\item{} avoids the issue of the existence of a minimizing arc, by
taking $V_{\mathcal{{C}}}(x,x')$ to be the {\em infimum} of the costs of all
arcs from $x$ to $x'$, 
\item{} is formulated directly for control problems
rather than for the more restricted calculus of variations, 
\ei
and
\bi
\item{} allows the
values of $V_{\mathcal{{C}}}(x,x')$ to be $-\infty$ and $+\infty$, the latter
case arising when there are no trajectories joining $x$ to $x'$\Ft{This
situation is, of course, impossible in the classical calculus of
variations, but it can easily occur in control problems.}.
\ei
We emphasize that a ``controlled dynamical system with cost'' (CDSWC)
$\mathcal{ C}$ is not at all the same as an ``optimal control problem.''
A CDSWC gives rise to {\bem many} optimal control problems, such as for
example the fixed-endpoint problems $P_{\mathcal{ C}}(x,x')$ described
before, as well as other problems obtained by making different choices
of $\mathcal{{B}}$.  (For example, one could choose $\mathcal{ B}$ to be the
set of all TCP's in $\mathcal{ A}$ that start at some given subset $A$ and
end at some other subset $B$ of $S$, obtaining the problem of
``joining $A$ to $B$ with minimum cost by a TCP $\gamma$ of $\dot
x=f(x,u)$, $u\in U$, $\gamma\in\mathcal{ A}$.'')  An optimal control
problem has a {\em value}, which is an extended real number, and if
the set of trajectories is $\neq\emptyset$ then the problem may have a
{\em solution}---i.e., a minimizer---or at least a minimizing
sequence or generalized solution, which {\em realizes} this value.  A
CDSWC $\mathcal{ C}$ has a {\em value function} $V_{\mathcal{ C}}$.  The
object that realizes the value is something called an ``optimal
synthesis,'' by which it is meant---roughly---a family
$\{\gamma_{x,x'}\}_{(x,x')\in C}$ in $\mathcal{ A}$ that selects, for each
$(x,x')\in C$ (where
$C=\{(x,x')\in{}S\times{}S:\mathcal{{B}}(x,x')\neq\emptyset\}$, 
an---ordinary or generalized---solution $\gamma_{x,x'}$ of the problem
$P_{\mathcal{ C}}(x,x')$.  (This is only a first approximation to the
concept of a synthesis.  In the control literature one looks for
syntheses having extra regularity properties, i.e., in selecting
solutions $\gamma_{x,x'}$ that depend ``nicely''---e.g. continuously,
or smoothly, or piecewise smoothly---on $x$ and $x'$.)

An important type of cost functionals are the {\em Lagrangian costs},
obtained when $J$ is of the form 
\begin{equation}\label{lagcost}
J(\xi,\eta)=\int_{D(\eta)}L(\xi(t),\eta(t))dt\,,
\end{equation}
for some function $L:S\times U\mapsto\rr\cup\{+\infty\}$
having the property that $L$ is ``admissible for $\mathcal{ A}$,''  i.e.,
that for every $(\xi,\eta)\in\mathcal{ A}$ the integrand of (\ref{lagcost}) 
is measurable and such that 
$\int_{D(\eta)}\max(0,-L(\xi(t),\eta(t)))dt<+\infty$. 

Lagrangian cost functionals are always additive. 
Within the Lagrangian costs, an important example is {\em time},
i.e., the functional $J$ obtained by taking $L\equiv 1$.

If the cost functional $J$ is additive, and the class $\mathcal{ A}$ is
closed under concatenations, then the value function $V_{\mathcal{ C}}$ 
satisfies
\bi
\item[a.] the {\bem dynamic programming inequality}
\begin{equation}
V_{\mathcal{ C}}(x,x'')\le V_{\mathcal{ C}}(x,x')+V_{\mathcal{ C}}(x',x'')
\,\,\,\,{\rm for\,\,\,all}\,\,\,x,x',x''\in S\,,
\end{equation}
\item[b.] the {\bem principle of optimality}, which says that {\em
any part of an optimal trajectory is optimal}. (That is, if
$\gamma=(\xi,\eta)\in\mathcal{ A}$ is a solution of the problem $P_{\mathcal{
C}}(x,x')$, $D(\eta)=[a,b]$, $a\le\alpha\le\beta\le b$,
$y=\xi(\alpha)$ and $y'=\xi(\beta)$, then the restriction
$\gamma\restr[\alpha,\beta]$ of $\gamma$ to the interval
$[\alpha,\beta]$ is a solution of $P_{\mathcal{ C}}(y,y')$.)  
\ei 
A CDSWC $\mathcal{ C}=(S,U,f,\mathcal{ A},J)$ is of the {\em classical
calculus of variations type} if $S$ is an open subset of $\rr^n$,
$U=\rr^n$, $f(x,u)=u$, and $J$ is Lagrangian. In that case, the CDL
$(S,U,f)$ is rather trivial, so nothing interesting really really
happens until one brings in the cost $J$, i.e., the Lagrangian $L$.

We illustrate the above definitions with a simple example.  Consider
the formal expression $\dot x=u$, where we want to think of $x$ and
$u$ as real-valued variables, $x$ being the ``state'' and $u$ the
``control.''  Let us take $S$, the state space, to be $\rr$.  If we
specify in addition a set $U$ of values of the control $u$, then we
get a CDL $(S,U,f)$, by taking $f$ to be given by $f(x,u)=u$.  If, for
example, we take $U$ to be the interval $[-1,1]$, then we can obtain
different CDS's---often referred to as ``the system $\dot x=u$, with
state $x\in\rr$ and control constraint $|u|\le 1$''---by making
different choices of $\mathcal{ A}$. One standard way to do that would be
to choose a class $\mathcal{ U}$ of open-loop controls, and let $\mathcal{
A}=\mathcal{ A}_{\mathcal{ U}}$. One could, for example, take $\mathcal{ U}$ to be
the class of all continuous $U$-valued maps defined on subintervals of
$\rr$, or the set of all maps of class $C^k$ for some $k$, or the
class of all polynomial maps. In all these cases, the resulting class
$\mathcal{ A}_{\mathcal{ U}}$ will not be closed under concatenations.  One
could also take $\mathcal{ U}$ to be the class of all measurable
$U$-valued maps, or that of all piecewise constant ones (or piecewise
continuous, or piecewise $C^1$).  For these classes, $\mathcal{
A}_{\mathcal{{U}}}$ is be closed under concatenations. By restricting
$\mathcal{ U}$ further to consist of controls $\eta$ whose domain
$D(\eta)$ is a compact interval, one can associate a CDSWC to each of
the above CDS's, by taking $J$ to be time.  The resulting optimal
control problems are the {\em minimum time problems} associated to the
CDS's $(\rr,[-1,1],\mathcal{ A}_{\mathcal{ U}})$.  The value function for all
these problems is given by $V_{\mathcal{ C}}(x,x')=|x-x'|$.

One could do the same thing with $U=\rr$, in which case we would be
considering ``the CDS $\dot x=u$ with $x\in\rr$ and unrestricted
$u\in\rr$,'' with the proviso, of course, that to specify a CDS we
have to be precise about the choice of the class $\mathcal{ A}$. If, for
example, we choose $\mathcal{ U}$ to be the set of Lebesgue integrable
controls---that is, $\mathcal{ U}=\cup_{-\infty<a\le b<+\infty
}L^1(\rr,\rr)$---and take $\mathcal{ A}=\mathcal{ A}_{\mathcal{ U}}$, then
we get the system $\dot x=u$, with $x\in\rr$, $u\in\rr$, and $u(\cdot)\in
L^1$.  If we take the cost functional to be time, as before, we get a
CDSWC, the minimum-time problem for the system $\dot x=u$ with
unrestricted $u\in\rr$ and $u(\cdot)$ integrable.  The value function
for this system is of course given by $V_{\mathcal{ C}}(x,y)=0$, since 
any two points can be connected in arbitrarily small time by using very
large controls. On the other hand, if $x\neq y$ then a minimizer does
not exist but, as explained in Example \ref{deltafun}, 
one can arrange for the problem to have a true solution
by considering a suitable extension $\hat{\mathcal{U}}$ of $\mathcal{ U}$ to a
space of ``generalized controls,'' which in this case can be taken to be
any space which is large enough to contain all multiples of the 
Delta function. (For example, we could take 
$\hat{\mathcal{U}}$ to be the set of all distributional derivatives
of Borel measures on $\rr$.) 

\ssubsectionap{Systems on manifolds, control bundles, partially defined vector
fields} 
The above definitions can be extended in  various ways. For example, 
one can allow $S$ to be a subset of a manifold $M$ other than $\rr^n$.
In  that case, the condition that $f$ be a map from $S\times U$ to 
$\rr^n\cup\{\#\}$ is replaced by the requirement that $f$ be 
a---possibly partially defined---map on $S\times U$ such that 
$f(x,u)$ is a tangent vector to $M$ at $x$ whenever $x\in S$, $u\in U$,
and $f(x,u)$ is defined.

One can also allow $U$ to be a ``control bundle'' rather than a
``control set.''  That is, $U$ can be taken to be a set $U(x)$ that
depends on the point $x\in S$. In this case, the concept of an
``open-loop control'' ceases to be meaningful, but that of a TCP makes
perfect sense, exactly as in the case of a state-independent control space.

A special case of the control bundle situation is that of {\em
calculus of variations problems on manifolds}, i.e., CDSWC's where the
state space is a manifold $M$, the control bundle $U$ is the tangent
bundle of $M$ (that is, $U(x)$ is the tangent space at $x$ for each
$x$), the dynamics $f$ is given by $f(x,u)=u$, and the cost functional
is Lagrangian.  Another important example of CDSWC's is obtained by
taking the state space $M$ to be a Riemannian manifold, $U(x)$ to be
the closed unit ball in the tangent space to $M$ at $x$, and
$f(x,u)=u$.  Taking the cost functional to be time, we see that the
minimizers are the arcs of minimum length, and the value function is
the Riemannian distance.  This example can be generalized in an obvious way 
to case of a {\em subriemannian manifold}, i.e., a smooth manifold 
$M$ endowed with a smooth subbundle $E$ of the tangent bundle,
and with a Riemannian metric defined on $E$. As in the Riemannian case,
one can take $U(x)$ to be the closed unit ball in $E(x)$, and define
$f(x,u)=u$.  

In addition, it is perfectly possible to allow $f$---and the
Lagrangians $L$---to depend on time.  

Finally, one can try to define analogues of all these concepts for
state spaces $S$ more general than subsets of finite-dimensional
manifolds. For example, one can take $S$ to a be a Banach 
space---often a function space---and be led to the very rich and active
field of ``distributed parameter control.'' Or one can consider many
other types of ``controlled systems,'' such as discrete-time 
or stochastic systems.  

\cchapter[$\quad\,\,$Lots of figures that could be useful]{Lots of figures 
that could be useful}


\bigskip
\[
\epsfxsize=12cm\epsfbox{sixtypes.eps}
\]
\figurecap{\snell's law\label{sixtypes-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{switchdubins.eps}
\]
\figurecap{\snell's law\label{switchdubins-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{abbb.eps}
\]
\figurecap{\snell's law\label{abbb-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{abbbb.eps}
\]
\figurecap{\snell's law\label{abbbb-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{twosing.eps}
\]
\figurecap{\snell's law\label{twosing-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{withsing1.eps}
\]
\figurecap{\snell's law\label{withsing1-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{withsing2.eps}
\]
\figurecap{\snell's law\label{withsing2-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{rotation.eps}
\]
\figurecap{\snell's law\label{rotation-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{appenddubins1.eps}
\]
\figurecap{\snell's law\label{appenddubins1-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{appenddubins2.eps}
\]
\figurecap{\snell's law\label{appenddubins2-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{crsoptdubins.eps}
\]
\figurecap{\snell's law\label{crsoptdubins-fig}}

\bigskip
\[
\epsfxsize=12cm\epsfbox{classenv.eps}
\]
\figurecap{\snell's law\label{classenv-fig}}

\addcontentsline{toc}{chapter}{References}

\clearpage

\bibliographystyle{amsalpha}
\begin{thebibliography}{A}

%\begin{thebibliography}{999}

\bibitem{agra1}{Agrachev,~A.~A. and R.~V.~Gamkrelidze,
\textit{A second-order optimality principle for a time-optimal problem.}
Math. Sbornik \textbf{100}, 142, 1976.
[GNU: UPDATE THIS REFERENCE, IF NEEDED. PAGE NOS. MISSING.]}

\bibitem{agra2}{Agrachev,~A.~A. and R.~V.~ Gamkrelidze,
\textit{The exponential representation of flows and the chronological
calculus.} Math. Sbornik \textbf{109}, 149, 1978.
[GNU: UPDATE THIS REFERENCE, IF NEEDED. PAGE NOS. MISSING.]}

\bibitem{agra3}{Agrachev,~A.~A. and R.~V.~Gamkrelidze,
\textit{The Morse index and the Maslov index for smooth control systems.}
Doklady Akad. Nauk USSR \textbf{287}, 1986.
[GNU: UPDATE THIS REFERENCE, IF NEEDED. PAGE NOS. MISSING.]}

\bibitem{anglin}{Anglin,~W.~S., \textit{Mathematics: A Concise History and
Philosophy.}  Springer-Verlag, New York, 1994.}

\bibitem{lavreball}{Ball,~J.~M. and V.~J.~Mizel,
\textit{One-dimensional variational problems whose minimizers do not
satisfy the Euler-Lagrange equation.}
Arch. Rational Mech. Anal. \textbf{63}, 1985, pp.~273--294.}


\bibitem{BARD}{Bardi, M., and I. Capuzzo-Dolcetta, {\em Optimal
\mbox{Control} and Viscosity
Solutions of Hamilton-Jacobi-Bellman Equations}, Birkh{\"a}user,
Boston, 1997.}

\bibitem{BDL97}{Bardi, Martino, and Francesca Da Lio, ``On the Bellman
equation for some unbounded \mbox{control} problems.'' {\em NoDEA
Nonlinear Differential \mbox{Equations} Appl.} {\bf 4} (1997),
pp.~276-285.}

\bibitem{bell}{Bell,~E.~T., \textit{Men of Mathematics.} Simon and Schuster,
New York, 1937.} 

\bibitem{belldev}{Bell,~E.~ T., \textit{The development of Mathematics.}
McGraw-Hill, New York, 1940.} 

\bibitem{belljac}{Bell,~D.~J. and D.~H.~Jacobson,
\textit{Singular Optimal Control Problems.} Academic Press,
New York, 1973.}

\bibitem{beltrami}{Beltrami,~E., \textit{Saggio di interpetazione 
della geometria non-euclidea.} Giornale di Matematiche \textbf{VI}, 1868,
pp.~284-312 (in Itialian). Eng. transl. in Stillwell, \cite{stillwell-hyp},
pp.~7--34.}

\bibitem{Bennett}{S. Benett, \textit{A History of Control Engineering.} 
Peter Pelegrinus, 1979.}

\bibitem{berkovitz}{Berkovitz,~L.~D., \textit{Optimal Control Theory.}
Springer-Verlag, New York, 1974.}

\bibitem{berna}{Berna,~P., N.~Conze, M.~Duflo, M.~L'evy-Nahas, M.~Rais,
P.~Renouard, M.~Vergne, \textit{Repr\'esentations des groupes r\'esolubles.}
Dunod, Paris, 1972.}

\bibitem{bernoulli-a}{Bernoulli,~Johann, \textit{Curvatura radii
in diaphanis nonuniformibus $\ldots$ [The curvature of a ray in nonuniform
media $\ldots$ ].} Acta Eruditorum, 
May 1697, pp.~206--211. (\textit{Opera omnia} \textbf{I}, 
pp~187--193}

\bibitem{bernoulli-b}{Bernoulli,~Jakob, \textit{Solutio problematum
fraternorum $\ldots$ una cum propositione reciproca aliorum
[Solution of problems of my brother $\ldots$ together with 
the proposition of others in turn].} 
Acta Eruditorum,  May 1697, pp.~211--217. (\textit{Opera} \textbf{I}, 
pp~768--778.}

\bibitem{bliss}{Bliss,~G.~A., \textit{Calculus of Variations.} Open Court
Publishing Co., Chicago, 1925.}

\bibitem{bolza}{Bolza,~O., \textit{\"Uber  der `Anormalen Fall' beim
Lagrangeschen und Mayerschen Problem mit gemischten Bedingungen 
und variablen Endpunkten.} Math. Ann. \textbf{LXXIV}, 1913, 
pp.~430--446.}

\bibitem{bonola}{Bonola,~R., \textit{Non-Euclidean Geometry.}
(Italian original: \textit{La Geometria non-Euclidea,} 
The Open Court Publishing Co., 1912.) Eng. transl. by H.~S.~Carslaw,
Dover, New York, 1955.}

\bibitem{borelli}{Borelli,~G.~A., \textit{Euclides restitutus.}
Pisa, Italy, 1658.}

\bibitem{bress2}{Bressan,~A., \textit{On differential relations
with lower continuous right-hand side. An existence theorem.}
J. Diff. Equations \textbf{37}, 1980, pp.~89--97.}

\bibitem{bress3}{Bressan,~A., \textit{Solutions of 
lower semicontinuous differential inclusions on closed sets.}
Rend. Sem. Mat. Univ. Padova \textbf{69}, 1983, pp.~99--107.}

\bibitem{bress1}{Bressan,~A.,
\textit{A high-order test for optimality of bang-bang controls.}
S.I.A.M. J. Control Opt. \textbf{23}, 1985, pp.~38--48.}

\bibitem{bress4}{Bressan,~A., \textit{Directionally continuous
selections and differential inclusions.} 
Funk. Ekvac. \textbf{31}, 1988, pp.~459--470.}

\bibitem{bress5}{Bressan,~A. and~ G.~Colombo, \textit{Extensions and
selections of  maps with decomposable values.}  Studia Math. \textbf{90},
1988, pp.~69--86.}

\bibitem{bressra1}{Bressan,~A. and F.~Rampazzo, \textit{On differential 
systems with vector-valued impulsive controls.} Boll. Unione Matematica
Italiana B(7) \textbf{3}, 1988, pp.~641--656.}

\bibitem{bress6}{Bressan,~A., \textit{On the qualitative
theory of lower semicontinuous differential inclusions.}
J. Diff. Equations \textbf{77}, 1989, pp.~379--391.}

\bibitem{bress7}{Bressan,~A., \textit{Upper and lower semicontinuous 
differential inclusions: a unified approach.} In \textit{Nonlinear
Controllability and Optimal Control}, H.~J.~Sussmann Ed., Marcel Dekker, 
New York, 1990, pp.~21--31.}

\bibitem{bressra2}{Bressan,~A. and F. Rampazzo, ``Impulsive control systems
with commutative vector fields,'' {\it J. Optim. Theory Appl.} {\bf 71}
(1991), pp.67-83.} 

\bibitem{bressra3}{Bressan,~A. and F. Rampazzo, ``Impulsive control systems
without commutativity assumptions, {\it J. Optim. Theory Appl.} {\bf 81}
(1994), pp.~435-457.}

\bibitem{brockett}{Brockett, R. W., \textit{System theory on group manifolds 
and coset spaces.} {\it S.I.A.M.~J.~Control} \textbf{10}, pp.~ 265-284 
(1972).}

\bibitem{buttazzo}{Buttazzo,~G., V.~Ferone, and B.~Kawohl,
\textit{Minimum problems over sets of concave functions and related
questions.} Math. Nachr. \textbf{173}, 1995, pp.~71--89.} 

\bibitem{cajori}{Cajori, F., \textit{A History of Mathematics.}
Chelsea Pub. Co., New York, 1980.}

\bibitem{cambini} {Cambini,~A., and S.~ Querci, \textit{Equazione 
differenziale
del primo ordine con secondo membro discontinuo rispetto all'incognita.}
{\it Rend. Ist. Mat. Univ. Trieste} {\bf 1}, 1969, pp.~89-97.}

\bibitem{canudas}{Canudas~de~Wit,~C.~A., \textit{Adaptive control for 
partially know systems: theory and applications}. Elsevier, 
Amsterdam-New York, 1988.}

\bibitem{carath}{Carath\'eodory,~C., \textit{Calculus of Variations
and Partial Differential Equations of the First Order. Part II:
Calculus of Variations.} Eng. transl. by R.~B.~Dean, 2nd edition,
Holden Day, Inc., San Francisco, 1967. (Originally published as
\textit{Variationsrechnung und Partielle Differentialgleichungen
erster Ordnung.} B.~G.~Teubner, Berlin, 1935.}

\bibitem{casti}{Casti, J. L., \textit{Alternate realities: mathematical
models  of nature and man}.  Wiley-Interscience, New York, 1989.}

\bibitem{cataldi}{Cataldi,~P.~A., \textit{Operetta delle linee rette 
equidistanti et non equidistanti.} Bologna, Italy, 1603.}

\bibitem{cataldi-ag}{Cataldi,~P.~A., \textit{Agguinta all'operetta delle 
linee rette equidistanti et non equidistanti.} Bologna, Italy, 1604.}

\bibitem{cesari}{Cesari, L., \textit{Optimization---Theory and Applications}.
Springer-Verlag, New York, 1983.}

\bibitem{clarkeminimal}{Clarke,~F.~H., \textit{The maximum principle 
under minimal hypotheses.} S.I.A.M. J. Control Optim. \textbf{14},n
1976, pp.~1078--1091.}

\bibitem{clarkebook}{Clarke,~F.~H., \textit{Optimization and Nonsmooth
Analysis.} Wiley Interscience, New York, 1983.}

\bibitem{clarvin}{Clarke, F.~H. and R.~B.~Vinter, ``The relationship
between the maximum principle and dynamic programming,'' 
{\sl S.I.A.M. J. Control
Optim.} {\bf 25} (1987), no. 5, pp.~1291-1311.}

\bibitem{clarkemethods}{Clarke, F.~H., \textit{Methods of Dynamic and
Nonsmooth  Optimization.} CBMS-NSF Regional Conference Series in Applied
Mathematics \textbf{57}. Society for Industrial and Applied Mathematics
(S.I.A.M.), Philadelphia, PA, 1989.}   

\bibitem{Cla-Le-a}
{Clarke, F.~H., and Yu.~S.~Ledyaev, ``Mean value \mbox{inequalities}.''
{\em   Proc. Amer. Math. Soc.} {\bf 122}  (1994),  no. 4, 
pp.~1075--1083.}

\bibitem{Cla-Le-b}
{Clarke, F.~H., and Yu.~S.~Ledyaev, ``Mean value \mbox{inequalities} in 
Hilbert space.''  {\em Trans. Amer. Math. Soc.}  {\bf 344}  (1994), 
no. 1, pp.~307--324.} 

\bibitem{clarkebooknew}{Clarke,~F.~H., Yu.~S.~Ledyaev, R.~J.~Stern,
and P.~R.~Wolenski, \textit{Nonsmooth Analysis and Control Theory.}
\underline{Graduate Texts in Mathematics} \textbf{178},
Springer-Verlag, New York,  1998.} 

\bibitem{conan}{Conan Doyle, A., \textit{Silver Blaze.} In \textit{The 
Complete Sherlock Holmes,} Doubleday, New York, 1930.}

\bibitem{coolidge}{Coolidge,~J.~L., \textit{A History of Geometrical Methods.}
Dover , New York, 1963. (Original edition; Oxford Univ. Press, 1940.)}   

\bibitem{descartes}{Descartes,~R., \textit{Dioptrique}, 1637.}

\bibitem{DGKF}{Doyle,~J.~C., K.~Glover, P.~Khargonekar and B.~A.~Francis,
\textit{State space solutions to standard $\mathfrak{H}_2$ and $\Hinf$ control
problems.} IEEE Transactions on Automatic Control, Vol. 34, 1989,
pp.~831-847.}
  
\bibitem{dubins}{Dubins,~L.~E., \textit{On curves of minimal length with a
constraint on average curvature and with prescribed initial and
terminal positions and tangents.}  Amer. J. Math. \textbf{79}, 1957,
pp.~497-516.}

\bibitem{dugundji}{Dugundji,~J., \textit{Topology.}  Allyn and Bacon, Inc., 
Boston, 1966.}

\bibitem{dyson}{Dyson,~F.~J., \textit{Missed opportunities.}
Bull. Amer. Math. Soc. \textbf{78}, 1972, pp.~635-652.}

\bibitem{markus}{Elworthy,~K.~D., W.~N.~Everitt and E.~B.~Lee Eds., 
\textit{Differential equations, dynamical systems, and control science : a
Festschrift in honor of Lawrence Markus.}  Marcel Dekker, 
New York, 1994.}

\bibitem{brit}{\textit{Encyclop\ae{}dia Britannica; Macrop\ae{}dia}.
Vol. 17. Chicago, 1980.}

\bibitem{euler}{Euler,~L., \textit{Methodus inveniendi $\ldots$ accepti.}
Lausanne, Geneva, 1744; \textit{Opera omnia}, ser. I, vol. 25, 1952.}

\bibitem{fermat}{Fermat,~P., \textit{Letters to Martin Cureau de la Chambre.}
In \textit{Oeuvres, II}, pp.~355--359 (1657) and pp.~457--463 (1662).}

\bibitem{fleming}{Fleming, W.~H., and H.~Mete Soner, 
\textit{Controlled Markov Processes and Viscosity Solutions.}
New York, Springer-Verlag, 1993.}

\bibitem{fuller}{Fuller,~A.~T., \textit{Study of an optimum nonlinear 
system.} J. Electronics Control \textbf{15}, 1963, pp.~63--71.}

\bibitem{gabakir}{Gabasov,~V., and F.~M.~Kirillova, \textit{High-order 
necessary conditions for optimality.} S.I.A.M. J. Control \textbf{10},
1972, pp.~127--168.}

\bibitem{gandz}{Gandz,~S., \textit{Die Harpedonaptai oder Seilspanner.} 
Quellen und Studien Gesch. der Math. \textbf{B~1}, 1930, 
pp.~255-277.}     

\bibitem{giaq}{Giaquinta,~M., and S.~Hildebrandt, \textit{Calculus of 
Variations.} Springer-Verlag, Berlin-New York, 1996 (2 vols.).}

\bibitem{goh}{Goh,~B.~S., \textit{The second variation for the Bolza 
problem.} S.I.A.M. J. Control \textbf{4}, 1966, pp.~309--325.}

\bibitem{goldstein}{Goldstein,~H., \textit{Classical Mechanics.}
Addison-Wesley, 1980.}

\bibitem{goldstine}{Goldstine,~H.~H., \textit{A History of the Calculus of 
Variations from the 17th to the 19th Century.} Springer-Verlag, 
New York, 1981.}

\bibitem{guillen}{Guillen,~M., \textit{Five equations that changed the world:
the power and poetry of mathematics.}  New York, Hyperion, 1995.}

\bibitem{heath}{Heath,~T.~L., \textit{A Manual of Greek Mathematics.}
Clarendon Press, Oxford, 1931.}

\bibitem{heeres}{Heeres,~J.~E., \textit{The Discovery of Tasmania:
Extracts from the Journal of Abel Janszoon Tasman.} Government
Printer, Hobart, Tasmania, 1985.}

\bibitem{hermann}{Hermann,~R., \textit{On the accessibility problem in control
theory.} In International Symposium on Nonlinear Differential
Equations and Nonlinear Mechanics, J.~P.~LaSalle and S.~Lefschetz Eds.,
Academic Press, New York, 1963.}

\bibitem{hermes}{Hermes,~H., \textit{Lie algebras of vector fields and local 
approximation of attainable sets.} S.I.A.M. J. Control and
Optimization \textbf{16}, 1974, pp.~715--727.}

\bibitem{hestenes}{Hestenes,~M.~R., \textit{A general problem in the calculus
of variations with applications to paths of least time.}  Rand Corporation
RM-100, ASTIA Document No. AD-112382, 1950.}

\bibitem{hestenespap}{Hestenes,~M.~R., \textit{Elements of the calculus of
variations.} In \textit{Modern Mathematics for the Engineer,}  
E.~F.~Beckenbach Ed., McGraw-Hill, New York, 1956.}

\bibitem{hestenesbook}{Hestenes,~M.~R., \textit{Calculus of Variations
and Optimal Control Theory.} Wiley, 1966.}

\bibitem{hughes}{Hughes,~ R., \textit{The Fatal Shore.}
Vintage Books, 1988; republished by Harvill Press, 1996.}

\bibitem{huygens}{Huygens,~C., \textit{Trait\'e de la lumi\`ere}
(Leiden, 1690). In \textit{Oeuvres compl\`etes} \textit{XIX}, 1737.}

\bibitem{jususs}{Jurdjevic,~V.,~and~H.~J.~Sussmann,
\textit{Control systems on Lie groups.} 
J.~Diff.~Equations \textbf{12}, 1972, pp.~313--329.}

\bibitem{jurdjevic}{Jurdjevic, V. \textit{Geometric Control Theory.}
Cambridge U. Press, New York, 1997.}

\bibitem{kazarinoff}{Kazarinoff,~N.~D., 
\textit{Geometric Inequalities.} 
\underline{New Mathematical Library} No. 4,
The Mathematical Association of America, Washington, DC, 1961.}

\bibitem{kelley}{Kelley,~H.~J., R.~E.~Kopp and H.~G.~Moyer,
\textit{Singular extremals.} In \textit{Topics in Optimization},
G. Leitman ed., Academic Press, New York, 1967.}

\bibitem{kline}{Kline, M., \textit{Mathematics, the Loss of Certainty.}
Oxford University Press, New York, 1980.}

\bibitem{knobloch}{Knobloch,~H.~W., \textit{High Order Necessary 
Conditions in Optimal Control.} Springer-Verlag, Berlin, 1975.}

\bibitem{krenerchow}{Krener,~A.~J., \textit{A generalization of Chow's 
theorem and the bang-bang theorem to nonlinear control systems.}
S.I.A.M. J. Control \textbf{12}, 1974, pp.~43--52.}

\bibitem{krener}{Krener,~A.~J., \textit{The higher order maximum principle 
and its application to singular extremals.} 
S.I.A.M. J. Control Opt. \textbf{15}, 1977, pp.~256--293.}

\bibitem{kupka} {Kupka, I., \textit{The ubiquity of Fuller's phenomenon.} In
\textit{Nonlinear Controllability and Optimal Control,}  
H.~J.~Sussmann Ed.,  Marcel Dekker, New York 1990, pp.~313--350.}

\bibitem{lanczos}{Lanczos,~C., \textit{The Variational Principles of
Mechanics.} Univ. of Toronto Press, 1949.}

\bibitem{landau}{Landau,~Y.~D., \textit{Adaptive control: the 
model reference approach}. Marcel Dekker, New York, 1979.} 

\bibitem{lavre}{Lavrentiev, M., \textit{Sur quelques probl\`emes du calcul
des variations.}  Ann. Mat. Pura Appl. \textbf{4}, 1926, pp.~7--28.}

\bibitem{leemarkus}{Lee,~E.~B., and L.~Markus, \textit{Foundations of
Optimal Control Theory.} Wiley, New York, 1967.}

\bibitem{leibniz-a}{Leibniz,~G.~W., \textit{Unicum opticae, catoptricae et
dioptricae principium.} Acta Eruditorum \textbf{1}, 1683, pp.~186-190.}

\bibitem{leibniz-b}{Leibniz,~G.~W., \textit{Nova methodus pro maximis et
minimis, itemque 
tangentibus, qua nec fractas nec irrationales quantitates moratur, et
singulari pro illis calculus genus.} Acta Eruditorum \textbf{3},
1684, pp.~467-473. Engl. transl. in Struik, \cite{struik}, pp.~272-280.} 

\bibitem{sussliu}{Liu,~W.~S., and H.~J.~Sussmann, 
\textit{Shortest paths for sub-riemannian metrics on rank-2
distributions.}  Memoirs of the American Math. Society \textbf{564},
Vol. 118, Nov. 1995.}

\bibitem{lobry}{Lobry,~C., \textit{Controlabilit\'e des syst\`emes
non-lin\'eaires.}  S.I.A.M.  J. Control \textbf{8}, 1970, pp.~573--605.}

\bibitem{lochak}{Lochak,~G., \textit{La G\'eom\'etrisation de la 
Physique.} Flammarion, Paris, 1994.}


\bibitem{Mdis}
{Malisoff, M., ``Viscosity Solutions of the Bellman Equation for
Optimal Control Problems with Exit Times.'' Ph.D. Thesis, Rutgers
University, New Brunswick, NJ, January 2000.}

\bibitem{M99b}
{Malisoff, M., ``Viscosity solutions of the Bellman equation for
exit time optimal control problems with vanishing Lagrangians.''
{\em SIAM J. Control \mbox{Optim.}}, to appear.}

\bibitem{MALI}
{Malisoff, M., ``Viscosity solutions of the Bellman equation for
exit time optimal control problems with non-Lipschitz dynamics.''
{\em ESAIM: Control, Optimisation and Calculus of Variations},
{\bf 6} (2001),   pp.~415-441.}

\bibitem{M01}
{Malisoff, M., ``Bounded-from-below solutions of the
Hamilton-Jacobi equation for optimal control problems with exit
times: Vanishing \mbox{Lagrangians}, eikonal equations, and
shape-from-shading.'' \mbox{submitted}.}

\bibitem{marchal} {Marchal,~C., \textit{Chattering arcs and chattering 
controls.} J. Optim Theory Appl. \textbf{11}, 1973, pp.~441--468.}  

\bibitem{markov} {Markov,~A.~A.,  \textit{Some examples of the solution of a 
special kind of problem in greatest and least quantities,}
(in Russian). Soobshch. Karkovsk. Mat. Obshch. \textbf{1}, 1887,
pp.~250--276.}

\bibitem{mcshane}{McShane,~E.~J., \textit{On multipliers for Lagrange
problems.} Amer. J. Math. \textbf{LXI}, 1939, pp.~809--819.}

\bibitem{Mayr}{O. Mayr, \textit{The Origins of Feedback Control.} 
MIT Press, 1970.}

\bibitem{bressra4}{Motta,~M. and F.~Rampazzo, \textit{Dynamic programming for
nonlinear systems driven by ordinary and impulsive controls.} S.I.A.M.
J. Control Optim. \textbf{34}, 1996, pp.~199--225.}

\bibitem{bressra5}{Motta,~M. and F.~Rampazzo, \textit{Nonlinear systems with
unbounded controls and state  constraints: a problem of proper extension.}
NoDEA -- Nonlinear Differential Equations and Applications \textbf{3,2},
1996, pp.~191--216. [GNU: CHECK THIS REFERENCE, IF NEEDED] }

\bibitem{pars}{Pars,~L.~A., \textit{An Introduction to the Calculus of
Variations.} Heinemann, London, 1962.}

\bibitem{pears}{Pears,~I., \textit{An Instance of the Fingerpost.}
Vintage, London, 1998.}

\bibitem{perec}{Perec,~G., \textit{La Disparition.} \'Editions 
Deno\"el, Paris, 1969.}

\bibitem{perecad}{Perec,~G., \textit{A Void},  English translation of
\cite{perec}, by G. Adair, Harper-Collins, London, 1994.}

\bibitem{pillai} {Pillai,~H.~K.,  and S.~Shankar, \textit{A behavioral 
approach to control of distributed systems. SIAM Journal on Control and 
Optimization} {\bf 37}, 1999, pp.~388-408.}

\bibitem{poicare}{Poincar\'e,~H., \textit{Theory of Fuchsian groups.} 
Acta Math. \textbf{I}, 1882, pp.~1--62.}

\bibitem{Book}{Polderman,~J.~W., and J.~C.~Willems,
\textit{Introduction to Mathematical Systems Theory: A Behavioral Approach.}
Springer-Verlag, 1998.}

\bibitem{pontrybook}{Pontryagin,~L.~S., V.~G.~Boltyanskii, R.~V.~Gamkrelidze,
and E.~F.~Mischenko, \textit{The Mathematical Theory of Optimal Processes.}
Wiley, New York, 1962.}

\bibitem{pucci}{Pucci, A., Sistemi di equazioni differenziali
con secondo membro discontinuo rispetto all'incognita.''
{\it Rend. Ist. Mat. Univ. Trieste} {\bf 3}, 1971, pp.~75-80.}

\bibitem{bressra6}{Rampazzo,~F. and C.~Sartori, \textit{The minimum time
function with unbounded controls.} to appear in J. of Math. Est. Systems 
Control. [GNU: UPDATE THIS REFERENCE, IF NEEDED]}

\bibitem{reedsshepp} {Reeds,~J.~A., and L.~A.~Shepp, \textit{Optimal paths 
for a car that goes both forwards and backwards.} Pacific J. Math.
\textbf{145}, 1990, pp.~367--393.}

\bibitem{saccheri-log} {Saccheri,~G., \textit{Logica demonstrativa.}
Turin, Italy, 1697.}

\bibitem{saccheri-euc} {Saccheri,~G., \textit{Euclides ab omni naevo
vindicatus: sive conatus geometricus quo stabiliuntur prima ipsa universae
Geometriae Principia.} Milan, Italy, 1733.}

\bibitem{sastry} {Sastry,~S.~S., \textit{Adaptive control: stability,
convergence and robustness}. Prentice Hall, Englewood Cliffs NJ, 
1989.}

\bibitem{seidenberg-a}{Seidenberg,~A., \textit{Peg and chord in ancient 
Greek geometry.} Scripta Mathematica \textbf{XXIV}, 1959, 
pp.~107--122.}

\bibitem{seidenberg-b}{Seidenberg,~A., \textit{The ritual origin of
geometry.} Arch. Hist. Exact Sciences \textbf{1}, 1962, 
pp.~488--527.}

\bibitem{seidenberg-c}{Seidenberg,~A., \textit{The origin of
mathematics.} Arch. Hist. Exact Sciences \textbf{18}, 1977, 
pp.~301--342.}

\bibitem{stillwell}{Stillwell,~J., \textit{Mathematics and its History.}
Springer-Verlag, New York, 1989.}

\bibitem{stillwell-hyp}{Stillwell,~J., \textit{Sources of Hyperbolic Geometry.}
\underline{History of Mathematics}, Vol. 10, Amer. Math. Society, 
Providence, RI, 1996.}

\bibitem{struik}{Struik,~D.~J.,~Ed., \textit{A Source Book in Mathematics, 
1200-1800.} Harvard U. Press, Cambridge, Mass. 1969.}

\bibitem{subbotin}
{Subbotin, A.~I., {\em Generalized solutions of first-order 
PDEs: the dynamical optimization perspective.} Birkh\"auser,
Boston, 1995.}

\bibitem{subbotina}{Subbotina,~N.~N., \textit{The maximum principle and the
superdifferential of the value function.}  Problems Control
Inform. Theory/Problemy Upravlen. Teor. Inform. \textbf{18}, 
1989, no. 3, pp.~151--160.}

\bibitem{sussju}{Sussmann,~H.~J., and V.~Jurdjevic, 
\textit{Controllability of nonlinear systems.}
J. Diff. Equations \textbf{12}, 1972, pp.~95--116.}

\bibitem{sussreg1}{Sussmann,~H.~J., \textit{Trajectory regularity and real
analyticity: some recent results.} In \textit{Proc. 25th I.E.E.E. 
Conference on Decision and Control, Athens, Greece, 1986,} 
I.E.E.E. publications, New York, 1986, pp.~3306--3312.}

\bibitem{sussreg2}{Sussmann, H. J., \textit{A weak regularity theorem for
real analytic optimal control problems.}
Re\_vis\_ta Ma\-te\-m\'a\-ti\-ca Ibe\-ro\-ame\-ri\-cana \textbf{2}, No.~3,  
1986, pp.~307--317.}

\bibitem{suss1}{Sussmann,~H.~J., \textit{A strong version of the
{\L}ojasiewicz maximum principle.} In \textit{Optimal Control of Differential 
Equations,} N.~H.~Pavel Ed., Marcel Dekker, New York, 1994,
pp.~293--309.}

\bibitem{suss2}{Sussmann,~H.~J., \textit{A strong version of the Maximum
Principle under weak hypotheses.} In \textit{Proc. 33rd I.E.E.E, Conference on
Decision and Control, Orlando, FL, 1994}, 
I.E.E.E. publications, New York, 1994, pp.~1950-1956.}

\bibitem{susstang} {Sussmann,~H.~J., and G.~Q.~Tang, \textit{Shortest paths
for the Reeds-Shepp car: a worked out example of the use of geometric
techniques in nonlinear optimal control.} Rutgers Center for Systems
and Control (Sycon) Report 91-10, September 1991.  To appear in S.I.A.M.
J. Control.}

\bibitem{sussthree} {Sussmann,~H.~J., \textit{Shortest 3-dimensional paths 
with a prescribed curvature bound.}  In \textit{Proc. 34th
I.E.E.E. Conference on Decision and Control, New Orleans, La, Dec. 1995}, 
I.E.E.E. publications, New York, 1995, pp.~3306--3312.}
  
\bibitem{susscofree}{Sussmann,~H.~J., \textit{An introduction to the
coordinate-free maximum principle.} In \textit{Geometry of Feedback and 
Optimal Control,} B.~Jakubczyk and W.~Respondek Eds., Marcel Dekker, 
New York, 1997, pp.~463--557.}

\bibitem{suss3}{Sussmann,~H.~J.,  
\textit{A strong maximum principle for systems of differential inclusions.}
In \textit{Proc. 35th I.E.E.E. Conference on Decision and Control,
Kobe, Japan, Dec. 1996,} I.E.E.E. publications, 1996,  pp.~1809--1814.}

\bibitem{Su-Japan}
{Sussmann,~H.~J., ``From the Brachystochrone \mbox{problem} to the maximum
principle.'' 
In \textit{Proc. 35th I.E.E.E. Conference on Decision and Control,
Kobe, Japan, Dec. 1996,} I.E.E.E. publications, 1996, pp.~1588-1594.}

\bibitem{suss4}{Sussmann,~H.~J.,  \textit{Some recent results on 
the maximum principle of optimal control theory.}
In \textit{Systems and Control in the Twenty-First Century,} 
C. I. Byrnes, B. N. Datta, D. S. Gilliam and C. F. Martin Eds.,
Birkh\"auser, Boston, 1997, pp.~351--372.}

\bibitem{sussjan}{Sussmann,~H.~J., and J.~C.~Willems,
\textit{300 years of optimal control: from the brachystochrone to the maximum 
principle.}  I.E.E.E. Control Systems Magazine, June 
1997.[GNU: PAGE NUMBERS]} 

\bibitem{sussangular} {Sussmann, H. J. \textit{The Markov-Dubins problem with
angular acceleration control.} In \textit{Proc.  
36th  I.E.E.E. Conference on Decision and Control (CDC),
San Diego, CA, Dec. 1997.} I.E.E.E. Publications, New York, 1997, pp.~2639-2643.}

\bibitem{suss5}{Sussmann,~H.~J., \textit{Multidifferential calculus: chain
rule, open mapping and transversal intersection theorems.} In \textit{Optimal
Control: Theory, Algorithms, and Applications,} W. W.  Hager and
P. M. Pardalos, Editors, Kluwer Academic Publishers, 1998, pp.~436--487.}

\bibitem{suss6}{Sussmann,~H.~J., \textit{Geometry and optimal control.}
In \textit{Mathematical Control Theory}, J. Baillieul and
J. C. Willems, Eds., Springer-Verlag, New York, 1998, pp.~140--198.}

\bibitem{suss7}{Sussmann,~H.~J., \textit{Transversality conditions and a
strong maximum principle for systems of differential inclusions.} 
In Proc. 37th I.E.E.E. Conference on Decision and Control, Tampa,
FL, Dec. 1998. I.E.E.E. publications, 1998, pp.~1--6.}

\bibitem{suss8} {Sussmann, H. J., \textit{Generalized differentials, flows,
variations, and the maximum principle.}
To appear in the book of proceedings of the 1999 AMS Short Course
on Nonlinear Control, held in San Antonio, TX, in January 1999, during the 
joint winter meeting of the AMS and the MAA.  The book will be published 
in the Symposia on Pure and Applied Mathematics series.}

\bibitem{susshybrid}{Sussmann, H.J., ``A nonsmooth hybrid maximum 
principle.'' In 
{\it \mbox{Stability} and \mbox{Stabilization of Nonlinear 
Systems, Ghent, Belgium, 1999}}. D.~Aeyels, 
\mbox{F.~Lamnabhi-Lagarrigue {\&} A.J.~van der Schaft \mbox{Editors}.  
$\,$\unde{Lect. Notes Control}} \unde{and Information Sciences} {\bf 246}, 
Springer-Verlag, 1999, pp.~325--354.}

\bibitem{sussfrance}{Sussmann, H.J., ``R\'esultats r\'ecents sur les
courbes optimales.''  In {\it Quelques aspects de la th\'eorie du
contr\^ole, Journ\'ee annuelle de la \mbox{Soci\'et\'e} Math\'ematique de
France}, Paris, June 2000, pp.~1-50.}

\bibitem{sussnewtheor}{Sussmann, H.J., ``New theories of set-valued
differentials and new \mbox{versions} of the maximum principle of optimal
control theory.''  In {\it \mbox{Nonlinear} Control in the Year 2000}, 
A.~Isidori,
F.~Lamnabhi-Lagarrigue and W.~Respondek Editors, Springer-Verlag, 2000,
pp.~487-526.}

\bibitem{synge} {Synge,~J.~L., \textit{The life and early work of Sir William
Rowan Hamilton.} Scripta Mathematica \textbf{X}, 1944,  pp.~13--25.}

\bibitem{taylor} {Taylor,~H.~G., \textit{The Discovery of Tasmania.}
Cat {\&} Fiddle Press, \mbox{Hobart}, Tasmania, 1973.} 

\bibitem{TSH} {H.L. Trentelman,~H.~L.,  A.~A.~Stoorvogel, and M.~L.~J.~Hautus, 
\textit{Control Theory for Linear Systems.} Springer-Verlag, 2001.}

\bibitem{tul1} {Tulczyjew,~W.~M., \textit{A Symplectic Formulation of
Relativistic Particle Dynamics.} Acta Physica Polonica,
Vol. \textbf{B}8, 1977, p.~39. GNU NEED COMPLETE PAGE REF}

\bibitem{tul2}{Tulczyjew,~W.~M., \textit{Control of Static Mechanical
Systems.} In \textit{\mbox{Proceedings}
of ``Dynamical Systems and Microphysics''.}
CISM, Udine, Italy, 1984, p.~359. GNU NEED COMPLETE PAGE REF}

\bibitem{vander}{van der Waerden,~B.~L., \textit{
Geometry and Algebra in \mbox{Ancient}
\mbox{Civilizations.}} Springer-Verlag, 
New York, 1983.}
 
\bibitem{vitale}{Vitale,~G., \textit{Euclide restituto over gli antichi
elementi geometrici \mbox{ristaurati}, e facilitati. Libri XV.}
Rome, Italy, 1680.}

\bibitem{wallis}{Wallis,~J., \textit{De Postulato Quinto; et Definitione
Quinta; Lib.~6 Euclidis; discertatio geometrica.} Opera Math.~t.~II, Oxford, 
1693, pp.~669--678.}

\bibitem{yourgrau}{Yourgrau,~W., and S.~Mandelstam,
\textit{Variational Principles in 
\mbox{Dynamics} and Quantum Theory.}
W.~B.~Saunders \&{} Co., Philadelphia, 1968.}

\bibitem{weier}{Weierstrass, K., 
\textit{Vorlesungen \"uber Variationsrechnung; Mathematische Werke, VII,} 
(in German). Leipzig, Akadem. Verlagsges, 1927. Reprinted by 
Johnson Reprint  Corp., New York, 1967.}
\bibitem{paradigms} J.C. Willems, Paradigms and puzzles in the theory
of dynamical systems, {\em IEEE Transactions on Automatic Control,}
volume 36, pages 259-294, 1991.

\bibitem{control}{Willems,~J.~C., \textit{On interconnections, control, 
and feedback.} IEEE Transactions on Automatic Control {\bf 42}, 1997,
pp.~326-339.}

\bibitem{zeli} {Zelikin,~M.~I., and V.~F.~Borisov, 
\textit{Theory of Chattering Control, with Applications to 
Astronautics, Robotics, Economics and Engineering.} 
Birkh\"auser, Boston, 1994.}

\end{thebibliography}

\newpage


\ssection{The optimal control and separation problems}\label{optisec}

\pparagraph{The data} \ The basic data for our optimal control problem 
will be 
\begin{itemize}
\item[(D.1)] a {\bf\em control system}
\begin{equation}\label{flasystem}
\Sigma:\quad\dot x=f(x,u,t)\,,\,\,\,\,x\in\Omega\,,\,\,\,\,
t\in I\,,\,\,\,\,u\in U\,,\,\,\,\,u(\cdot)\in\mathcal{U}\,,
\end{equation}
(which will be also referred to, somewhat more concisely, as 
``the control \mbox{system} \mbox{$\Sigma=(\Omega,U,I,f,\mathcal{U})\,$};''
\ the letters $x$, $u$, $t$ are, \mbox{respectively}, 
the ``state variable,'' the ``control variable,'' 
and the ``time \mbox{variable}'' of the system);
\item[(D.2)] a {\bf\em Lagrangian} $L$.
\item[(D.3)] an {\bf\em endpoint constraint set} $\mathcal{S}$;
\item[(D.4)] a {\bf\em reference trajectory-control pair}
$\gamma_*=(\xi_*,\eta_*)$;
\end{itemize}

\pparagraph{The assumptions; controls, trajectories, 
\mbox{endpoint} conditions} \ We will assume that 
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{chapproofmax}.A1.1)] {\em $n$ is a positive integer,
and $\Omega$ is an open subset 
of $\rr^n$, known as the ``state space'' of the system;} 
\item[(\ref{chapproofmax}.A1.2)] 
{\em  $U$ is a set, known as the ``control set'' 
(or ``control space'') of the \mbox{system};}
\end{itemize}\end{itemize}
Under these conditions, we can define the concept of a
``$U$-control.''
\begin{definition}\label{ucontrol}
{\rm If $U$ is a set, a {\bf\em $U$-control} is a map 
$\eta:[a_\eta,b_\eta]\mapsto U$, which is defined
on a compact subinterval $\dom(\eta)=[a_\eta,b_\eta]$
of $\rr$}.\proofend
\end{definition}
We will also assume that  
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{chapproofmax}.A1.3)] {\em $I$ is a subinterval 
of the real line, known as the ``time interval'' of the system;}
\item[(\ref{chapproofmax}.A1.4)] {\em the  ``controlled dynamical law'' $f$
is a map}
$$
\Omega\times U\times I\ni(x,u,t)\mapsto f(x,u,t)\in\rr^n\,;
$$
\item[(\ref{chapproofmax}.A1.5)] {\em the  ``class of admissible controls'' 
$\mathcal{U}$ is a set of $U$-controls whose domains are subintervals of $I$;}
\item[(\ref{chapproofmax}.A1.6)] {\em $\mathcal{S}$ is a subset of} 
$\Omega\times\Omega$;
\end{itemize}\end{itemize}
We can then define the concepts of ``trajectory,''
``full trajectory,'' ``trajectory-control pair,'' 
and ``boundary value.''
\begin{remark} 
{\rm In Definitions \ref{remonfull} and \ref{defoaugmen} below we use
the relatively more \mbox{advanced} mathematical ideas of ``local
absolute \mbox{continuity}'' and ``equality \mbox{almost} everywhere.''
Those readers who wish to see a proof of the maximum \mbox{principle} for
classical systems (with piecewise continuous controls) that avoids
these notions can refer instead to Definition \ref{shoddy-b}, where
trajectories for classical systems are defined in a way which is
equivalent to that of Definition \ref{remonfull} but does not involve
the more advanced concepts.}\proofend
\end{remark}

\begin{definition}\label{remonfull}
{\rm Given a control system $\Sigma$ as in (\ref{flasystem}), 
a {\bf\em $\Sigma$-trajectory} for a control 
\mbox{$\eta:[a_\eta,b_\eta]\mapsto U$} is a locally
\mbox{absolutely} continuous curve $\xi:I_\xi\mapsto\Omega$, defined on some 
nonempty subinterval $I_\xi$ of $[a_\eta,b_\eta]$  such that 
\begin{equation}\label{equatraj}
\dot\xi(t)=f(\xi(t),\eta(t),t)\quad{\rm for\,\,almost\,\, all}\quad
t\in I_\xi\,.
\end{equation}

A {\bf\em full $\Sigma$-trajectory} for $\eta$ is a $\Sigma$-trajectory
$\xi$ whose domain $I_\xi$ is the whole interval $[a_\eta,b_\eta]$.

We use $TCP(\Sigma;\eta)$ to denote the set of all full 
$\Sigma$-trajectories for $\eta$.

A {\bf\em trajectory-control pair} for $\Sigma$ is a pair
$\gamma=(\xi,\eta)$ such that $\eta\in\mathcal{U}$ and
$\xi$ belongs to $TCP(\Sigma;\eta)$.

We use $TCP(\Sigma)$ to denote the set of all trajectory-control pairs
of $\Sigma$.} \proofend
\end{definition}

\begin{definition}\label{bvalue}
{\rm If $\xi:[a,b]\mapsto\Omega$ the {\bf\em endpoint value}
(or  {\bf\em endpoint condition}, or  {\bf\em boundary value},
or  {\bf\em boundary  condition}) {\bf\em of} $\xi$ is the pair
$$
\partial\xi\equaldef(\xi(a),\xi(b))\,.
$$
We use $\evset(\Sigma)$ to denote the set
$\{\partial\xi:(\xi,\eta)\in TCP(\Sigma)\}$  of all endpoint values
of all full trajectories $\xi$ corresponding to controls
$\eta\in\mathcal{U}$.} \proofend
\end{definition}

We will then make three additional assumptions:
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{chapproofmax}.A1.7)] $\gamma_*=(\xi_*,\eta_*)\in TCP(\Sigma)$;
\item[(\ref{chapproofmax}.A1.8)] {\em $\partial\xi_*\in\mathcal{S}$ 
(cf. Definition \ref{bvalue} below);}
$U$-control $\eta$
\item[(\ref{chapproofmax}.A1.9)] {\em the ``Lagrangian'' $L$ is a function}
$$
\Omega\times U\times I\ni(x,u,t)\mapsto L(x,u,t)\in\rr\,.
$$
\end{itemize}\end{itemize}

 
\pparagraph{Minimizers}
Suppose we are given a system $\Sigma$ as in (\ref{flasystem}),
and a Lagrangian $L$ as in (\ref{chapproofmax}.A1.9).

\begin{definition}\label{defoaugmen}
{\rm An {\bf\em $L$-augmented $\Sigma$-trajectory} for a $U$-control $\eta$
is an ordered pair $\Xi=(\xi^0,\xi)$ such that $\xi$ is a 
$\Sigma$-trajectory for $\eta$, 
$\xi^0:I_\xi\mapsto\rr$ is a locally \mbox{absolutely} continuous
function, and 
\begin{equation}\label{equacost}
\dot\xi^0(t)=L(\xi(t),\eta(t),t)\qquad{\rm for\,\,\,almost\,\,\,all}
\qquad t\in I_\xi\,.
\end{equation}

A {\bf\em full $L$-augmented $\Sigma$-trajectory} for $\eta$
is an augmented trajectory $\Xi=(\xi^0,\xi)$ such that 
$I_\xi=[a_\eta,b_\eta]$. We will use the expression $TCP(\Sigma,L;\eta)$
to denote the set of all full $L$-augmented
$\Sigma$-trajectories for $\eta$. 

An {\bf\em $L$-augmented trajectory-control pair 
for $\Sigma$} is a pair $\Gamma=(\Xi,\eta)$
such that $\Xi\in TCP(\Sigma,L;\eta)$. We  write 
$TCP(\Sigma,L)$ to denote the set of all \mbox{trajectory-control} pairs
for $\Sigma$.}\proofend
\end{definition}

\begin{definition}\label{defminim}
{\rm The {\bf\em cost functional} for the system
$\Sigma$ and the \mbox{Lagrangian} $L$ is the function 
${\cal J}_{\Sigma,L}:TCP(\Sigma,L)\mapsto\rr$ that assigns 
to each $L$-augmented 
trajectory-control pair $\Gamma=(\Xi,\eta)=(\xi^0,\xi,\eta)$ for $\Sigma$ 
the number ${\cal J}_{\Sigma,L}(\Gamma)$ given by}
\begin{eqnarray*}
{\cal J}_{\Sigma,L}(\Gamma)&=&{\cal J}_{\Sigma,L}(\Xi,\eta)\\
&\equaldef&{\cal J}_{\Sigma,L}(\xi,\eta)\\
&=&\xi^0(b_\eta)-\xi^0(a_\eta)\\
&=&\int_{a_\eta}^{b_\eta}L(\xi(t),\eta(t),t)\,dt\,.
\end{eqnarray*}

{\rm A {\bf\em minimizer} for the optimal control problem defined
by $\Sigma$, $L$, and $\mathcal{S}$, is an \mbox{$L$-augmented}
trajectory-control pair $\Gamma_*=(\xi_*^0,\xi_*,\eta_*)$ of $\Sigma$ 
such that $\partial\xi_*\in\mathcal{S}$ and 
\begin{itemize}\item[]\begin{itemize}
\item[(MIN)$\,\,$] {\sl ${\cal J}_{\Sigma,L}
(\xi_*^0,\xi_*,\eta_*)\le{\cal J}_{\Sigma,L}(\xi^0,\xi,\eta)$
for all $(\xi^0,\xi,\eta)\in TCP(\Sigma,L)$
such that $\partial\xi\in{\cal S}\,$,}
\end{itemize}\end{itemize}
i.e., $\Gamma_*$ minimizes the cost functional ${\cal J}_{\Sigma,L}$ 
among all $L$-augmented trajectory-control pairs $(\xi^0,\xi,\eta)$ of 
$\Sigma$ such that $\partial\xi\in{\cal S}$.}\proofend
\end{definition}

\begin{remark}\label{remiminim}
{\rm In Definition \ref{defminim} we introduced the notation 
${\cal J}_{\Sigma,L}(\xi,\eta)$ as an alternative for 
${\cal J}_{\Sigma,L}(\Xi,\eta)$, if $\Xi=(\xi^0,\xi)$.
This is justified because, if the pair $(\xi,\eta)$ is fixed,
then any two functions $\xi_1^0$, $\xi_2^0$, such that
$(\xi_1^0,\xi,\eta)$ and $(\xi_1^0,\xi,\eta)$ belong to
$TCP(\Sigma,L)$ must differ by a constant, so 
$\xi_1^0(b_\eta)-\xi_1^0=\xi_2^0(b_\eta)-\xi_2^0(a_\eta)$, 
and then ${\cal J}_{\Sigma,L}(\xi_1^0,\xi,\eta)=
{\cal J}_{\Sigma,L}(\xi_1^0,\xi,\eta)$.

If we think of ${\cal J}_{\Sigma,L}$ as a functional
$(\xi,\eta)\mapsto{\cal J}_{\Sigma,L}(\xi,\eta)$, then the domain
of ${\cal J}_{\Sigma,L}$ should be the set
$TCP_{L}(\Sigma)$ of all ``$L$-admissible'' trajectory-control
pairs of $\Sigma$, where a trajectory-control pair 
$(\xi,\eta)\in TCP(\Sigma)$ is said to be {\bf\em $L$-admissible}
if the function $[a_\eta,b_\eta]\ni t\mapsto L(\xi(t),\eta(t))$ 
is Lebesgue integrable.}\proofend
\end{remark}


\pparagraph{The optimal control problem} We will try to solve the following
\medbreak\begin{center}
\fbox{\begin{minipage}{12cm}
\begin{center}
{\large\bf OPTIMAL CONTROL PROBLEM (OCP)}
\end{center}

{\bf Given} data $n$, $\Sigma=(\Omega,U,I,f,\mathcal{U})$,  $L$,  
${\cal S}$, $\gamma_*=(\xi_*,\eta_*)$, such that \mbox{conditions}
(\ref{chapproofmax}.A1.1-2-3-4-5-6-7-8-9) hold;

{\bf find}: a necessary condition for the pair $\gamma_*=(\xi_*,\eta_*)$ 
to be a minimizer.
\end{minipage}}\end{center}

\pparagraph{The separation problem}
The data for the separation problem are the same as those for the
optimal control problem, except that there is no Lagrangian, and
instead of looking for minimizers we look for trajectories
that satisfy a separation condition. 

\begin{definition}\label{reacha}
{\rm The {\bf\em endpoint value set} (or 
{\bf\em endpoint condition set}, or
{\bf\em boundary value set}, or {\bf\em boundary condition set})
for a control system $\Sigma$ is the set  $\evset(\Sigma)$
of all \mbox{endpoint} values $\partial\xi$, for all pairs
$(\xi,\eta)\in TCP(\Sigma)$.}\proofend
\end{definition}

\begin{definition}\label{locsep}
{\rm If $A$, $B$ are subsets of a metric space $M$ (cf. \S \ref{secmetric}), 
we say  that $A$ is {\bf\em separated} from $B$ at a 
point $m\in M$ if $A\cap B=\{m\}$
We say  that $A$ is {\bf\em locally separated} from $B$ at a 
point $m\in M$ if $A\cap B\cap\mathcal{N}=\{m\}$
for some neighborhood $\mathcal{N}$ of $m$ in $M$.}\proofend
\end{definition}


\label{pagesepa}
\medbreak\begin{center}
\fbox{\begin{minipage}{12cm}
\begin{center}
{\large\bf SEPARATION PROBLEM (SP)}
\end{center}

{\bf Given}: data $n$, $\Sigma=(\Omega,U,I,f,\mathcal{U})$,  ${\cal S}$, 
$\gamma_*=(\xi_*,\eta_*)$,  such that conditions
(\ref{chapproofmax}.A1.1-2-3-4-5-6-7-8) hold;

{\bf find}: a necessary condition for the endpoint value set
$\evset(\Sigma)$ to 
be locally separated from ${\cal S}$ at $\partial\xi_*$.
\end{minipage}}\end{center}

\pparagraph{Reduction of the optimal control problem to the separation
problem} Having solved the SP, we show how to derive as a corollary a
solution of the OCP. This will be an easy matter, for reasons that we
now briefly outline. (The detailed proof, with a precise specification
of all relevant technical conditions, will be given later.)

Suppose that the $L$-augmented pair $\Gamma_*=(\Xi_*,\eta_*)
=(\xi_{*}^0,\xi_*,\eta_*)$ is a  \mbox{minimizer} for the 
OCP with \mbox{data} \ $n$, $\Sigma=(\Omega,U,I,f,\mathcal{U})$,  $L$, and 
${\cal S}$. We let $\hat\Sigma$ be the ``\mbox{augmented} control 
system''  $(\hat\Omega,U,I,\hat f,\mathcal{U})$,  where 
$\hat\Omega=\rr\times\Omega$ and
\begin{equation}\label{eqforaugsys}
\hat f(x^0,x,u,t)=(L(x,u,t),f(x,u,t))\quad{\rm if}\quad
(x^0,x)\in\hat\Omega\,,\,\,u\in U\,,\,\,t\in I\,.
\end{equation}
That is, $\hat\Sigma$ is the system 
\begin{equation}\label{flasforaugsys}
\hat\Sigma:\qquad\dot x^0=L(x,u,t)\,,\,\,
\dot x=f(x,u,t)\,,\,\,(x^0,x)\in\rr\times\Omega\,,\,\,u\in U\,,
\,\,u(\cdot)\in\mathcal{U}\,,
\end{equation}
obtained from $\Sigma$ by adding the ``running cost'' $x^0$ as a new state 
variable. 

If we write  \ $a_*=a_{\eta_*}$,  \ $b_*=b_{\eta_*}$, 
\ $x_*=\xi_*(a_{*})$, \ $\tilde x_*=\xi_*(b_{*})$, 
\ $x_*=\xi_*^0(a_{*})$, \mbox{$\tilde x_*=\xi_*^0(b_{*})$}, 
\mbox{$c_*=\tilde x_*^0-x_*^0=\mathcal{J}_{\Sigma,L}(\Gamma_*)$}, and define
$\hat{\cal S}$ to be the set of all ordered pairs
$\Big((x^0,x),(\tilde x^0,\tilde x)\Big)\in\hat\Omega\times\hat\Omega$
such that
\begin{itemize}
\item[(a)] $(x,\tilde x)\in{\cal S}$,
\item[(b)] $\tilde x^0-x^0\le c_*-\|x-x_*\|^2-\|\tilde x-\tilde x_*\|^2$
\item[(c)] $x^0=x_*^0$,
\end{itemize}
then it is clear that $\Xi_*$ is a full $\hat\Sigma$-trajectory for 
$\eta_*$ and $\partial\Xi_*\in\hat{\cal S}$.

Furthermore, if $Q\in\evset(\hat\Sigma)\cap\hat{\mathcal{S}}$,
then there must exist a trajectory-control pair 
\mbox{$\Gamma=(\Xi,\eta)$} of
$\hat\Sigma$ such that $Q=\partial\Xi\in\hat{\mathcal{S}}$.  \ If we write
$a=a_\eta$, $b=b_\eta$, \mbox{$\Xi=(\xi^0,\xi)$}, $x=\xi(a)$, 
$\tilde x=\xi(b)$, $x^0=\xi^0(a)$, $\tilde x^0=\xi^0(b)$, 
$c=\tilde x^0-x^0=\mathcal{J}_{\Sigma,L}(\Gamma)$, then the fact
that $\partial\Xi\in\hat{\mathcal{S}}$ implies that
$\partial\xi\in\mathcal{S}$ and
\begin{equation}\label{jess}
c\le c_*-\|x-x_*\|^2-\|\tilde x-\tilde x_*\|^2\,.
\end{equation}
Since $\Gamma_*$ is a minimizer, and $\partial\xi\in\mathcal{S}$, we have 
\mbox{$\mathcal{J}_{\Sigma,L}(\Gamma_*)\le\mathcal{J}_{\Sigma,L}(\Gamma)$}, 
i.e., $c_*\le c$. Therefore 
$c_*\le c\le c_*-\|x-x_*\|^2-\|\tilde x-\tilde x_*\|^2$,
from which it follows that $x=x_*$, $\tilde x=\tilde x_*$,  and $c=c_*$. 
Since $x^0=x_*^0$, we can conclude that $\tilde x^0=\tilde x_*^0$, so 
that $\partial\Xi=\partial\Xi_*$.   Hence $Q=\partial\Xi_*$. 

We have shown that
$\evset(\hat\Sigma)\cap\hat{\mathcal{S}}=\{\partial\Xi_*\}$.
Therefore $\evset(\hat\Sigma)$ and $\hat{\mathcal{S}}$ are locally
separated at $\partial\Xi_*$.  We can the apply the necessary
condition for separation given by the solution of the SP, and obtain a
necessary condition for minimization for OCP. We will carry out the
details later, after we have described the precise technical hypotheses
for our results. \proofend

\ssection{Proof of a classical maximum principle}\label{secproofclass}

As explained in the previous section, the optimal control problem can be 
reduced to the separation problem. We will therefore focus our efforts 
from now on the SP.

So we assume we are given data $n$, $U$, $f$, $\Sigma$, $\eta_*$,
$\xi_*$, $a$, $b$, ${\cal S}$ as in the Separation Problem box (page
\pageref{pagesepa}), and
begin by outlining the construction of packets of needle variations.
For simplicity, we will present the outline only for the fixed time
interval case, although when we do the technical details rigorously 
we will take care of both cases.


The main purpose of the construction of packets of needle variations
is to compute, for our given reference trajectory-control pair
$(\xi_*,\eta_*)$, a ``first-order approximation'' ${\cal P}$ of the
set ${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$ near the point $\xi_*(b)$. The
set ${\cal P}$ should be a convex cone in $\rr^n$, and the fact that
it is a ``first-order approximation'' of ${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$
near $\xi_*(b)$ requires a precise definition to be given later, but means,
to begin with, that each member of ${\cal P}$ is the tangent 
vector $e$ at $\eps=0$ to a curve $\gamma:[0,\bar\eps]\mapsto
{\cal R}_{[a,b]}^\Sigma(\xi_*(a))$ such that 
$\gamma(0)=\xi_*(b)$. 

If we have another convex cone ${\cal Q}$ which is 
``tangent'' to ${\cal S}$ in a similar way, then it is natural
to guess that a necessary condition for separation of the sets 
${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$ and ${\cal S}$ should be 
the linear separation of the cones ${\cal P}$ and 
${\cal Q}$,  i.e., the existence of a nontrivial linear functional
$\lambda:\rr^n\mapsto\rr$ such that 
$\langle\lambda,v\rangle\ge 0$ for all $v\in{\cal P}$ 
and $\langle\lambda,v\rangle\le 0$ for all $v\in{\cal Q}$.

It turns out, however, that if we use the na\"\i{}ve concept
of ``tangent cone'' hinted at in the previous discussion, 
it is not true that ``if two sets ${\cal R}$ and ${\cal S}$ are separated 
at a point $x_*$, and have tangent cones 
${\cal P}$ and ${\cal Q}$ at $x_*$, then the cones 
${\cal P}$ and ${\cal Q}$ are linearly separated.''



Roughly speaking, a (fixed time interval) ``variation'' of the
reference control $\eta_*$ is a 
``curve'' $[0,\bar\eps]\ni\eps\mapsto\eta^\eps$
in the space $A_{[a,b]}(\Sigma)$ of all control
functions $[a,b]\ni t\mapsto\eta(t)\in U$
defined on $[a,b]$ that starts at 
$\eta_*$. That is, a {\bf\em variation} of
$\eta_*$ is a one-parameter family 
$V=\{(\xi^\eps,\eta^\eps):\eps\in[0,\bar\eps\}$ of 
trajectory-control pairs of $\Sigma$ such that 
$(\xi^0,\eta^0)=(\xi_*,\eta_*)$.


Roughly speaking, a (fixed time interval) ``variation'' of the
reference trajectory-control pair $(\xi_*,\eta_*)$ is a 
``curve'' $[0,\bar\eps]\ni\eps\mapsto(\xi^\eps,\eta^\eps)$
in the space $TCP_{[a,b]}(\Sigma)$ of all trajectory-control
pairs of $\Sigma$ defined on $[a,b]$ that starts at 
$(\xi_*,\eta_*)$. That is, a {\bf\em variation} of
$(\xi_*,\eta_*)$ is a one-parameter family 
$V=\{\eta^\eps:\eps\in[0,\bar\eps\}$ of controls defined on $[a,b]$
such that $\eta^0=\eta_*$.

Each control function $\eta:[a,b]\mapsto U$ gives rise to a 
{\bf\em time-varying vector field} $f_\eta$, defined by letting
$f_\eta(x,t)=f(x,\eta(t),t)$.



We assume that we are given 
 


of the given 
at \mbox{various} times $\tau$ in the domain $[a,b]$ of  $\xi_*$.
The effects of these
variations are then \mbox{propagated} to the terminal time b$b$
by means of the \mbox{differentials} $D\Phi_{b,\tau}(\xi_*(\tau))$
of the reference flow maps $\Phi_{b,\tau}$. This
produces a convex cone (the ``Pontryagin cone'') which in some sense
is a first-order approximation to the attainable set near the
point $\xi_*(b)$. The proof is
then concluded by means of a \mbox{topological} \mbox{argument}, based
on the Brower fixed point theorem, to
pass from the \mbox{separation} of two sets to the separation of their
approximating cones. 

Each needle variation consists of a family 
$V=\{\xi^\eps\}_{\eps\in[0,\bar\eps]}$ of trajectories, which are defined 
on $[a,b]$ if $V$ is an ``equal time variation,'' or on intervals
$[a^\eps,b^\eps]$ if $V$ is a ``variable time variation,''
and are such that $\xi^0=\xi_*$ and $\xi^\eps(a^\eps)=\xi_*(a)$ 
afor all $\eps$. Under suitable technical conditions, the terminal
points $\xi^\eps(b^\eps)$ of the arcs $\xi^\eps$ describe a 
continuous curve $\gamma_{_V}:[0,\bar\eps]\mapsto X$ (where $X$ is the state
space of our control system), which is differentiable 
at $\eps=0$. The ``effect'' of the variation $V$ at the terminal 
point is then the vector 
$e_{_V}=\frac{d\,\,}{d\eps}\Big|_{\eps=0}\gamma_{_V}(\eps)$.
This vector is, by construction, ``tangent'' to the set 
${\cal R}$






\gnu{{\large THE MAP IN PAGE \pageref{tasman1page} COMES FROM THE WEB SITE
\medbreak\begin{center}
 http://pacific.vita.org/pacific/dutch/tasman.htm
\end{center}
\medbreak
IS THERE A COPYRIGHT PROBLEM?
}}

\gnu{{\large THE PICTURE IN PAGE \pageref{blogopage} COMES FROM THE WEB SITE
\medbreak\begin{center}
http://www.norfacad.pvt.k12.va.us/project/bernoull/bern.htm
\end{center}
\medbreak
IS THERE A COPYRIGHT PROBLEM?
}}

\gnu{Re: Page \pageref{dynapage}. Maybe we should put in some stuff
about transversality conditions, after all!}

\gnu{Re: Page \pageref{dynapage}. Say something about dynamic 
programming here.}

\gnu{Re: Footnote \ref{footprecise} in page
\pageref{pageprecise}. Do we really do that in the 
appendix?}


\gnu{Re: Remark \ref{remdubins} in page \pageref{remdubins}:
must put more stuff on the Dubins-Reeds-Shepp problem.}

\gnu{Re: Page \pageref{pageispercheck}.
Check that we do indeed show in \S\ref{isoper-are-standard}
what we say we do.}

\gnu{Re: Page \pageref{soliscircle}
Put explanation of why (II) is equivalent to (I).}

\gnu{Re: Page \pageref{zenodorus}. Put elementary proof of Z1 and Z2}

\gnu{The Euler footnote \ref{euler-foot}  in page 
\pageref{euler-foot} has to be expanded,
especially adding stuff about Euler after 1727, and in particular
on Euler's friendship with Daniel B.,
and how it affected the relation with Johann B. (I am thinking 
of Euler passing on to Johann the manuscript he got from Daniel
on the hydrodinamics law, which was then plagiarized by Johann.
{\bf Maybe this footnote should be changed into a Scholium?}}

\gnu{Re: Page \pageref{needbiolege}.
We need something biographical on Lagrange. It should include
the moving story of his great love for a much younger woman when 
he was old. (I think her name was Ren\'ee).}

\gnu{Re: Page \pageref{needbiolegendre}
We need some biographical stuff on Legendre.}

\gnu{Re: page \pageref{needbio}. Need biographical sketch for Hamilton}

\gnu{Re: page \pageref{needbio}. We should clarify the following issue: 
was Hamilton the first person
to write ``Hamilton's equations''?  I have heard somewhere
(I wish I could remember where) that it was Cauchy.  
Notice that in the text I  talk about the ``classical 
Hamiltonian'' but I avoid saying ``the Hamiltonian that Hamilton wrote''
for this reason. We need some research here.}

\gnu{Re: Page \pageref{needbioweie} Need biographical stuff about 
Weierstrass}

\gnu{Re: Page \pageref{needbicar}. Need biographical stuff about Caratheorody}

\ssection{The optimal control and separation problems}\label{optisec}

\pparagraph{The data} \ The basic data for our optimal control problem 
will be 
\begin{itemize}
\item[(D.1)] a {\bf\em control system}
\begin{equation}\label{flasystem}
\Sigma:\quad\dot x=f(x,u,t)\,,\,\,\,\,x\in\Omega\,,\,\,\,\,
t\in I\,,\,\,\,\,u\in U\,,\,\,\,\,u(\cdot)\in\mathcal{U}\,,
\end{equation}
(which will be also referred to, somewhat more concisely, as 
``the control \mbox{system} \mbox{$\Sigma=(\Omega,U,I,f,\mathcal{U})\,$};''
\ the letters $x$, $u$, $t$ are, \mbox{respectively}, 
the ``state variable,'' the ``control variable,'' 
and the ``time \mbox{variable}'' of the system);
\item[(D.2)] a {\bf\em Lagrangian} $L$.
\item[(D.3)] an {\bf\em endpoint constraint set} $\mathcal{S}$;
\item[(D.4)] a {\bf\em reference trajectory-control pair}
$\gamma_*=(\xi_*,\eta_*)$;
\end{itemize}

\pparagraph{The assumptions; controls, trajectories, 
\mbox{endpoint} conditions} \ We will assume that 
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{chapproofmax}.A1.1)] {\em $n$ is a positive integer,
and $\Omega$ is an open subset 
of $\rr^n$, known as the ``state space'' of the system;} 
\item[(\ref{chapproofmax}.A1.2)] 
{\em  $U$ is a set, known as the ``control set'' 
(or ``control space'') of the \mbox{system};}
\end{itemize}\end{itemize}
Under these conditions, we can define the concept of a
``$U$-control.''
\begin{definition}\label{ucontrol}
{\rm If $U$ is a set, a {\bf\em $U$-control} is a map 
$\eta:[a_\eta,b_\eta]\mapsto U$, which is defined
on a compact subinterval $\dom(\eta)=[a_\eta,b_\eta]$
of $\rr$}.\proofend
\end{definition}
We will also assume that  
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{chapproofmax}.A1.3)] {\em $I$ is a subinterval 
of the real line, known as the ``time interval'' of the system;}
\item[(\ref{chapproofmax}.A1.4)] {\em the  ``controlled dynamical law'' $f$
is a map}
$$
\Omega\times U\times I\ni(x,u,t)\mapsto f(x,u,t)\in\rr^n\,;
$$
\item[(\ref{chapproofmax}.A1.5)] {\em the  ``class of admissible controls'' 
$\mathcal{U}$ is a set of $U$-controls whose domains are subintervals of $I$;}
\item[(\ref{chapproofmax}.A1.6)] {\em $\mathcal{S}$ is a subset of} 
$\Omega\times\Omega$;
\end{itemize}\end{itemize}
We can then define the concepts of ``trajectory,''
``full trajectory,'' ``trajectory-control pair,'' 
and ``boundary value.''
\begin{remark} 
{\rm In Definitions \ref{remonfull} and \ref{defoaugmen} below we use
the relatively more \mbox{advanced} mathematical ideas of ``local
absolute \mbox{continuity}'' and ``equality \mbox{almost} everywhere.''
Those readers who wish to see a proof of the maximum \mbox{principle} for
classical systems (with piecewise continuous controls) that avoids
these notions can refer instead to Definition \ref{shoddy-b}, where
trajectories for classical systems are defined in a way which is
equivalent to that of Definition \ref{remonfull} but does not involve
the more advanced concepts.}\proofend
\end{remark}

\begin{definition}\label{remonfull}
{\rm Given a control system $\Sigma$ as in (\ref{flasystem}), 
a {\bf\em $\Sigma$-trajectory} for a control 
\mbox{$\eta:[a_\eta,b_\eta]\mapsto U$} is a locally
\mbox{absolutely} continuous curve $\xi:I_\xi\mapsto\Omega$, defined on some 
nonempty subinterval $I_\xi$ of $[a_\eta,b_\eta]$  such that 
\begin{equation}\label{equatraj}
\dot\xi(t)=f(\xi(t),\eta(t),t)\quad{\rm for\,\,almost\,\, all}\quad
t\in I_\xi\,.
\end{equation}

A {\bf\em full $\Sigma$-trajectory} for $\eta$ is a $\Sigma$-trajectory
$\xi$ whose domain $I_\xi$ is the whole interval $[a_\eta,b_\eta]$.

We use $TCP(\Sigma;\eta)$ to denote the set of all full 
$\Sigma$-trajectories for $\eta$.

A {\bf\em trajectory-control pair} for $\Sigma$ is a pair
$\gamma=(\xi,\eta)$ such that $\eta\in\mathcal{U}$ and
$\xi$ belongs to $TCP(\Sigma;\eta)$.

We use $TCP(\Sigma)$ to denote the set of all trajectory-control pairs
of $\Sigma$.} \proofend
\end{definition}

\begin{definition}\label{bvalue}
{\rm If $\xi:[a,b]\mapsto\Omega$ the {\bf\em endpoint value}
(or  {\bf\em endpoint condition}, or  {\bf\em boundary value},
or  {\bf\em boundary  condition}) {\bf\em of} $\xi$ is the pair
$$
\partial\xi\equaldef(\xi(a),\xi(b))\,.
$$
We use $\evset(\Sigma)$ to denote the set
$\{\partial\xi:(\xi,\eta)\in TCP(\Sigma)\}$  of all endpoint values
of all full trajectories $\xi$ corresponding to controls
$\eta\in\mathcal{U}$.} \proofend
\end{definition}

We will then make three additional assumptions:
\begin{itemize}\item[]\begin{itemize}
\item[(\ref{chapproofmax}.A1.7)] $\gamma_*=(\xi_*,\eta_*)\in TCP(\Sigma)$;
\item[(\ref{chapproofmax}.A1.8)] {\em $\partial\xi_*\in\mathcal{S}$ 
(cf. Definition \ref{bvalue} below);}
$U$-control $\eta$
\item[(\ref{chapproofmax}.A1.9)] {\em the ``Lagrangian'' $L$ is a function}
$$
\Omega\times U\times I\ni(x,u,t)\mapsto L(x,u,t)\in\rr\,.
$$
\end{itemize}\end{itemize}

 
\pparagraph{Minimizers}
Suppose we are given a system $\Sigma$ as in (\ref{flasystem}),
and a Lagrangian $L$ as in (\ref{chapproofmax}.A1.9).

\begin{definition}\label{defoaugmen}
{\rm An {\bf\em $L$-augmented $\Sigma$-trajectory} for a $U$-control $\eta$
is an ordered pair $\Xi=(\xi^0,\xi)$ such that $\xi$ is a 
$\Sigma$-trajectory for $\eta$, 
$\xi^0:I_\xi\mapsto\rr$ is a locally \mbox{absolutely} continuous
function, and 
\begin{equation}\label{equacost}
\dot\xi^0(t)=L(\xi(t),\eta(t),t)\qquad{\rm for\,\,\,almost\,\,\,all}
\qquad t\in I_\xi\,.
\end{equation}

A {\bf\em full $L$-augmented $\Sigma$-trajectory} for $\eta$
is an augmented trajectory $\Xi=(\xi^0,\xi)$ such that 
$I_\xi=[a_\eta,b_\eta]$. We will use the expression $TCP(\Sigma,L;\eta)$
to denote the set of all full $L$-augmented
$\Sigma$-trajectories for $\eta$. 

An {\bf\em $L$-augmented trajectory-control pair 
for $\Sigma$} is a pair $\Gamma=(\Xi,\eta)$
such that $\Xi\in TCP(\Sigma,L;\eta)$. We  write 
$TCP(\Sigma,L)$ to denote the set of all \mbox{trajectory-control} pairs
for $\Sigma$.}\proofend
\end{definition}

\begin{definition}\label{defminim}
{\rm The {\bf\em cost functional} for the system
$\Sigma$ and the \mbox{Lagrangian} $L$ is the function 
${\cal J}_{\Sigma,L}:TCP(\Sigma,L)\mapsto\rr$ that assigns 
to each $L$-augmented 
trajectory-control pair $\Gamma=(\Xi,\eta)=(\xi^0,\xi,\eta)$ for $\Sigma$ 
the number ${\cal J}_{\Sigma,L}(\Gamma)$ given by}
\begin{eqnarray*}
{\cal J}_{\Sigma,L}(\Gamma)&=&{\cal J}_{\Sigma,L}(\Xi,\eta)\\
&\equaldef&{\cal J}_{\Sigma,L}(\xi,\eta)\\
&=&\xi^0(b_\eta)-\xi^0(a_\eta)\\
&=&\int_{a_\eta}^{b_\eta}L(\xi(t),\eta(t),t)\,dt\,.
\end{eqnarray*}

{\rm A {\bf\em minimizer} for the optimal control problem defined
by $\Sigma$, $L$, and $\mathcal{S}$, is an \mbox{$L$-augmented}
trajectory-control pair $\Gamma_*=(\xi_*^0,\xi_*,\eta_*)$ of $\Sigma$ 
such that $\partial\xi_*\in\mathcal{S}$ and 
\begin{itemize}\item[]\begin{itemize}
\item[(MIN)$\,\,$] {\sl ${\cal J}_{\Sigma,L}
(\xi_*^0,\xi_*,\eta_*)\le{\cal J}_{\Sigma,L}(\xi^0,\xi,\eta)$
for all $(\xi^0,\xi,\eta)\in TCP(\Sigma,L)$
such that $\partial\xi\in{\cal S}\,$,}
\end{itemize}\end{itemize}
i.e., $\Gamma_*$ minimizes the cost functional ${\cal J}_{\Sigma,L}$ 
among all $L$-augmented trajectory-control pairs $(\xi^0,\xi,\eta)$ of 
$\Sigma$ such that $\partial\xi\in{\cal S}$.}\proofend
\end{definition}

\begin{remark}\label{remiminim}
{\rm In Definition \ref{defminim} we introduced the notation 
${\cal J}_{\Sigma,L}(\xi,\eta)$ as an alternative for 
${\cal J}_{\Sigma,L}(\Xi,\eta)$, if $\Xi=(\xi^0,\xi)$.
This is justified because, if the pair $(\xi,\eta)$ is fixed,
then any two functions $\xi_1^0$, $\xi_2^0$, such that
$(\xi_1^0,\xi,\eta)$ and $(\xi_1^0,\xi,\eta)$ belong to
$TCP(\Sigma,L)$ must differ by a constant, so 
$\xi_1^0(b_\eta)-\xi_1^0=\xi_2^0(b_\eta)-\xi_2^0(a_\eta)$, 
and then ${\cal J}_{\Sigma,L}(\xi_1^0,\xi,\eta)=
{\cal J}_{\Sigma,L}(\xi_1^0,\xi,\eta)$.

If we think of ${\cal J}_{\Sigma,L}$ as a functional
$(\xi,\eta)\mapsto{\cal J}_{\Sigma,L}(\xi,\eta)$, then the domain
of ${\cal J}_{\Sigma,L}$ should be the set
$TCP_{L}(\Sigma)$ of all ``$L$-admissible'' trajectory-control
pairs of $\Sigma$, where a trajectory-control pair 
$(\xi,\eta)\in TCP(\Sigma)$ is said to be {\bf\em $L$-admissible}
if the function $[a_\eta,b_\eta]\ni t\mapsto L(\xi(t),\eta(t))$ 
is Lebesgue integrable.}\proofend
\end{remark}


\pparagraph{The optimal control problem} We will try to solve the following
\medbreak\begin{center}
\fbox{\begin{minipage}{12cm}
\begin{center}
{\large\bf OPTIMAL CONTROL PROBLEM (OCP)}
\end{center}

{\bf Given} data $n$, $\Sigma=(\Omega,U,I,f,\mathcal{U})$,  $L$,  
${\cal S}$, $\gamma_*=(\xi_*,\eta_*)$, such that \mbox{conditions}
(\ref{chapproofmax}.A1.1-2-3-4-5-6-7-8-9) hold;

{\bf find}: a necessary condition for the pair $\gamma_*=(\xi_*,\eta_*)$ 
to be a minimizer.
\end{minipage}}\end{center}

\pparagraph{The separation problem}
The data for the separation problem are the same as those for the
optimal control problem, except that there is no Lagrangian, and
instead of looking for minimizers we look for trajectories
that satisfy a separation condition. 

\begin{definition}\label{reacha}
{\rm The {\bf\em endpoint value set} (or 
{\bf\em endpoint condition set}, or
{\bf\em boundary value set}, or {\bf\em boundary condition set})
for a control system $\Sigma$ is the set  $\evset(\Sigma)$
of all \mbox{endpoint} values $\partial\xi$, for all pairs
$(\xi,\eta)\in TCP(\Sigma)$.}\proofend
\end{definition}

\begin{definition}\label{locsep}
{\rm If $A$, $B$ are subsets of a metric space $M$ (cf. \S \ref{secmetric}), 
we say  that $A$ is {\bf\em separated} from $B$ at a 
point $m\in M$ if $A\cap B=\{m\}$
We say  that $A$ is {\bf\em locally separated} from $B$ at a 
point $m\in M$ if $A\cap B\cap\mathcal{N}=\{m\}$
for some neighborhood $\mathcal{N}$ of $m$ in $M$.}\proofend
\end{definition}


\label{pagesepa}
\medbreak\begin{center}
\fbox{\begin{minipage}{12cm}
\begin{center}
{\large\bf SEPARATION PROBLEM (SP)}
\end{center}

{\bf Given}: data $n$, $\Sigma=(\Omega,U,I,f,\mathcal{U})$,  ${\cal S}$, 
$\gamma_*=(\xi_*,\eta_*)$,  such that conditions
(\ref{chapproofmax}.A1.1-2-3-4-5-6-7-8) hold;

{\bf find}: a necessary condition for the endpoint value set
$\evset(\Sigma)$ to 
be locally separated from ${\cal S}$ at $\partial\xi_*$.
\end{minipage}}\end{center}

\pparagraph{Reduction of the optimal control problem to the separation
problem} Having solved the SP, we show how to derive as a corollary a
solution of the OCP. This will be an easy matter, for reasons that we
now briefly outline. (The detailed proof, with a precise specification
of all relevant technical conditions, will be given later.)

Suppose that the $L$-augmented pair $\Gamma_*=(\Xi_*,\eta_*)
=(\xi_{*}^0,\xi_*,\eta_*)$ is a  \mbox{minimizer} for the 
OCP with \mbox{data} \ $n$, $\Sigma=(\Omega,U,I,f,\mathcal{U})$,  $L$, and 
${\cal S}$. We let $\hat\Sigma$ be the ``\mbox{augmented} control 
system''  $(\hat\Omega,U,I,\hat f,\mathcal{U})$,  where 
$\hat\Omega=\rr\times\Omega$ and
\begin{equation}\label{eqforaugsys}
\hat f(x^0,x,u,t)=(L(x,u,t),f(x,u,t))\quad{\rm if}\quad
(x^0,x)\in\hat\Omega\,,\,\,u\in U\,,\,\,t\in I\,.
\end{equation}
That is, $\hat\Sigma$ is the system 
\begin{equation}\label{flasforaugsys}
\hat\Sigma:\qquad\dot x^0=L(x,u,t)\,,\,\,
\dot x=f(x,u,t)\,,\,\,(x^0,x)\in\rr\times\Omega\,,\,\,u\in U\,,
\,\,u(\cdot)\in\mathcal{U}\,,
\end{equation}
obtained from $\Sigma$ by adding the ``running cost'' $x^0$ as a new state 
variable. 

If we write  \ $a_*=a_{\eta_*}$,  \ $b_*=b_{\eta_*}$, 
\ $x_*=\xi_*(a_{*})$, \ $\tilde x_*=\xi_*(b_{*})$, 
\ $x_*=\xi_*^0(a_{*})$, \mbox{$\tilde x_*=\xi_*^0(b_{*})$}, 
\mbox{$c_*=\tilde x_*^0-x_*^0=\mathcal{J}_{\Sigma,L}(\Gamma_*)$}, and define
$\hat{\cal S}$ to be the set of all ordered pairs
$\Big((x^0,x),(\tilde x^0,\tilde x)\Big)\in\hat\Omega\times\hat\Omega$
such that
\begin{itemize}
\item[(a)] $(x,\tilde x)\in{\cal S}$,
\item[(b)] $\tilde x^0-x^0\le c_*-\|x-x_*\|^2-\|\tilde x-\tilde x_*\|^2$
\item[(c)] $x^0=x_*^0$,
\end{itemize}
then it is clear that $\Xi_*$ is a full $\hat\Sigma$-trajectory for 
$\eta_*$ and $\partial\Xi_*\in\hat{\cal S}$.

Furthermore, if $Q\in\evset(\hat\Sigma)\cap\hat{\mathcal{S}}$,
then there must exist a trajectory-control pair 
\mbox{$\Gamma=(\Xi,\eta)$} of
$\hat\Sigma$ such that $Q=\partial\Xi\in\hat{\mathcal{S}}$.  \ If we write
$a=a_\eta$, $b=b_\eta$, \mbox{$\Xi=(\xi^0,\xi)$}, $x=\xi(a)$, 
$\tilde x=\xi(b)$, $x^0=\xi^0(a)$, $\tilde x^0=\xi^0(b)$, 
$c=\tilde x^0-x^0=\mathcal{J}_{\Sigma,L}(\Gamma)$, then the fact
that $\partial\Xi\in\hat{\mathcal{S}}$ implies that
$\partial\xi\in\mathcal{S}$ and
\begin{equation}\label{jess}
c\le c_*-\|x-x_*\|^2-\|\tilde x-\tilde x_*\|^2\,.
\end{equation}
Since $\Gamma_*$ is a minimizer, and $\partial\xi\in\mathcal{S}$, we have 
\mbox{$\mathcal{J}_{\Sigma,L}(\Gamma_*)\le\mathcal{J}_{\Sigma,L}(\Gamma)$}, 
i.e., $c_*\le c$. Therefore 
$c_*\le c\le c_*-\|x-x_*\|^2-\|\tilde x-\tilde x_*\|^2$,
from which it follows that $x=x_*$, $\tilde x=\tilde x_*$,  and $c=c_*$. 
Since $x^0=x_*^0$, we can conclude that $\tilde x^0=\tilde x_*^0$, so 
that $\partial\Xi=\partial\Xi_*$.   Hence $Q=\partial\Xi_*$. 

We have shown that
$\evset(\hat\Sigma)\cap\hat{\mathcal{S}}=\{\partial\Xi_*\}$.
Therefore $\evset(\hat\Sigma)$ and $\hat{\mathcal{S}}$ are locally
separated at $\partial\Xi_*$.  We can the apply the necessary
condition for separation given by the solution of the SP, and obtain a
necessary condition for minimization for OCP. We will carry out the
details later, after we have described the precise technical hypotheses
for our results. \proofend


So we assume we are given data $n$, $U$, $f$, $\Sigma$, $\eta_*$,
$\xi_*$, $a$, $b$, ${\cal S}$ as in the Separation Problem box (page
\pageref{pagesepa}), and
begin by outlining the construction of packets of needle variations.
For simplicity, we will present the outline only for the fixed time
interval case, although when we do the technical details rigorously 
we will take care of both cases.


The main purpose of the construction of packets of needle variations
is to compute, for our given reference trajectory-control pair
$(\xi_*,\eta_*)$, a ``first-order approximation'' ${\cal P}$ of the
set ${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$ near the point $\xi_*(b)$. The
set ${\cal P}$ should be a convex cone in $\rr^n$, and the fact that
it is a ``first-order approximation'' of ${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$
near $\xi_*(b)$ requires a precise definition to be given later, but means,
to begin with, that each member of ${\cal P}$ is the tangent 
vector $e$ at $\eps=0$ to a curve $\gamma:[0,\bar\eps]\mapsto
{\cal R}_{[a,b]}^\Sigma(\xi_*(a))$ such that 
$\gamma(0)=\xi_*(b)$. 

If we have another convex cone ${\cal Q}$ which is 
``tangent'' to ${\cal S}$ in a similar way, then it is natural
to guess that a necessary condition for separation of the sets 
${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$ and ${\cal S}$ should be 
the linear separation of the cones ${\cal P}$ and 
${\cal Q}$,  i.e., the existence of a nontrivial linear functional
$\lambda:\rr^n\mapsto\rr$ such that 
$\langle\lambda,v\rangle\ge 0$ for all $v\in{\cal P}$ 
and $\langle\lambda,v\rangle\le 0$ for all $v\in{\cal Q}$.

It turns out, however, that if we use the na\"\i{}ve concept
of ``tangent cone'' hinted at in the previous discussion, 
it is not true that ``if two sets ${\cal R}$ and ${\cal S}$ are separated 
at a point $x_*$, and have tangent cones 
${\cal P}$ and ${\cal Q}$ at $x_*$, then the cones 
${\cal P}$ and ${\cal Q}$ are linearly separated.''



Roughly speaking, a (fixed time interval) ``variation'' of the
reference control $\eta_*$ is a 
``curve'' $[0,\bar\eps]\ni\eps\mapsto\eta^\eps$
in the space $A_{[a,b]}(\Sigma)$ of all control
functions $[a,b]\ni t\mapsto\eta(t)\in U$
defined on $[a,b]$ that starts at 
$\eta_*$. That is, a {\bf\em variation} of
$\eta_*$ is a one-parameter family 
$V=\{(\xi^\eps,\eta^\eps):\eps\in[0,\bar\eps\}$ of 
trajectory-control pairs of $\Sigma$ such that 
$(\xi^0,\eta^0)=(\xi_*,\eta_*)$.


Roughly speaking, a (fixed time interval) ``variation'' of the
reference trajectory-control pair $(\xi_*,\eta_*)$ is a 
``curve'' $[0,\bar\eps]\ni\eps\mapsto(\xi^\eps,\eta^\eps)$
in the space $TCP_{[a,b]}(\Sigma)$ of all trajectory-control
pairs of $\Sigma$ defined on $[a,b]$ that starts at 
$(\xi_*,\eta_*)$. That is, a {\bf\em variation} of
$(\xi_*,\eta_*)$ is a one-parameter family 
$V=\{\eta^\eps:\eps\in[0,\bar\eps\}$ of controls defined on $[a,b]$
such that $\eta^0=\eta_*$.

Each control function $\eta:[a,b]\mapsto U$ gives rise to a 
{\bf\em time-varying vector field} $f_\eta$, defined by letting
$f_\eta(x,t)=f(x,\eta(t),t)$.



We assume that we are given 
 


of the given 
at \mbox{various} times $\tau$ in the domain $[a,b]$ of  $\xi_*$.
The effects of these
variations are then \mbox{propagated} to the terminal time b$b$
by means of the \mbox{differentials} $D\Phi_{b,\tau}(\xi_*(\tau))$
of the reference flow maps $\Phi_{b,\tau}$. This
produces a convex cone (the ``Pontryagin cone'') which in some sense
is a first-order approximation to the attainable set near the
point $\xi_*(b)$. The proof is
then concluded by means of a \mbox{topological} \mbox{argument}, based
on the Brower fixed point theorem, to
pass from the \mbox{separation} of two sets to the separation of their
approximating cones. 

Each needle variation consists of a family 
$V=\{\xi^\eps\}_{\eps\in[0,\bar\eps]}$ of trajectories, which are defined 
on $[a,b]$ if $V$ is an ``equal time variation,'' or on intervals
$[a^\eps,b^\eps]$ if $V$ is a ``variable time variation,''
and are such that $\xi^0=\xi_*$ and $\xi^\eps(a^\eps)=\xi_*(a)$ 
afor all $\eps$. Under suitable technical conditions, the terminal
points $\xi^\eps(b^\eps)$ of the arcs $\xi^\eps$ describe a 
continuous curve $\gamma_{_V}:[0,\bar\eps]\mapsto X$ (where $X$ is the state
space of our control system), which is differentiable 
at $\eps=0$. The ``effect'' of the variation $V$ at the terminal 
point is then the vector 
$e_{_V}=\frac{d\,\,}{d\eps}\Big|_{\eps=0}\gamma_{_V}(\eps)$.
This vector is, by construction, ``tangent'' to the set 
${\cal R}$




\end{document}









