%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% This message contains:  
%
%   1. A Latex file with the paper   
%
%                   GEOMETRY AND OPTIMAL CONTROL
%
%                      by Hector J. Sussmann
%
%   to appear in ``Mathematical Control Theory'' (a book of essays a in
%   honor of Roger W. Brockett on the occasion of his 60th Birthday),
%   J. Baillieul and J. C. Willems Eds., Springer-Verlag, 1998. (ISBN
%   0-387-98317-1.)   
%
%   2. The style file ``svcon2ehjs.sty''.
%
%   To run Latex and produce a .dvi file, you should first save
%   svcon2ehjs.sty separately. You can find where ``svcon2ehjs.sty'' 
%   starts by looking for the keyword GNUSTYLE.
%
%   The paper is 59 pages long. 
%
%   The date of this version  is
%
%                       October 3, 1998.
%
%   This version differs slightly from the one to be published, in that
%   some typographical errors found after the book went to press have
%   been corrected.  
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentclass{report}
  \def\dedicate{
  \vspace{0.2in}
\noindent{\large{\bf\em Dedicated to Roger W. Brockett on his 60th Birthday}}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  \usepackage{svcon2ehjs,latexsym}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\font\mynine=cmr9
\font\mysmall=cmr7
\font\mysmallit=cmti7
\font\mynineit=cmti9
\font\myninebf=cmbx9
  \begin{document}
  \baselineskip=10.6pt
  \pagenumbering{arabic} 
  %\tableofcontents 
  \chapter{Geometry $\!$and $\!$Optimal $\!$Control$^*$}\label{geome}
  \chapterauthors{H. J. Sussmann$^\dagger$}

\pagestyle{myheadings}
%\iographystyle{abbrv}
%\input{macros}
 
% \def\EOP{\hfill \vrule height4pt width4pt depth2pt\par\bigskip}
\def\md{M\!D}
\def\sd{S\!D}
 \newtheorem{theorem}{Theorem}[section]
 \newtheorem{proposition}{Proposition}[section]
 \newtheorem{lemma}{Lemma}[section]
 \newtheorem{definition}{Definition}[section]
 \newtheorem{example}{Example}[section]
 \newtheorem{corollary}{Corollary}[section]
 \newtheorem{remark}{Remark}[section]
 \newtheorem{assumption}{Assumption}[section]
 \newtheorem{conjecture}{Conjecture}[section]
 
%\textwidth 


\renewcommand{\theequation}{\arabic{equation}}

\renewcommand{\thesection}{5.\arabic{section}}

\newcommand{\dis}{\displaystyle}
\newcommand{\res}{(sI-A)^{-1}}
\newcommand{\resa}{(sI-A_{11})^{-1}}
\newcommand{\resam}{(-sI-A_{11})^{-1}}
\newcommand{\resamt}{(-sI-A_{11}^T)^{-1}}
\newcommand{\Wp}{W(s)^{-1}}
\newcommand{\Wm}{W(-s)^{-1}}
\newcommand{\sigone}{\Sigma_1}
\newcommand{\sigtwo}{\Sigma_2}
\newcommand{\cross}{\otimes}

%%%%Macros begin here
\def\mybox{$\lower3.5pt\vbox{\hrule height2.6pt depth-2pt\hbox{\vrule width0.6pt\kern1pt\vbox{\kern1pt\phantom{nj}\kern1pt}\kern1.5pt\vrule width0.6pt}\hrule height2.6pt depth-2pt}$\medbreak}
%\def\proofend{\enspace\hfill \mybox}
\def\proofend{\enspace\hfill$\Diamond$}
\def\restr{\lceil}
\def\compo{\raise 0.1em\hbox{$\,_\circ\,$}}
\def\nn{{\rm I}\kern-.2em{\rm N}}
\def\pmb#1{\setbox0=\hbox{$#1$}%
  \kern-.025em\copy0\kern-\wd0
  \kern.05em\copy0\kern-\wd0
  \kern-.025em\raise.0433em\box0 }
\def\equaldef{\,{\buildrel \rm def \over =}\,}
\def\rr{{\rm I}\kern-.2em{\rm R}}
\def\bb{{\rm I}\kern-.2em{\rm B}}
\newcommand{\RR}{\mbox{${\rm \: 
R\!\!\!\!\!\!\!\; I \;\;}$}} 
\newcommand{\II}{\mbox{${\rm \: 
I\!\!\! I \:}$}} 
\newcommand{\CC}{\mbox{${\rm \: 
C\!\!\! I \;\;}$}} 
\newcommand{\proj}[1]{\mbox{${\bf P} _{#1} $}}
%\newcommand{\abinv}[1]{\mbox{$(A,B)$-invariant}}
\newcommand{\abinv}[1]{\mbox{controlled-invariant }}

\newcommand{\ba}{{\bf a}} 
\newcommand{\bc}{{\bf c}}
\newcommand{\be}{{\bf e}}
\newcommand{\bg}{{\bf g}}
\newcommand{\bh}{{\bf h}}
\newcommand{\bi}{{\bf i}}
\newcommand{\bj}{{\bf j}}
\newcommand{\bk}{{\bf k}}
\newcommand{\bl}{{\bf l}}
\newcommand{\bm}{{\bf m}}
\newcommand{\bn}{{\bf n}}
\newcommand{\bo}{{\bf o}}
\newcommand{\bp}{{\bf p}} 
\newcommand{\bq}{{\bf q}} 
\newcommand{\br}{{\bf r}}
\newcommand{\bs}{{\bf s}}
\newcommand{\bt}{{\bf t}}
\newcommand{\bu}{{\bf u}}
\newcommand{\bv}{{\bf v}}
\newcommand{\bw}{{\bf w}}
\newcommand{\bx}{{\bf x}}
\newcommand{\by}{{\bf y}}
\newcommand{\bz}{{\bf z}}
\newcommand{\te}{\tilde{e}}
\newcommand{\tA}{\tilde{A}}
\newcommand{\tB}{\tilde{B}}
\newcommand{\tN}{\tilde{N}}
\newcommand{\tM}{\tilde{M}}
\newcommand{\tV}{\tilde{V}}
\newcommand{\tU}{\tilde{U}}
\newcommand{\tT}{\tilde{T}}
\newcommand{\tW}{\tilde{W}}
\newcommand{\tP}{\tilde{P}} 
\newcommand{\tQ}{\tilde{Q}} 
\newcommand{\tR}{\tilde{R}} 
\newcommand{\tX}{\tilde{X}}
\newcommand{\tY}{\tilde{Y}} 
\newcommand{\tZ}{\tilde{Z}} 
\newcommand{\Ht}{{H}^2} 
\newcommand{\tD}{\tilde D}
\newcommand{\tE}{\tilde E}
\newcommand{\tF}{\tilde F}
\newcommand{\tK}{\tilde K}
\newcommand{\tL}{\tilde L}
\newcommand{\Hf}{  {H}^{\infty} }
\newcommand{\hf}{\hat{f}}
\newcommand{\hv}{\hat{v}}
\newcommand{\hA}{\hat{A}} 
\newcommand{\hB}{\hat{B}} 
\newcommand{\hC}{\hat{C}} 
\newcommand{\hD}{\hat{D}} 
\newcommand{\hE}{\hat{E}} 
\newcommand{\hF}{\hat{F}} 
\newcommand{\hG}{\hat{G}} 
\newcommand{\hH}{\hat{H}}
\newcommand{\hh}{\hat{h}} 
\newcommand{\hM}{\hat{M}} 
\newcommand{\hN}{\hat{N}} 
\newcommand{\hP}{\hat{P}} 
\newcommand{\hQ}{\hat{Q}} 
\newcommand{\hR}{\hat{R}} 
\newcommand{\hS}{\hat{S}} 
\newcommand{\hT}{\hat{T}} 
\newcommand{\hU}{\hat{U}} 
\newcommand{\hV}{\hat{V}} 
\newcommand{\hX}{\hat{X}} 
\newcommand{\hY}{\hat{Y}} 
\newcommand{\hZ}{\hat{Z}} 
\newcommand{\CA}{{\cal A}} 
\newcommand{\CB}{{\cal B}} 
%\newcommand{\CC}{{\cal C}} 
\newcommand{\CP}{{\cal P}} 
\newcommand{\CQ}{{\cal Q}} 
\newcommand{\CV}{{\cal V}} 
\newcommand{\cA}{{\cal A}} 
\newcommand{\cB}{{\cal B}}
\newcommand{\cC}{{\cal C}}
\newcommand{\cD}{{\cal D}}
\newcommand{\cF}{{\cal F}} 
\newcommand{\cG}{{\cal G}} 
\newcommand{\G}{{\cal G}}
\newcommand{\cH}{{\cal H}}
\newcommand{\cK}{{\cal K}}
\newcommand{\cI}{{\cal I}}
\newcommand{\cL}{{\cal L}}
\newcommand{\cM}{{\cal M}}
\newcommand{\cN}{{\cal N}}
\newcommand{\cO}{{\cal O}} 
\newcommand{\cP}{{\cal P}} 
\newcommand{\cQ}{{\cal Q}}
\newcommand{\cR}{{\cal R}}
\newcommand{\cS}{{\cal S}}
\newcommand{\cT}{{\cal T}}
\newcommand{\cU}{{\cal U}}
\newcommand{\cV}{{\cal V}}
\newcommand{\cW}{{\cal W}}
\newcommand{\cX}{{\cal X}}
\newcommand{\cY}{{\cal Y}}
\newcommand{\cZ}{{\cal Z}}

\newcommand{\beeq}{\begin{eqnarray*}}
\newcommand{\eeeq}{\end{eqnarray*}}
\newcommand{\obo}[1]{
  \left[ \begin{array}{c}
                   #1
 \end{array} \right] }


\newcommand{\obt}[2]{
  \left( \begin{array}{cc}
 #1 & #2
\end{array} \right) }
\newcommand{\tbo}[2]{
  \left( \begin{array}{c}
 #1 \\ #2
\end{array} \right) }


\newcommand{\proof}{{\noindent \bf Proof:\ }}
\newcommand{\eproof}{\hfill $\Box$

\vspace{10mm} \noindent
}
\newcommand{\sq}{\frac{1}{2}}
\newcommand{\ts}[1]{{\hbox{\tiny $#1$}}}

\newcommand{\oA}{\overline{A}}
\newcommand{\oB}{\overline{B}}
\newcommand{\oC}{\overline{C}}
\newcommand{\oD}{\overline{D}}
\newcommand{\oE}{\overline{E}}
\newcommand{\oF}{\overline{F}}
\newcommand{\oG}{\overline{G}}
\newcommand{\oH}{\overline{H}}
\newcommand{\oN}{\overline{N}}
\newcommand{\oK}{\overline{K}}
\newcommand{\oL}{\overline{L}}
\newcommand{\oM}{\overline{M}}
\newcommand{\oP}{\overline{P}}
\newcommand{\oQ}{\overline{Q}}
\newcommand{\oR}{\overline{R}}
\newcommand{\oS}{\overline{S}}
\newcommand{\oT}{\overline{T}} 
\newcommand{\oU}{\overline{U}}
\newcommand{\oV}{\overline{V}}
\newcommand{\oW}{\overline{W}}
\newcommand{\oX}{\overline{X}}
\newcommand{\oY}{\overline{Y}}
\newcommand{\oZ}{\overline{Z}}
\newcommand{\os}{\overline{s}}
\newcommand{\ob}{\overline{b}}
\newcommand{\oa}{\overline{a}}
\newcommand{\ow}{\overline{w}}
\newcommand{\oz}{\overline{z}}
\newcommand{\bff}{{\bf f}}
\newcommand{\bA}{{\bf A}}
\newcommand{\bE}{{\bf E}}
\newcommand{\bF}{{\bf F}}
\newcommand{\bH}{{\bf H}}
\newcommand{\bP}{{\bf P}}
\newcommand{\bR}{{\bf R}}
\newcommand{\bS}{{\bf S}}
\newcommand{\bX}{{\bf X}}
\newcommand{\bY}{{\bf Y}}
\newcommand{\bZ}{{\bf Z}}
\newcommand{\bfR}{{\bf R}}
\newcommand{\bfC}{{\bf C}}

\newcommand{\codim}{\mbox{codim$\,$}}
\newcommand{\Ker}{\mbox{\rm Ker$\,$}}
\newcommand{\Img}{\mbox{\rm Im$\,$}}
\newcommand{\In}{\mbox{\rm In$\,$}}
\newcommand{\cir}{\mbox{\rm circ$\,$}}
\newcommand{\mod}{\mbox{\rm mod$\,$}}
\newcommand{\rank}{\mbox{\rm rank$\,$}}
\newcommand{\sign}{\mbox{\rm sign$\,$}}
\newcommand{\diag}{\mbox{\rm diag$\,$}}
\newcommand{\adj}{\mbox{\rm adj$\,$}}
\newcommand{\nullit}{\mbox{\rm null$\,$}}
\newcommand{\Res}{\mbox{\rm Res$\,$}}
\newcommand{\trace}{\mbox{\rm trace$\,$}}
\newcommand{\spn}{\mbox{\rm span$\,$}}
\newcommand{\projh}{\mbox{\bf $\hat{P}$}}
\newcommand{\projt}{\mbox{\bf $\tilde{P}$}}

%%%%%%%%%%%%%intro.tex%%%%%%%%%%%%%%%%

\markboth{H. J. Sussmann}{5.$\,\,\,$Geometry and Optimal Control}

\setcounter{page}{140}
\dedicate

\vspace{0.5cm}

\begin{minipage}{109mm}\baselineskip=9.6pt
\noindent ABSTRACT Optimal control has strongly influenced geometry since
the early days of 
both subjects.  In particular, it played a crucial role in the birth of
differential geometry in the nineteenth century through the revolutionary
ideas of redefining the notion of ``straight line'' (now renamed
``geodesic'') by 
means of a curve minimization problem, and of emphasizing general
invariance and covariance conditions.  More recently, modern control theory
has been heavily influenced by geometry. One aspect of this
influence is the geometrization of the necessary conditions for optimality,
which are recast as geometric conditions about reachable sets, thus becoming
special cases of the broader question of the structure and properties of
these sets.  Recently, this has led to a new general version of the
finite-dimensional maximum principle, stated here in full detail for the
first time. A second aspect---in which Roger Brockett's ideas
have played a crucial role---is the use in control theory of concepts and
techniques from differential geometry.  In particular, this leads to
regarding a control system as a collection of vector fields, and exploiting
the algebraic structure given by the Lie bracket operation.
This approach has led to new important developments on various nonlinear
control problems.  In optimal control, the vector-field view has produced
invariant formulations of the maximum principle on manifolds---using
either Poisson brackets or connections along curves---which, besides being
more general and mathematically natural, actually have real advantages for
the solution of concrete problems. 
\end{minipage}

\begin{minipage}{109mm}
\phantom{aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}
\end{minipage}

\vspace{-0.7cm}

\section{Introduction}
\label{INTRO}
%\bigskip

The interplay among control, geometry, and physics goes back to ancient
times. The words ``control theory'' are, of course, of recent origin, but
the subject itself is much older, since it contains the classical calculus
of variations as a special case, and the first calculus of variations
problems go back to classical Greece.  (The same is true of the calculus of
variations, whose name was introduced by Euler in 1755, although the
subject itself is at least as old as the classical isoperimetric problem.)

In this discussion we will describe some aspects of the relationship between
optimal control and geometry, beginning with a brief outline of some events
prior to the formulation of the Maximum Principle in the 1950s, and 

\noindent{\hbox to 3cm{\hrulefill}}


\noindent\begin{minipage}{112mm}\mysmall\baselineskip=7.7pt\parindent=7pt

\noindent$^*$ This paper has appeared in ``Mathematical Control Theory'' (a
book of essays a in honor of Roger W. Brockett on the occasion of his 60th
Birthday), J. Baillieul and J. C. Willems Eds., Springer-Verlag, 1998 (ISBN
0-387-98317-1), pages 140-198. The date of this version is {\mysmallit
October 3, 1998}. This version differs slightly from the one that has
appeared in print, because some typographical errors found after the book
went to press have been corrected.

\noindent$^\dagger$ Work supported in part by NSF Grant
DMS95-00798 and AFOSR Grant 0923.
\end{minipage}

\noindent then 
focusing on the geometrization of optimal control theory that took place
after 1960.  The pre-Maximum Principle relationship is primarily one-sided,
with optimal control (in the form of its ancestor, the calculus of
variations) exercising a decisive influence on geometry, and serving as the
source of some of the ideas that revolutionized geometry, such as
differential geometry.  In the much shorter post-Maximum Principle era, the
influence goes mainly in the other direction, with control theory being
heavily influenced by geometric ideas.  (An important exception is provided 
by recent developments in subriemannian geometry, discussed by V. Jurdjevic 
in Chapter 7 of this volume.)

The geometrization of control theory has brought into the field many
mathematical concepts and structures borrowed from differential geometry
and related areas---such as Lie algebras of vector fields, integral
manifolds, systems of differential forms and Cartan prolongations,
distributions\footnote{\baselineskip=7pt That is, subbundles of a tangent
bundle.}, Carnot-Carath\'eodory metrics, Lie groups, homogeneous
spaces, symplectic structures,
Hamiltonian mechanics---together with tools from real analytic geometry
such as subanalytic sets, stratifications and desingularization. The
combined use of these tools has led to new results on several important
control-theoretic questions, such as (i)~controllability, observability and
minimal realizations of nonlinear systems; (ii)~path finding;
(iii)~feedback stabilization; (iv)~system equivalence under state space
diffeomorphisms, static feedback, and dynamic feedback; (v)~system
linearization under state space diffeomorphisms and under feedback;
(vi)~disturbance decoupling; (vii)~tracking; and (viii)~optimal control.

Our primary area of interest here will be {\sl optimal control}, and
we will not review other aspects of the differential-geometrization of
nonlinear control theory since those topics have been extensively
discussed elsewhere---for example in the books by Jurdjevic
\cite{vel}, Isidori \cite{isidori}, Nijmeijer and var der Schaft
\cite{nijmeijer}, and the survey books \cite{bromilsus},
\cite{amsbook}, \cite{jakures}, \cite{mordusus},
\cite{sussworkshop}---or are covered by other papers in this volume,
e.g. in the contributions by A.J. Krener and V. Jurdjevic.  One of the
aspects that will not be covered here is the relationship with
mechanics, which is beautifully described in the paper by Bloch and Crouch
(Chapter 8 in this volume).

Even more narrowly, we will focus on the {\sl geometrization of the Maximum
Principle} based on two independent ideas: {\sl first}, the ``differential
geometric'' formulation, invariant under arbitrary nonlinear changes of
coordinates, and {\sl second}, the reformulation of the Maximum Principle
as a geometric property of reachable sets, namely, a condition for a
reachable set and some other set to be {\sl separated}.  Underlying both
ideas lies the emphasis on regarding a control system as a {\sl system of
vector fields}\footnote{\baselineskip=7pt For example, Kaskosz and
\L{}ojasiewicz proposed in \cite{kaskoloja} to refer to a general family
of time-varying vector fields as a ``generalized control system.''  We
adopt the more radical view that the Kaskosz-\L{}ojasiewicz approach
provides the most reasonable way to define the concept of a {\em control
system}.}. This makes it possible to exploit the algebraic structure of the
space of vector fields, given by the usual linear operations and the Lie
bracket.  As will be explained below, this can be used to do at least two
important things, namely, (a)~give a coordinate-free user-friendly
formulation of the Maximum Principle, and (b)~generate high-order
variations and obtain high-order necessary conditions for optimality.
Since the first idea, pertaining to the differential-geometrization of the
Maximum Principle, has already been described at length in Sussmann
\cite{sussfree}, we will pay special attention here to the second idea, and
mainly concentrate on one of its by-products, namely, our recent general
version of the finite-dimensional Maximum Principle.  This result, of which
several special cases have already been presented in preliminary
announcements in Sussmann [42]-[45],
will be stated here in a complete, self-contained form.

A natural alternative to the systems of vector fields point of view is that
of {\sl differential inclusions.}  For about 30 years it has been widely
believed that the differential inclusions approach is strictly more general
than the one based on vector field systems, owing to the fact that
set-valued maps, even if they are very smooth, do not usually admit
continuous single-valued selections.  (For example, the map
$z\to|z|^{N}\sqrt{z}$, on the complex plane, has two values at each nonzero
point, and is of class $C^N$, but has no continuous single-valued
selections on any neighborhood of the origin.)

We will argue that---at least as far as the finite-dimensional Maximum
Principle is concerned---the vector-field point of view is actually
sufficient to cover the differential inclusions case. The reason for this
is that all the differential inclusions for which there appears to be a
version of the Maximum Principle are given by set-valued maps that admit
sufficiently many selections that, even though they may fail to be
continuous, are nice enough to make it possible to construct needle
variations and extend the method of proof of the classical Maximum
Principle to a differential-inclusions setting.  (These facts are based on
ideas due to A. Bressan [5]-[7].
The application to the Maximum Principle is discussed in Sussmann
\cite{sussrecent}.)  So
the vector-fields point of view suffices, provided that we allow our vector
fields to be very nonsmooth (i.e., not necessarily Lipschitz continuous or
even continuous).

The key mathematical tools needed to carry out this program turn out to be
\begin{itemize}
\item[I.] A concept of ``generalized differential'' having the right
properties, especially the chain rule and suitable open mapping and
transversal intersection theorems.  Two such concepts---the
``semidifferentials'' and the \ ``multidifferentials''---were introduced in
\ Sussmann [42]-[45], [47], \mbox{and will be reviewed here.}
\item[II.] Separation and transversality theorems, giving necessary
conditions for two sets $S_1$, $S_2$ to be ``separated'' at a point $q$, in
the sense that $S_1\cap S_2=\{q\}$.  These conditions are stated in terms of
objects $D_j$ that ``linearly approximate'' the sets $S_j$ near
$q$. A natural choice is to take the $D_j$ to be 
multidifferentials---at $(0,q)$---of set-valued maps $F_j$ that map cones 
$C_j$ into the $S_j$.    
\item[III.] An extension to set-valued maps (cf. Theorem 
\ref{theobrowder} and Remark \ref{rembrowder}) of 
a theorem from homotopy theory, originally formulated 
by Leray and Schauder \cite{leray}, and proved rigorously by Browder in
\cite{browder} (cf. also Rabinowitz \cite{rabi1,rabi2}),
about the existence of connected sets of zeros of certain homotopies.
This makes it possible to extend the usual transversality results 
and include the sufficient condition for local controllability 
along a reference trajectory
as a special case of the maximum principle.
\item[IV.] An extension of Bressan's ideas of
\cite{bressanondif}-\cite{bressandir} to ``almost lower semicontinuous
maps,'' with a strengthening of the conclusions of his selection theorems, 
to obtain selections that are continuous at a given point.   
\item[V.] An extension to the pseudo-Lipschitz case 
of the results of Colombo {\it et al.\/} \cite{colomboetal}, and those of
Fryszkowski and 
Rze$\check{{\rm{z}}}$uchowski \cite{fry}, on uniform approximation of 
relaxed trajectories of a differential inclusion by ordinary 
trajectories. 
\end{itemize}



\noindent As explained in Sections \ref{BEYOND} and \ref{MULTI}, this
theory makes it possible to give a geometric, intrinsic version of the
Maximum Principle that covers the nonsmooth case of families of
Lipschitz-continuous vector fields, the differential inclusions case, and
many problems with discontinuous vector fields, while at the same time it
includes, in the smooth case, the extra results arising from high-order
conditions corresponding to complicated variations constructed using Lie
brackets.

\section{From Queen Dido to the Maximum Principle}
\label{DIDO}
%\bigskip

The first optimal control result ever discovered must have been the
statement that

\bigbreak\noindent (LS) {\it The shortest path
joining two points is a straight line segment.}


\bigbreak\noindent This is of course a rather trivial observation, but it
is undoubtedly (a)~very old, (b)~geometric, and (c)~a result in optimal
control. Moreover, we shall see later that, in spite of its deceptive
obviousness, (LS) is a fact of enormous importance that, properly
interpreted or reinterpreted, has led to key discoveries in geometry and
theoretical physics.

Next came the {\bf\em isoperimetric problem}
(IP)\footnote{\baselineskip=7pt According to the story told by Virgil in
the {\em Aeneid} about the foundation of Carthage ({\it ca.} 850 B.C.E.),
``when Dido landed in North Africa, she persuaded the local chief to sell
her as much land as an oxhide could contain. She then cut a hide into very
narrow strips, and joined them to make a continuous thread more than
two-and-a-half miles long. She was then able to enclose between the thread
and the sea the land on which Carthage was built'' (Pars \cite{pars}).},
the solution of which was known in classical Greece.  The IP is the problem
of finding, among all simple closed plane curves of a given length, one
that encloses the largest possible area.  Once again, this is obviously a
geometric question, and it is also an optimal control problem in at least
two ways:

\begin{itemize} 
\item[1.] If one uses the well-known formula
$A(\xi)=\frac{1}{2}\int_{\xi}(x\,dy-y\,dx)$ for the area enclosed by a
simple closed curve $\xi$, oriented counterclockwise, then the IP asks us
to maximize the integral $A(\xi)$ among all the curves $\xi:[0,1]\to\rr^2$
that satisfy endpoint constraints $\xi(0)=\xi(1)=(0,0)$ and the side
constraint $\int_0^1\Vert\dot\xi(t)\Vert\,dt=P$, where $P>0$ is given.

\item[2.] Alternatively, our curves can be thought of as trajectories
of a three-dimensional control system $\dot x=u$, $\dot y =v$,
$\dot{z}=vx-uy$, and the IP as the minimum-time optimal control problem for
this system, with control constraint $u^2+v^2=1$, as well as 
endpoint constraints
\mbox{$x_{\rm initial}=x_{\rm final}=y_{\rm initial}=y_{\rm final}=
z_{\rm initial}=0$}, $z_{\rm final}=\bar A$, where $\bar A>0$ is given.
\end{itemize} 

\noindent (The justification for the second formulation is that
minimizing the length of the boundary of a region given the area
is easily proved to be equivalent to maximizing the area
given the length of the boundary.)

If we use Formulation 1, then the IP is a standard fixed-endpoint
calculus of variations 
problem---minimize $I_0\equaldef\int_a^bL(\xi(t),\dot\xi(t))dt$ among all
curves $[a,b]\ni t\to \xi(t)$ such that $\xi(a)=\bar q$, 
$\xi(b)=\hat q$---with an extra constraint $I_1=\alpha$, where 
$I_1$ is an integral of the form $\int_a^bL_1(\xi(t),\dot\xi(t))dt$.
Calculus of variations problems of this form, with one or several
constraints $\int_a^bL_i(\xi(t),\dot\xi(t))dt=\alpha_i$, are in fact 
called {\it isoperimetric problems}.
\footnote{\baselineskip=7pt Throughout this paper, the letters 
$q$, $u$, and $p$ will denote the usual state, control and momentum variables. 
Greek letters will be used for {\sl curves} in the spaces of these
variables. For example, the letters $\xi$, $\eta$, and $\psi$ will denote 
curves in  $q$, $u$, and $p$ space, respectively.} 

If we use Formulation 2, then the IP is just a standard fixed-endpoint
minimum-time optimal control problem, with no extra qualifications.

The solution of the IP was known to be what the most obvious guess
immediately suggests, namely, a circle.  (It is almost evident that the
solution---modulo translations---has to be rotationally symmetric, since
the question, as stated, treats all directions in the plane equally.  This,
however, is not a rigorous argument, since it might conceivably happen that
there is ``symmetry breaking,'' so that the solution modulo translations is
not unique, and each solution individually is not rotationally symmetric,
even though the {\it set} of all solutions is.
\footnote{\baselineskip=7pt This is exactly what happens, for example, for
the problem of {\it minimizing} rather than maximizing an enclosed area. In
this case, the set $S$ of solutions is very large, and consists of {\sl
all} loops of length $P$ that enclose a zero area. (For example, $S$
contains all ``degenerate rectangles'' with two opposite sides of length
$P/2$ and the other two of length zero.) None of these solutions is
rotationally symmetric, but of course the family of all the solutions is
symmetric.}  To transform the rotational symmetry argument into a rigorous
proof that the solution of the IP is a circle, one would have to establish,
for example, uniqueness of the solution modulo translations.)

% GNU: Original was
% for example, uniqueness of the solution modulo translations.
% Not corrected for book. 

Knowing that the solution of the IP was a circle, Greek scholars could not
help thinking that {\it this fact had to mean something} about the
physical world, even if they couldn't quite tell what this might be.  For
example, Aristotle reasoned that   
\begin{center}\begin{minipage}{99mm}\baselineskip=9pt\noindent 
\noindent Clearly, the movement of the heavens must be the swiftest of all
movements.  Now of lines which return upon themselves the line which bounds
the circle is the shortest; and that movement is the swiftest which follows
the shortest line.  Therefore, if the heaven moves in a circle and moves
more swiftly than anything else, it must necessarily be spherical.
\end{minipage}\end{center}
To the extent that one can make sense of Aristotle's argument, it would
appear to be saying that planetary motion is circular because the circle is
the solution of the IP. For Aristotle's logic to work, even if we ignore
minor details such as the failure to distinguish between circles and
spheres, one needs to know or assume that, for some reason, planets are
trying to move as fast as possible, that each planet wants its orbit to
enclose in its motion some fixed---planet-dependent---area, and, finally,
that the speed of motion is a fixed---planet-dependent---constant.  These
assumptions---especially the planets' insistence on wanting to
enclose a given area---would be considered somewhat implausible today, even
by those who believe in a Creator who for some reason is seeking to
minimize some cost functional.

The first two examples of minimization properties of curves that were
truly tied to physics in a way that we still find credible today were 
two minimum principles of geometrical optics, namely:
\begin{itemize}
\item[1.] The observation by {\bf\em Hero of Alexandria} ({\it ca.}
65-{\it ca.} 125) that the laws of the {\it reflection} of light can be 
explained by assuming that light follows a shortest path.  
\item[2.]
The {\it minimum time principle} of {\bf\em Fermat} (1601-1665), 
according to which, in a medium of variable velocity of
propagation of light, the light rays are precisely the minimum-time
paths. 
\end{itemize}
Fermat's principle provides, in particular, an ``explanation'' for Snell's
Law of  refraction. The statement of Fermat's principle can be
interpreted in at least two ways, namely, (a)~as a description of the {\it
physics} of a certain process (light propagation)  that takes places in
ordinary space, in which the governing principle is {\it time minimization},
or (b)~as asserting that the governing principle is still that of
{\it motion along shortest paths}, except that the geometry of space is now
different---the ``length'' of a curve being, by definition, travel time along
the curve---so shortest paths no longer have to be ordinary line
segments. The first point of view looks simpler and more natural, but the
second one has turned out, in the long run, to be more
fruitful, as we shall see below. 

In 1686, {\bf\em Newton} studied the problem of characterizing 
the solid of revolution of least resistance. Mathematically, this means
that,  for given $a,b,\bar y,\hat y$, with $a<b$, 
we have to find the function $y=y(x)$ that minimizes the integral
$$I=\int_{a}^{b}\frac{x}{1+y'(x)^2}dx$$
subject to $y(a)=\bar y$, $y(b)=\hat y$. 

1n 1696, {\bf\em Johann Bernoulli} (1667-1748)
challenged the mathematicians of his time to
solve the ``brachystochrone problem,'' in which it is desired to determine,
among all curves $\xi$ going from a point $A$ to a point $B$ in a vertical
plane, the one for which a particle falling freely along $\xi$---subject
only to the action of the gravitational force plus whatever ``virtual
force'' is needed to keep the particle on $\xi$---will reach $B$ from $A$
in minimum time.  Mathematically, one can formulate this by observing that,
if our curve $\xi$ is given---in terms of a parameter $s$ which is an
increasing function of time $t$---by $s\to(x(s),y(s))$, $0\le s\le 1$, 
$A=(a,\alpha)$, $B=(b,\beta)$, and $t$ is time, initialized so that
$t(0)=0$, then the total energy 
$$E=\frac{1}{2}\Biggl(\Big(\frac{dx}{dt}\Big)^2+\Big(\frac{dy}{dt}\Big)^2\Biggr)+gy$$
is constant (taking the mass to be $1$).  So
$$dt=\sqrt{\frac{dx^2+dy^2}{2E-2gy}}\,.$$
If we assume that $a<b$, and make
the Ansatz that our minimizing curve is the graph of a function
\mbox{$x\to y(x)$}, then $dt=\sqrt{\frac{1+y'(x)^2}{2E-2gy}}\,dx$, so our
problem becomes that of minimizing the integral $\int_a^bL(y(x),y'(x))\,dx$,
subject to the constraints $y(a)=\alpha$, $y(b)=\beta$, where
\begin{equation}
L(y,u)\equaldef \sqrt{\frac{1+u^2}{2E-2gy}}\,.
\end{equation}
On the other hand, without assuming that $a<b$ and without making any 
{\it Ansatz} about $y$ being a function of $x$, we can formulate our problem as
that of minimizing the integral
\begin{equation}
I=\int_0^1\Lambda(x(s),y(s),\dot x(s),\dot y(s))\,ds\,,
\end{equation}
subject to $x(0)=a$, $y(0)=\alpha$, $x(1)=b$, $y(1)=\beta$, where 
\begin{equation}
\Lambda(x,y,u,v)\equaldef \sqrt{\frac{u^2+v^2}{2E-2gy}}\,.
\end{equation}
(The curves under consideration are, of course, restricted to lie 
in the closed half-plane $H_E=\{(x,y):y\le\frac{E}{g}\}$.)  
Finally, we can also use a {\it minimum-time optimal control} formulation, by
writing  the equations of motion (after 
choosing units so that $2g=1$, and changing $y$ to $2E-y$) as 
\begin{equation}\label{braccontrol}
\dot x=u\sqrt{y}\,,\,\,\,\dot y=v\sqrt{y}\,,\,\,\,u^2+v^2=1\,,
\end{equation}
where $x$ and $y$ are the state variables, $u$ and $v$ are the controls,
and $y$ is restricted to satisfy $y\ge 0$.   This third formulation
is clearly more natural than the first two, since it is a direct
mathematical translation of the problem. This is the first of a number of
reasons that in our view show that Johann 
Bernoulli's question properly  belongs to optimal control. (Cf. Sussmann and
Willems \cite{sussjan1,sussjan2} for a more detailed argument).

The brachystochrone problem attracted a lot of interest in 1696-97, leading
to the publication, in June 1697, of an issue of {\it Acta Eruditorum}
containing six responses to Johann Bernoulli's challenge, by 
Johann Bernoulli himself, his brother Jakob, Leibniz, Newton,
l'H\^opital, and Tschirnhaus.   

What made the problem appealing at the time was that the solution was a
curve that everybody knew---a cycloid---but there was no obvious way to
guess that this was so, and one had to resort to the new methods of
infinitesimal calculus that were being invented at that time in order to
figure out the formula for the solution.  (This should be contrasted with
the case of the isoperimetric problem---for which, as indicated above, it is
a fairly natural guess that the solution must be a circle---and that of
Newton's 1686 problem, whose solution is a curve of no particular 
significance.)

What makes the solution of the brachystochrone problem important for us
now, 300 years later, is that it marked the beginning of the systematic
study---which would be given the name ``calculus of variations'' by Euler
in 1755---of minimization problems of the form

\bigbreak
(SCV)\ \ \ \ \ \ \fbox{\begin{minipage}{65mm}
\noindent{\it Minimize} $\quad\quad \int_a^bL(\xi(t),\dot\xi(t),t)dt$

\noindent{\it subject to} $\quad\quad \xi(a)=\bar q\,,\ \ \ \ \ \ \xi(b)=\hat
q\,.$
\end{minipage}
}

\bigbreak\noindent
(Here we will refer to (SCV) as a {\it standard} calculus of variations
problem, to distinguish this question from other, ``nonstandard'' ones such
as isoperimetric problems and problems with constraints other than fixed 
endpoints.)

\begin{remark}{\rm  
Johann Bernoulli's problem happens to have a very significant
differential-geometric aspect, that Bernoulli himself failed to appreciate,
thereby failing to make an important discovery for which he had all the
clues in front of his eyes.  In fact, (\ref{braccontrol}) describes
the curves parametrized by arc-length for a Riemannian metric on the upper
half-plane, given by 
\begin{equation}\label{metricbrac}
ds^2=\frac{dx^2+dy^2}{y}\,.
\end{equation}
The cycloids that solve
the problem are the geodesics of this metric.  Johann Bernoulli studied the
same problem with other functions instead of $\sqrt{y}$ in
(\ref{braccontrol}), and in particular for the function
$y$, which in modern terms corresponds to the metric 
\begin{equation}\label{metricpoinc}
ds^2=\frac{dx^2+dy^2}{y^2}\,.
\end{equation}
In this case, he found that the minimum-time paths were {\bf\em
half-circles} rather
than cycloids.  So Johann Bernoulli had found what we would call today the
{\em Poincar\'e half-plane}, i.e., the simplest model of a non-Euclidean
(hyperbolic) geometry.  All that was missing was for him to take the
``small step'' of calling his travel time ``length,'' i.e. of thinking that
an expression such as (\ref{metricbrac}) or (\ref{metricpoinc}) could serve
as just another way of measuring {\em length}.  Had he hit upon 
this idea, it
would have been natural for him to turn (LS) around and regard it as a {\em
definition} of ``segment,'' for a new ``geometry'' in which ``length'' was
measured differently. This step---which would eventually be taken by
Riemann in 1854---was perfectly within Johann Bernoulli's reach, and would
have enabled him to show that his minimum-time arcs---for the Poincar\'e
half-plane---gave rise to a geometry where all of Euclid's axioms held,
except for the famous Fifth Postulate.\footnote{\baselineskip=7pt The 
cycloids that minimize arc-length for the metric (\ref{metricbrac}) 
do not have this property. For example, the length of these cycloids
is finite.} 

It is worth noting that Johann Bernoulli lived at a time when the question
of Euclid's postulate was at the center of mathematicians' interest. (For
example, the book {\rm Euclides ab Omni Naevo Vindicato}, by G. Saccheri
(1667-1733), claiming to give a proof of Euclid's Fifth Postulate in terms
of the other ones, was published in 1733.)  Moreover, Johann Bernoulli had
an obsessive desire for fame and glory,\footnote{\baselineskip=7pt He was
involved in many fights about 
priorities. He was jealous of the scientific achievements of his brother
Jakob and his own son Daniel, whom he once expelled from the house for
having dared to win a scientific prize for which the father was also a
candidate.}
so it is a sad irony that he, of all people, should have come so close
to a truly momentous discovery, and missed it.}\proofend
\end{remark}

%W. R. Hamilton: 1805-1865

With {\bf\em Euler} (1707-1783) and {\bf\em Lagrange} (1736-1813), 
it became clear that 
a necessary condition for a curve $[a,b]\ni t\to \xi_*(t)\in\rr^n$ 
to be a solution of a problem (SCV) 
is that it satisfy the {\it Euler-Lagrange equation}
\begin{equation}\label{eulerlag}
\frac{d}{dt}\frac{\partial L}{\partial\dot q}=\frac{\partial L}
{\partial q}\,,
\end{equation}
i.e., if we use $u$ instead of $\dot q$: 
\begin{equation}
\frac{d}{dt}\frac{\partial L}{\partial u}\Big(\xi_*(t),\dot\xi_*(t),t\Big)
=\frac{\partial L}{\partial q}\Big(\xi_*(t),\dot\xi_*(t),t\Big)\,,
\end{equation}
or
\begin{equation}
\frac{d}{dt}\frac{\partial L}{\partial u^i}\Big(\xi_*(t),\dot\xi_*(t),t\Big)
=\frac{\partial L}{\partial q^i}\Big(\xi_*(t),\dot\xi_*(t),t\Big)\,,\,\,\,\,
i=1,\ldots,n\,.
\end{equation}
In addition, Lagrange tried to free mechanics from ``geometry'' by doing
everything analytically.  But he unwittingly took a fundamental step
leading towards the geometrization of physics and the birth of differential
geometry, by proving the {\bf\em invariance of the Euler-Lagrange equations 
under arbitrary, nonlinear coordinate changes}. 

Lagrange's invariance theorem was the first example of what we would call
today a ``proof that some object is differential-geometrically intrinsic.''
In modern terminology, Lagrange's result says that (\ref{eulerlag}) makes
sense intrinsically on manifolds.  What Lagrange proved, in his {\it
M\'ecanique Analytique}, was that {\bf (\ref{eulerlag}) has the same form
in arbitrary ``curvilinear'' or ``generalized'' coordinates.}  

\begin{remark}{\rm 
A more modern formulation of Lagrange's invariance result would be that,
for a sufficiently smooth curve $\xi_*$ in a smooth manifold $Q$, if
$TQ$ denotes the tangent bundle of $Q$, and $L:TQ\times\rr\to\rr$ is a 
sufficiently smooth function, and we define
\begin{equation}\label{elform}
{\cal E\!L}_{\xi_*}(t)=
\frac{d}{dt}\frac{\partial L}{\partial u}(\xi_*(t),\dot\xi_*(t),t)
-\frac{\partial L}{\partial q}(\xi_*(t),\dot\xi_*(t),t)\,,
\end{equation}
then ${\cal E\!L}_{\xi_*}$---the ``Euler-Lagrange form along 
$\xi_*$''---is a well-defined field of covectors along $\xi_*$.   This
implies  
that the equation ${\cal E\!L}_{\xi_*}(t)=0$ makes sense intrinsically, but 
it also implies that other things that can be written using
${\cal{}E\!L}_{\xi_*}$ make sense  
intrinsically. For example, if the Lagrangian $L$ governs the motion of a
particle that is subject in addition to constraints 
$\omega_i(q).\dot q=0$, where the $\omega_i$, $i=1,\ldots,k$, are 
differential 1-forms on $Q$, such that 
$\omega_1(q),\ldots,\omega_k(q)$ are linearly independent at each $q$, then
the statement that 
\begin{equation}\label{const}
{\cal E\!L}_{\xi_*}(t)\in{\rm linear\,\,\,span}\Big(
\{\omega_1(\xi(t)),\ldots,\omega_k(\xi(t))\}\Big)\,\,\,\,\,{\rm for\,\,\,\,\,all}
\,\,\,\,\,\,t\,
\end{equation}
also makes sense intrinsically.  

Formula (\ref{const}) is the well-known
{\em D'Alembert principle}, governing constrained motion. If we think of 
the covector ${\cal E\!L}_{\xi_*}(t)$ as a ``virtual force,''  then 
(\ref{elform}) says that the ``force''---i.~e., $\frac{d}{dt}\frac{\partial
L}{\partial\dot q}$, the time derivative 
of the momentum $p=\frac{\partial L}{\partial\dot q}$---is equal to 
$\frac{\partial L}{\partial q}$ plus the virtual force. Then 
(\ref{eulerlag}) says that for unconstrained motion the virtual force
should be zero, and (\ref{const}) says that 
that for constrained motion the virtual force a point $q$ 
must be orthogonal 
to the set $\{v:\omega_i(q).v=0\,,\,i=1,\ldots,k\}$ of feasible 
velocity vectors at $q$.} \proofend
\end{remark}

Lagrange was aware of the importance of his invariance proof. He wrote:

\begin{center}\begin{minipage}{103mm}\baselineskip=9pt\noindent 
\noindent It is perhaps one of the principal advantages of our method that it
expresses the equations of every problem in the most simple form
relative to each set of variables and that it enables us to see
beforehand which variables one should use in order to facilitate the
integration as much as possible.
\end{minipage}\end{center}
\begin{remark}{\rm 
An obvious illustration of the power of this idea is provided by the 
calculation showing that the three laws of Kepler follow from Newton's
inverse-square law.   The calculation is made much simpler by 
using spherical coordinates, i.e., by writing 
the equations in terms of those variables that 
``facilitate the integration as much as possible.''}\proofend
\end{remark}

\begin{remark}{\rm 
The era of Euler and Lagrange also saw the birth of the {\bf\em least
action principle}, that made the link between curve minimization and
physics much stronger.  According to the principle, the behavior of
mechanical systems is such as to minimize an {\sl action functional}
$$
S=\int_a^bL(\xi(t),\dot\xi(t),t)\,dt\,.
$$
Moreover, at least some of the discoverers of this principle 
felt, like Aristotle, that the fact that Nature obeys 
a minimization principle had to have a profound meaning.
For example, the least action principle was thought to express the fact that
the Creator was making the wisest possible use of his
power, as Maupertuis wrote in 1759:

\begin{center}\begin{minipage}{103mm}\baselineskip=9pt\noindent 
{\rm Notre principe, le plus conforme aux id\'{e}es que nous devons
avoir des choses, laisse le monde dans le besoin continuel de la puissance
du Cr\'{e}ateur, et est une suite n\'{e}cessaire de l'emploi le plus sage
de cette puissance \ldots Ces lois si belles et si simples sont
peut-\^{e}tre les seules que le Cr\'{e}ateur et l'Ordonnateur des choses a
\'{e}tablies dans la mati\`{e}re pour y op\'{e}rer tous les
ph\'{e}nom\`{e}nes de ce Monde visible.}
\end{minipage}\end{center}

\noindent A radically different view was expressed by Hamilton in 1833:

\begin{center}\begin{minipage}{103mm}\baselineskip=9pt
{\rm \noindent Although the law of least action has thus attained a
rank among the highest theorems of physics, yet its pretensions to
a cosmological necessity, on the ground of economy of the universe,
are now generally rejected.  And the rejection appears just,
for this, among other reasons, that the quantity pretended to be economized
is in fact often lavishly expended $\ldots$. We cannot,
therefore, suppose the economy of this quantity to have been designed in
the divine idea of the universe.}
\end{minipage}\end{center}

\noindent The least action principle remains to this day one of the basic
principles 
of physics, classical as well as relativistic and quantum mechanical
(cf. Yourgrau and Mandelstam \cite{yourgrau}).}\proofend
\end{remark}

\medbreak In 1834 and 1835, {\bf\em W.R. Hamilton} (1805-1865) published
papers on dynamics showing how to rewrite the Euler-Lagrange
equations in what we would now call ``Hamiltonian form'' :
for a curve $t\mapsto{}\xi_*(t)$,  (\ref{eulerlag}) is exactly equivalent
to the system of equations
\begin{equation}\label{hamileq}
\left.
\begin{array}{rcl}\frac{d\xi_*}{dt}(t)&=&\phantom{-}\frac{\partial H}{\partial
p}\Big(\xi_*(t),\dot\xi_*(t),\psi(t),t\Big)\,,\\
\frac{d\psi}{dt}(t)&=&-\frac{\partial H}{\partial
q}\Big(\xi_*(t),\dot\xi_*(t),\psi(t),t\Big)\,,\\ 
0&=&\phantom{-}\frac{\partial
H}{\partial{}u}\Big(\xi_*(t),\dot\xi_*(t),\psi(t),t\Big)\,,\end{array}\right.
\end{equation}
where the {\sl Hamiltonian} $H$ and the {\sl momentum} $\psi$ are defined
by  
\begin{eqnarray}
H(q,u,p,t)&\equaldef& \langle p,u\rangle-L(q,u,t)\,,\label{defofh}\\
\psi(t)\,\,&\equaldef&\,\,\frac{\partial L}{\partial
u}\Big(\xi_*(t),\dot\xi_*(t),t\Big)\,.\label{defofp}
\end{eqnarray}
The system (\ref{hamileq})-(\ref{defofp})
is not exactly what is commonly known as ``Hamilton's equations,''
although in our view it is how Hamilton's equations should be
written.  The usual Hamilton equations are
\begin{equation}\label{hamileqsysham}
\frac{dq}{dt}=\frac{\partial{\cal{}H}}{\partial p}\,,
\,\,\,\,\frac{dp}{dt}=-\frac{\partial{\cal{}H}}{\partial q}\,,
\end{equation}
where $(q,p,t)\to{\cal H}(q,p,t)$ is a function of $p,q$ and $t$ alone,
defined by the formula 
\begin{equation}\label{hamileqsyshamnewhham}
{\cal H}(q,p,t)\equaldef\langle p,\dot q\rangle-L(q,\dot q,t)\,.
\end{equation}
This is certainly similar to
(\ref{hamileq})-(\ref{defofp}),  but with a
crucial 
difference:  in (\ref{hamileqsysham}) and (\ref{hamileqsyshamnewhham}) $\dot q$
is supposed to be 
treated {\sl not} as an independent variable, but as a function 
of  $q$, $p$, and $t$,   defined implicitly by the equation
\begin{equation}\label{pimpl}
p=\frac{\partial L}{\partial\dot q}(q,\dot q,t)\,,
\end{equation}
whereas in (\ref{hamileq})-(\ref{defofp})
$q$, $u$, $p$, and $t$ are treated as independent variables. 

The fact that Hamilton's equations were written in the form
(\ref{hamileqsysham},\ref{hamileqsyshamnewhham}) rather than
(\ref{hamileq},\ref{defofh},\ref{defofp}) has had
important consequences, discussed in detail in Sussmann and Willems
\cite{sussjan1,sussjan2}.  Indeed, it is argued in \cite{sussjan1,sussjan2}
that the maximum principle of optimal control theory could have been
discovered much earlier, perhaps by Weierstrass, if only the ``correct''
Hamiltonian system
(\ref{hamileq},\ref{defofh},\ref{defofp}) been used.

Hamilton's equations also have the {\sl coordinate invariance property}
proved by Lagrange for the Euler-Lagrange equations, provided that the 
$p$ variables are transformed according to the correct law: for a
coordinate change $q^{\rm new}=F(q)$, i.e. $q_i^{\rm new}=F_i(q_1,\ldots,q_n)$,
the new variables $p_i^{\rm new}$ must be related to the old ones 
via 
$$p_i=\sum_{j}\frac{\partial F_j}{\partial q_i}p_j^{\rm new}\,,$$
which is the dual of the transformation law 
$$\dot q_i^{\rm new}=\sum_{j}\frac{\partial F_i}{\partial q_j}\dot q_j\,,$$
and guarantees that 
$$\sum_{i}p_i\dot q_i=\sum_{i}p_i^{\rm new}\dot q_i^{\rm new}\,.$$
In other words, using modern terminology, the $p_i$ are the components of a
{\sl covector}.

In 1854, {\bf\em B. Riemann} (1826-1866) 
introduced a number of  ideas 
that today lie at the foundation of differential geometry:
\begin{itemize}
\item[1.] He turned Statement (LS) around, and made it into 
a {\it definition} of ``line segment,'' now called
``geodesic.''
\item[2.] He introduced more general ways of 
measuring the length of an arbitrary curve
$t\to(q^1(t),\ldots,q^n(t))$ by using the {\it arc-length} element
\begin{equation}\label{rmetric}
ds^2=\sum_{ij}g_{ij}(q)\,dq^idq^j\,,
\end{equation}
where $(g_{ij})_{i,j=1}^n$ is an $n\times n$ matrix of smooth functions 
of $q$, assumed to be positive definite for each $q$. 
\item[3.] He introduced the systematic use of ``covariant quantities,''
i.~e., quantities that transform under general nonlinear coordinate changes
according to a specific law. 
Two examples of that, namely, ``vectors'' such as $\dot q$, and
``covectors'' such as $p$, have already been discussed before.
A third example is the Riemannian metric itself, which is a tensor 
with two covariant indices, meaning that the $g_{ij}$ transform according 
to 
$$g_{ij}=\sum_{k,\ell}\frac{\partial F_k}{\partial q_i}
\frac{\partial F_{\ell}}{\partial q_j}g_{k,\ell}^{\rm new}\,.$$
\end{itemize}
Moreover, in his effort to understand the  properties of 
what we now call ``Riemannian metrics,'' given locally by 
(\ref{rmetric}), Riemann introduced the 
{\bf\em curvature tensor}, whose components are given, in coordinates, by 
\begin{equation}\label{rcurv}
R_{jk\ell}^i=\frac{\partial \Gamma_{j\ell}^i}{\partial q^k}-
\frac{\partial \Gamma_{jk}^i}{\partial q^\ell}
+\sum_{s}\Bigl(\Gamma_{sk}^i\Gamma_{j\ell}^s
-\Gamma_{s\ell}^i\Gamma_{jk}^s\Bigr)\,,
\end{equation}
where the $\Gamma_{jk}^i$ are the ``Christoffel symbols,'' given by 
\begin{equation}\label{rchris}
\Gamma_{jk}^i=\frac{1}{2}\sum_{\ell}g^{i\ell}\Biggl(
\frac{\partial g_{\ell k}}{\partial q^j}+
\frac{\partial g_{\ell j}}{\partial q^k}-
\frac{\partial g_{jk}}{\partial q^\ell}\Biggr)\,,
\end{equation}
and the $g^{ij}$ are the entries of the inverse of the matrix
$(g_{ij})$, so that 
$$\sum_jg^{ij}g_{jk}=\delta_k^i\,. $$
(Here $\delta_k^i=0$ if $k\neq i$, $\delta_i^i=1$.)

The crucial points about the curvature tensor $R$ are:
\begin{itemize}
\item[(1)] Its {\bf\em covariance property}, i.e.,  the fact that
the components $\tilde R_{\beta\gamma\delta}^\alpha$ 
of $R$ relative to a coordinate chart 
$\tilde q=F(q)$ are related by a specific formula to the components
$R_{jk\ell}^i$ relative to $q$. The transformation law is
$$
\tilde R_{\beta\gamma\delta}^\alpha=
\sum_{ijk\ell}R_{jk\ell}^i\cdot\frac{\partial F^\alpha}{\partial q^i}
\cdot\frac{\partial G^j}{\partial\tilde q^\beta}\cdot\frac{\partial G^k}
{\partial\tilde q^\gamma}\cdot\frac{\partial G^\ell}{\partial\tilde
q^\delta}\,, 
$$
where $\tilde q\to G(\tilde q)$ is the inverse transformation
of $q\to F(q)$. This implies, in particular, that {\sl 
if $R\equiv 0$ in {\bf\em some} coordinate system, then 
$R\equiv 0$ in {\bf\em every} coordinate system.}
So $R\equiv 0$ is an {\it invariant property}.
\item[(2)] The fact that $R$ locally determines the metric up to isometry;
for example, $R\equiv 0$ near a point $\bar q$ iff the metric is {\bf\em flat} 
(i.e., such that, for some coordinate chart, the $g_{ij}$ 
are constant functions) near  $\bar q$.\footnote{\baselineskip=7pt The 
general statement of the result that ``$R$ determines the metric
up to isometry'' is more delicate, because, if $(M,g)$ and 
$(\tilde M,\tilde g)$ are two different Riemannian manifolds, 
then the corresponding curvature tensors live in different spaces, 
so it is not clear how to define what it means for them to be the same. 
One way to  do  this is given in Do Carmo \cite{docarmo}.}
\end{itemize}
It then follows that flatness can be detected even if the metric is
presented to us in completely arbitrary 
``curvilinear'' coordinates, since it suffices to check
whether $R\equiv 0$, and this property holds in {\bf\em one}
coordinate system iff it holds in {\bf\em every} 
such system.

Through Riemann's ideas, optimal control exercised a decisive influence on
the birth of differential geometry by showing that certain optimal control
problems could be used as {\bf\em sources of new geometries}. Moreover, the
concept of invariance under general nonlinear changes of coordinates, which
had first surfaced with Lagrange's proof of the invariance of the
Euler-Lagrange system, turned out to play a crucial role in the theory. In
fact, the systematic use of {\bf\em general covariant quantities} and the
search for {\bf\em intrinsic formulations} became the hallmarks of modern
differential geometry. As we shall see below, the ideas of covariant
objects and intrinsic formulations have in turn been influential in recent
developments in optimal control.


\section{Invariance, Covariance and Lie Brackets}
\label{INVA}
%\bigskip

The evolution of nonlinear control theory exhibits some interesting
similarities to that of differential geometry.  In both cases, a special
class of situations (flat Euclidean spaces, linear control systems) was
studied first, and a rich theory was developed.  In both cases, there was a
mathematical language (Euclidean geometry, linear algebra) adequate for the
special theory, but not appropriate for the general case. In both cases,
the general theory requires the use of the language of covariant
quantities, some of which turn out to play a key role. For differential
geometry, one needs the various spaces of tensors and other covariant
quantities (densities, spinors, jets, connections, etc.), of which the most
important ones for classical (Riemannian) differential geometry are the
metric and the curvature tensor.  For nonlinear control, one needs many
covariant objects, such as vector fields and differential forms, and
covariant operations such as the Lie bracket, exterior multiplication, and
exterior differentiation.

The analogue of the Riemann curvature tensor in this setting is the {\sl
structure of the accessibility Lie algebra of a system.} More precisely,
for an autonomous control system $\Sigma:\dot q=f^\Sigma(q,u)$, $u\in U$,
on a smooth manifold $Q^\Sigma$, define $\Lambda^\Sigma$ to be the free Lie
algebra generated by a family $\{F_u:u\in U\}$ of indeterminates indexed by
the set $U$.  Let $L^\Sigma$ be the Lie algebra of vector fields generated
by the $f_u^\Sigma$, $u\in U$, where we let
$f_u^\Sigma(q)=f^\Sigma(q,u)$. Then there is a unique Lie-algebra
homomorphism $P^\Sigma:\Lambda^\Sigma\to L^\Sigma$ that sends $F_u$ to
$f_u^\Sigma$ for each $u$.  Given a point $q\in Q^\Sigma$, we can define a
linear map $EV_q^\Sigma:\Lambda^\Sigma\to T_qQ^\Sigma$---where
$T_qQ^\Sigma$ is the tangent space of $Q^\Sigma$ at $q$---by letting
$EV_q^\Sigma(V)=P^\Sigma(V)(q)$.

\begin{definition}
{\rm The kernel 
$$REL_q^\Sigma\,\,\equaldef\,\,\ker\Big(EV_q^\Sigma\Big)
$$
is the {\em set of Lie-bracket relations at $q$} of the vector fields of
$\Sigma$.}\proofend  
\end{definition}
\begin{example}{\rm 
Suppose $u$, $v$, and $w$ are three members of $U$. Then 
the formal expression $[F_u,[F_v,F_u]]+7[F_u,F_w]+F_v$ 
belongs to $REL_q^\Sigma$ if and only if 
$[f_u,[f_v,f_u]](q)+7[f_u,f_w](q)+f_v(q)=0.$}\proofend   
\end{example}
The following can be proved by a simple application of E. Cartan's graph
method. 
\begin{theorem}\label{invariath}
For a fixed control set $U$, if we consider  pairs $(\Sigma,q)$
such that $\Sigma$ is a real analytic system with control set $U$,
$q$ belongs to the state space $Q^\Sigma$ of $\Sigma$, and $\Sigma$ has
the accessibility property at $q$, then the space 
$REL_q^\Sigma$ is a complete set of local invariants  for the germ of
$\Sigma$ at $q$ under diffeomorphisms. \proofend
\end{theorem}
(Cf. for example Sussmann \cite{susscrac}, Theorem 7). The precise meaning
of the conclusion of Theorem \ref{invariath} is the following:

\medbreak\noindent{\bf (CSI)} 
{\sl Given two pairs $(\Sigma_i,q_i)$, $i=1,2$, the following two
conditions are equivalent:
\begin{itemize}
\item[(i)]
There exists a 
diffeomorphism $\Phi$ from a neighborhood $U_1$ of $q_1$ in $Q^{\Sigma_1}$ 
onto  a neighborhood $U_2$ of $q_2$ in $Q^{\Sigma_2}$ that maps 
each vector field $f_u^{\Sigma_1}$ to the corresponding 
vector field $f_u^{\Sigma_2}$ and is such that $\Phi(q_1)=q_2$,
\item[(ii)] $REL_{q_1}^{\Sigma_1}= REL_{q_2}^{\Sigma _2}$.
\end{itemize}
}

\medbreak\noindent (Recall that the ``accessibility property at $q$,'' for
a system $\Sigma$, is the property that $\{X(q):X\in
L^\Sigma\}=T_qQ^\Sigma$.)

Theorem \ref{invariath} tells us that---for real-analytic systems---all
properties of interest that are invariant under nonlinear coordinate
changes are determined by the Lie bracket relations.  For systems where $U$
is a linear space and the control enters the equations in an affine-linear
way, it can be shown the class of {\sl linear systems} is characterized by
the vanishing of certain brackets. (Precisely: for a real-analytic system
$\dot q=f(q)+\sum_{i=1}^mu_ig_i(q)$, if $f(\bar q)=0$ and the system has
the accessibility property at $\bar q$, then the system is linear near $\bar
q$---up to a nonlinear change of coordinates that sends $\bar q$ to 
$0$---if and only if all the iterated brackets of $f$ and the $g_i$ that
involve at least two $g_i$'s vanish at $\bar q$.)  

The fact that the Lie bracket relations give a complete set of invariants
under diffeomorphism makes it reasonable to expect that the structural,
coordinate invariant properties of a nonlinear system should be expressible
in terms of Lie brackets, in perfect analogy with the way that the
structural, coordinate invariant properties of a Riemannian manifold are
expressible in terms of the curvature tensor.  For example, the structure
of the time-optimal trajectories (e.~g., whether the time-optimal controls
are bang-bang) should be determined by Lie bracket conditions. Since the
main tool for analyzing this structure is the necessary condition for
optimality given by the Maximum Principle, it is clear that it would be
useful to have a statement of the Maximum Principle where the Lie brackets
appear explicitly.  We will describe such a formulation in Section \ref{INTRI}
(outlining the much more detailed exposition of Sussmann \cite{sussfree}),
and then show by means of a simple example how this formulation can be used
to derive qualitative properties of optimal trajectories.

But first we must proceed to our discussion of the Maximum Principle.

\section{The Maximum Principle}
\label{MAXP}
%\bigskip

The {\bf\em Pontryagin Maximum Principle}  \cite{berkovitz,pontrybook}
extends the necessary 
conditions for optimality of the calculus of variations to the much more
general setting of fixed-endpoint optimal control problems
with a state space $Q$ which is an open subset of $\rr^n$, and 
a set $U$ of control values:

\medbreak
(FEOCP)\ \ \ \ \fbox{\begin{minipage}{20mm}
\noindent{\it Minimize}

\noindent{\it subject to} 

\noindent\phantom{{\it subject to} }

\noindent{\it and} \end{minipage} 
$\quad\quad$
\begin{minipage}{54mm}
\noindent $\int_a^bL(\xi(t),\dot\xi(t),t)dt$

\noindent $\dot\xi(t)=f(\xi(t),\eta(t),t)$ \ for \ a.e. \ $t$, 

\noindent $(\xi(t),\eta(t))\in Q\times U$ \ for \ all \ $t$, 

\noindent $\xi(a)=\bar q\,,\ \ \ \ \ \ \xi(b)=\hat q\,,$
\end{minipage}
}

\medbreak\noindent or to problems 

\medbreak
(GEOCP)\ \ \ \ \fbox{\begin{minipage}{20mm}
\noindent{\it Minimize}

\noindent{\it subject to} 

\noindent\phantom{{\it subject to} }

\noindent{\it and} \end{minipage} 
$\quad\quad$
\begin{minipage}{54mm}
\noindent $\int_a^bL(\xi(t),\dot\xi(t),t)dt$

\noindent $\dot\xi(t)=f(\xi(t),\eta(t),t)$ \ for \ a.e. \ $t$, 

\noindent $(\xi(t),\eta(t))\in Q\times U$ \ for \ all \ $t$, 

\noindent $(\xi(b),\xi(a))\in S\,.$
\end{minipage}
}

% GNU Original was
% \noindent $(\xi(a),\xi(b))\in S\,.$
% Not corrected for book. 

\medbreak\noindent with a more general endpoint condition.  

The result is, modulo technical conditions that will not be discussed at
this point, that, for a trajectory-control pair
\mbox{$[a,b]\ni t\to (\xi_*(t),\eta_*(t))$} to be a minimizer,
it is necessary that there
exist an absolutely continuous curve $[a,b]\ni t\to \psi(t)\in \rr_n$ and
a constant $\psi_0\ge 0$ such that the {\it adjoint equation}
$$
-\dot \psi(t)=\frac{\partial H}{\partial
q}(\xi_*(t),\eta_*(t),\psi(t),\psi_0,t) 
$$
and the {\it maximization condition}
$$
H(\xi_*(t),\eta_*(t),\psi(t),\psi_0,t)=\max\{H(\xi_*(t),u,
\psi(t),\psi_0,t):u\in U\}
$$
hold for a.e. $t\in [a,b]$, as well as the {\it nontriviality condition}
$$
(\psi(t),\psi_0)\neq (0,0)\,\,\,\,{\rm for\,\,\,\,some\,\,\,\,
(and\,\,\,\,hence\,\,\,\,every)}\,\,\,\,t\,,
$$ 
and the {\it transversality condition}
$$
(-\psi(b),\psi(a))\in C^\dagger\,.
% GNU Original was
% (\psi(a),-\psi(b))\in C^\dagger\,.
% Not corrected for book. 
$$
Here 
\begin{itemize}
\item[1.] $H$ is the {\it Hamiltonian}, defined by 
$$
H(q,u,p,p_0,t)=p.f(q,u,t)-p_0L(q,u,t)\,,
$$
\item[2.] $C$ is a ``tangent cone'' at the point $(\xi_*(b),\xi_*(a))$ to
% GNU Original was
% \item[2.] $C$ is a ``tangent cone'' at the point $(\xi_*(a),\xi_*(b))$ to
% Not corrected for book. 
the set $S$,\footnote{\baselineskip=7pt The question of how to give a
precise definition of ``tangent cone'' will be discussed later,
cf. Section \ref{WEAK}.} 
\item[3.] if $K$ is any subset of a real linear space $X$, 
then $K^\dagger$ is the {\sl polar cone} of $K$, i.e. the set 
$$K^\dagger\,\,\equaldef\,\,\{v\in X^*:\langle
v,w\rangle\le 0\,\,\,{\rm for \,\,\,all}\,\,\,w\in K\}\,,$$
where $X^*$ is the dual space of $X$, 
\end{itemize}
and
\begin{itemize}
\item[4.] 
the symbol $\rr_n$ denotes the set  
of $n$-dimensional real {\it row vectors}, whereas the members
of $\rr^n$ are {\it column vectors}.
\end{itemize}

\section{The Maximum Principle as a Necessary Condition for 
Set Separation}
\label{SETSEP}
%\bigskip

It turns out that the maximum principle, as formulated, is really a 
% GNU Original was
% It turns out the the maximum principle, as formulated, is really a 
% Not corrected for book. 
``geometric'' result, giving a necessary condition for a separation 
property between a reachable set of a control system and some other 
set.   

To see this, we first observe that the Maximum Principle for 
problems of the form (GEOCP) can easily be reduced to the 
special case of a problem of the same form but with a more
restrictive endpoint condition of the form 
\begin{equation}\label{specialep}
\xi(a)=\bar q\,,\quad\quad \xi(b)\in S\,,
\end{equation}
where $S$ is  a given subset of  $Q$.  Indeed, 
suppose we are given a problem
(GEOCP), and an optimal trajectory-control pair $(\xi_*,\eta_*)$.  Consider
a new problem 

\bigbreak\noindent
(GEOCP${\,}_{\rm{new}}$)\ \fbox{\begin{minipage}{88mm}
\noindent\hbox to 17mm{{\it Minimize}\hfill}
$\int_{a-1}^{b} L_{\rm{new}}(\xi_{\rm{new}}(t),\dot\xi_{\rm{new}}(t),t)dt$\\
\noindent\hbox to 17mm{{\it subject to}\hfill}
$\dot\xi_{\rm{new}}(t)=f_{\rm{new}}(\xi_{\rm{new}}(t),\eta_{\rm{new}}(t),t)$
for \ a.e. \ $t$,\\
\noindent\hbox to 17mm{$\,\,$\hfill}
$(\xi_{\rm{new}}(t),\eta_{\rm{new}}(t))\in Q_{\rm{new}}\times U_{\rm{new}}$ \ for \ all \ $t$,\\
\noindent\hbox to 17mm{and \hfill}
$\xi_{\rm{new}}(a-1)=\bar q_{\rm{new}}$, $\,\,\,\xi_{\rm{new}}(b)\in S_{\rm{new}}\,,$
\end{minipage} 
}

\bigbreak\noindent where $Q_{\rm{new}}=Q\times Q$, 
$U_{\rm{new}}=U\times\rr^n$,  $S_{\rm{new}}=S$,  $\bar q_{\rm{new}}=(\xi_*(a),\xi_*(a))$, 
and, for $q^1\in Q$, $q^2\in Q$, $u\in U$, $v\in\rr^n$, 
$a-1\le t\le b$, we let 
\begin{eqnarray*}
L_{\rm{new}}(q^1,q^2,u,v,t)&=&\chi_{_{[a,b]}}(t)L(q^1,u,t)\,,\\
f_{\rm{new}}(q^1,q^2,u,v,t)&=&\chi_{_{[a,b]}}(t).\Big(f(q^1,u,t),0\Big)
+(1\!-\!\chi_{_{[a,b]}}(t)).(v,v)\,,
\end{eqnarray*}
where, if $E$ is a set, then $\chi_{_E}$ denotes the indicator function of
$E$, i.~e., $\chi_{_E}(x)=1$ if $x\in E$ and $\chi_{_E}(x)=0$ if $x\notin E$.

It is then readily shown that the new reference trajectory-control pair,
consisting of the curve
\mbox{$[a,b]\!\ni\!t\!\to\!\xi_{*,\rm{new}}(t)\equaldef(\xi_*(t),\xi_*(a))$},
and the corresponding control \mbox{$[a,b]\ni t\to
\eta_{*,\rm{new}}(t)\equaldef (\eta_*(t),0)$} (extended to the interval
$[a-1,b]$ by letting \mbox{$\xi_{*,\rm{new}}(t)=(\xi_*(a),\xi_*(a))$} and
\mbox{$\eta_{*,\rm{new}}(t)=(0,0)$} for $a-1\le t<a$) is optimal for
(GEOCP${\,}_{\rm{new}}$). Moreover, it is easy to see that
(GEOCP${\,}_{\rm{new}}$) is of our more special form.  Applying to our
situation the conclusion of the Maximum Principle for problems of this
special form, we get precisely the conclusion of the general result.

\vbox{
We are now ready to ``geometrize'' the problem by reducing it 
to a question about separation of sets.  

\begin{definition}{\rm 
We say that two subsets ${\cal R}$ and $S$ of a topological space 
$Q$ are {\em separated} at a point $\hat q\in Q$ if 
${\cal R}\cap S=\{\hat q\}$, and that ${\cal R}$ and $S$ 
are {\em locally separated} at $\hat q$ if there exists a neighborhood
$V$ of $\hat q$ such that ${\cal R}\cap S\cap V=\{\hat q\}$.}\proofend
\end{definition}
}

For a control system $\Sigma$ with dynamical law
\begin{equation}\label{dynalaw}
\dot q=f(q,u,t)\,,\,\,\,q\in Q\,,\,\,\,u\in U\,,\,\,\,t\in I\,,
\end{equation}
where $I$ is a subinterval of $\rr$ (and for the
time being we take $Q$ to be an open subset of $\rr^n$), let us
define ${\cal R}_{[a,b]}^\Sigma(\bar q)$---the {\it $\Sigma$-reachable set
from $\bar q$ over $[a,b]$}---to be the set of all points $\xi(b)$, for
all possible trajectory-control pairs $\gamma=(\xi,\eta)$ of $\Sigma$ such
that $\xi$ is defined on $[a,b]$ and $\xi(a)=\bar q$.  Also, define
$H^\Sigma$---the {\it Hamiltonian} of $\Sigma$---to be the function
\begin{equation}\label{hamildef}
Q\times U\times\rr^n\times
I\ni (q,u,p,t)\to H^\Sigma(q,u,p,t)=\langle p,f(q,u,t)\rangle\,.
\end{equation}

\noindent Then it is not hard to show that the Maximum Principle follows
from a necessary condition for a reachable set
${\cal{R}}_{[a,b]}^\Sigma(\bar q)$ to be separated from some other set $S$.

Before we state this condition, we give a precise definition of 
the class of control systems to which it will apply.

\begin{definition}{\rm 
A {\bf\em control system in $\rr^n$} is  a
5-tuple $\Sigma=(Q,U,I,{\cal U},f)$ such that
{\em 
\begin{itemize}\begin{itemize}
\item[(CS1)] $Q$---the ``state space'' of $\Sigma$---is an open subset of
$\rr^n$, 
\item[(CS2)] $U$---the ``space of control values'' of $\Sigma$---is a
set, 
\item[(CS3)] $I\subseteq\rr$---the ``time interval'' of $\Sigma$---is a
subinterval of $\rr$, 
\item[(CS4)] ${\cal U}$---the ``class of admissible controls''
of $\Sigma$---is a set of maps ${\rm Dom}(\eta)\ni t\to\eta(t)\in U$ such
that each $\eta\in{\cal U}$ is defined on a set
${\rm{}Dom}(\eta)=[a_\eta,b_\eta]$ which is a compact subinterval of $I$, 
\end{itemize}\end{itemize}}
\noindent and 
{\em
\begin{itemize}\begin{itemize}
\item[(CS5)] $f$---the ``dynamics'' of $\Sigma$---is a map from $Q\times
U\times I$ to $\rr^n$. \proofend
\end{itemize}\end{itemize}}}
\end{definition}

\begin{remark}{\rm 
We emphasize that the control set $U$ is {\em just an abstract set},
with no extra structure.  Working on this level of generality has many 
advatages. (For a simple illustration, we refer the reader to 
the derivation of the equation of the 
Riemannian geodesics from the maximum principle, outlined in Example 
\ref{exageo} below, where we take $U$ to be the set of all smooth vector fields
on a manifold.)}\proofend
\end{remark}

Then the following statement is the {\bf\em set separation
version of the Maximum Principle}:

\bigbreak\noindent(SSMP)$\,\,$ 
\fbox{\begin{minipage}{96mm}
\baselineskip=7pt
\noindent{\sl Let $\Sigma=(Q,U,I,{\cal U},f)$ be a control system in
$\rr^n$.  Assume that $S\subseteq Q$, $a\le b$, $a,b\in I$, $\xi_*:[a,b]\to
Q$ is a trajectory of $\Sigma$ corresponding to a control $\eta_*:[a,b]\to U$,
and $\xi_*(b)\in{}S$. Let $C$ be a tangent cone to $S$ at $\xi_*(b)$ which is
not a linear subspace of $\rr^n$. Then a necessary condition for
${\cal{R}}_{[a,b]}^\Sigma(\xi_*(a))$ to be locally separated from $S$ at
$\xi_*(b)$ is that there exist a nonzero row-vector-valued map $[a,b]\ni
t\to{}\psi(t)\in\rr_n$ that satisfies:

\medbreak\noindent{\bf 1.}\ \ the {\bf Hamiltonian maximization condition} 

\quad(HMC)\quad\begin{minipage}{76mm}
\begin{eqnarray*}
H^\Sigma(\xi_*(t),\eta_*(t),\psi(t),t)=\quad\quad\quad\quad\quad\quad\quad&\quad&\\
\max\Big\{H^\Sigma(\xi_*(t),u,\psi(t),t):u\in U\Big\}\,,&\,\,&
\end{eqnarray*}
\end{minipage}

\medbreak\noindent {\bf 2.}\ \ the {\bf adjoint equation}

\quad(AE)\quad\quad\begin{minipage}{76mm}
$$
\dot \psi(t)=-\frac{\partial H^\Sigma}{\partial q\,\,\,\,}
\Big(\xi_*(t),\eta_*(t),\psi(t),t\Big)\,,
$$
\end{minipage}

\medbreak\noindent and 

\medbreak\noindent {\bf 3.}\ \ the {\bf transversality condition} 

\quad(TC)\quad\quad\begin{minipage}{76mm}
$$
-\psi(b)\in C^\dagger\,.
$$
\end{minipage}
}
\end{minipage}
}
\bigbreak\noindent Naturally, (SSMP) as stated is not yet a rigorous
mathematical theorem, because some crucial 
technical hypotheses and definitions are still
missing.  What is needed to render (SSMP) precise is the following:

\medbreak
\begin{itemize}\begin{itemize}
\item[(TH1)] a specification of technical conditions on the map $f$ 
and the class ${\cal U}$ of open-loop controls;
\vspace{-1mm}
\item[(TH2)] a precise definition of ``trajectory-control pair of $\Sigma$;''
\vspace{-1mm}
\item[(TH3)] precise technical conditions on the reference 
trajectory-control pair $(\xi_*,\eta_*)$; 
\vspace{-1mm}
\item[(TH4)] a precise definition of ``tangent cone;''
\vspace{-1mm}
\item[(TH5)] a precise specification of what it means, for an $\rr_n$-valued
map $p$ on $[a,b]$, 
\begin{itemize}
\vspace{-1mm}
\item[(TH5.i)] to satisfy the maximization condition (HMC), 
\end{itemize}
\vspace{-1.5mm}
\noindent\item[] $\!\!\!\!\!\!\!\!\!{\rm and}$
\begin{itemize}
\vspace{-1mm}
\item[(TH5.ii)] to be a ``solution'' of the adjoint
equation (AE). 
\end{itemize}\end{itemize}\end{itemize}

\begin{remark}{\rm 
We include (TH3) as a separate item, because we wish to allow for the
possibility that in the final rigorous version of (SSMP) the 
reference trajectory-control pair  might be required to satisfy special
conditions that are not necessarily true of general 
trajectory-control pairs.\footnote{\baselineskip=7pt
This is a common situation in
optimization theory.  For example, we all learn in freshman calculus 
that the simplest necessary condition for a function $f$ of one variable 
to have a local minimum at an interior point $p$ of its domain
is that $f'(p)=0$, {\it provided that $p$ is a point where 
$f$ is differentiable. But $f$ need not be differentiable anywhere else.}}
Notice that, once (TH1,2,3,4,5) 
are taken care of, nothing else is needed to make (SSMP) precise,
since the concepts of ``polar cone,'' ``local separation,''  
and ``reachable set'' are unambiguously defined.}\proofend
\end{remark}
                
We claim that {\it (SSMP), once it is made
precise by filling all the technical gaps, implies the usual Maximum
Principle for optimal control.}  In fact, each {\it version} $V$ 
of (SSMP)---i.~e., each true theorem obtained from (SSMP) under appropriate
technical assumptions---implies a version $V'$ of the usual Maximum
Principle, with technical conditions corresponding in a natural way to
those of $V$.   

To see this, consider the standard optimal control 
problem (GEOCP), with the special endpoint condition (\ref{specialep}).
Suppose $(\xi_*,\eta_*)$ is a solution. Form the ``augmented system''
$\Sigma^{\#}=(Q^{\#},U,I,f^{\#})$, where 

\begin{itemize}\begin{itemize}
\item[(A.1)] $Q^{\#}=\rr\times Q$, 
% GNU Original was
% \item[(A.1)] $Q^{\#}=Q\times\rr$, 
% Not corrected for book. 
\item[(A.2)] $f^{\#}:Q^{\#}\times U\times I\to\rr^{n+1}$ is the map 
% GNU Original was
% \item[(A.2)] $f^{\#}:Q^{\#}\to\rr^{n+1}$ is the map 
% Not corrected for book. 
$$(r,q,t,u)\to (-L(q,u,t),f(q,u,t))\,,$$
% GNU Original was
% $(q,u,t)\to (-L(q,u,t),f(q,u,t))$,
% Not corrected for book. 
\item[(A.3)] $S^{\#}\equaldef\Big\{(c,q):q\in S\,,\,c\ge\hat c+\Vert q-\hat
q\Vert^2\Big\}$, 
\item[(A.4)] $c_*(t)=-\int_a^tL(\xi_*(s),\eta_*(s),s)\,ds$ for $a\le t\le b$,
\item[(A.5)] $\hat c=c_*(b)$ and $\hat q=\xi_*(b)$. 
\end{itemize}\end{itemize}

If we define $\xi_*^{\#}(t)=(c_*(t),\xi_*(t))$, then 
$(\xi_*^{\#},\eta_*)$ is a trajectory-control pair for $\Sigma^{\#}$, 
such that $\xi_*^{\#}(a)=\bar q^{\#}\equaldef(0,\bar q)$ and $\xi_*^{\#}(b)\in
S^{\#}$.  
The fact that $(\xi_*,\eta_*)$ is a solution of (GEOCP) implies that
${\cal R}_{[a,b]}^{\Sigma^{\#}}(\bar q^{\#})\cap S^{\#}=\{\bar q^{\#}\}$. 
If we apply (SSMP), and make the reasonable assumption that 
\begin{itemize}
\item[(P1)] {\it if $S\subseteq\rr^n$, $\hat q\in S$, $\hat c\in\rr$,
$C$ is a tangent cone to $S$ at $\hat q$, and $S^{\#}$ is defined by 
(A.3),  then 
$C^{\#}\equaldef [0,+\infty[\times C$ is a tangent cone to $S^{\#}$ at
$(\hat c,\hat q)$,}
\end{itemize} 
then we get the usual Maximum Principle.  {\sl Notice that 
for this application it is not necessary to require that the cone $C$ 
not be a linear subspace, because $C^{\#}$ will never 
be a linear subspace,  even if $C$ is.}

\begin{remark}{\rm 
The hypothesis that $C$ is not a linear subspace of $\rr^n$ is
necessary, for otherwise (SSMP) can obviously fail for trivial reasons.  To
see this, take $Q=\rr^n$, let $S$ be a linear subspace of $\rr^n$, let $T$
be a complementary subspace of $S$, and consider the control system $\dot
q=u$, $u\in T$, $q\in\rr^n$. Then the reachable set from $0$ over any
interval $[a,b]$ with $a<b$ is $T$. So this set is separated from $S$ at
$0$. On the other hand, the adjoint equation just says that $\dot\psi=0$,
and the Hamiltonian maximization condition implies that $\psi(t)$ must
annihilate $T$.  An obvious choice for $C$ is $S$ itself, and then the
requirement that $-\psi(b)\in C^{\dagger}$ says that $\psi(b)$ must
annihilate $S$. So $\psi(t)\equiv 0$, contradicting the nontriviality of
$\psi$.}\proofend
\end{remark}

\noindent In addition to implying the usual necessary conditions for
optimality, the separation theorem (SSMP) has the additional advantage that,
properly interpreted, it also implies a {\it sufficient condition for
local controllability along a reference trajectory},
i.e., equivalently,  a {\it necessary condition for a system not to be
locally controllable along a reference trajectory}. 

Precisely, recall that $\Sigma$ is {\it locally controllable} along a
trajectory-control pair $(\xi_*,\eta_*)$ defined on an interval $[a,b]$ if
$\xi_*(b)$ is an interior point of ${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$.
So, if $\Sigma$ is not locally controllable along $(\xi_*,\eta_*)$, then
there must exist a sequence $\{q_j\}$ of points of $Q$ that converge to
$\xi_*(b)$ and do not belong to ${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$.  By
taking a subsequence, if necessary, we can assume that the limiting
direction
\begin{equation}\label{limdir}
v=\lim_{j\to\infty}\frac{q_j-\hat q}{\|q_j-\hat q\|}
\end{equation}
exists. Let 
\begin{equation}\label{limdirs}
S=\{\hat q\}\cup\{q_j:j=1,2,\ldots\}\,.
\end{equation}
Then it is clear that the sets $S$ and ${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$
are separated at $\xi_*(b)$. Suppose we could take the tangent cone $C$ to
$S$ at $\xi_*(b)$ to be the half-line $\{rv:r\ge 0\}$. Then we could apply
(SSMP) to get the following {\it local controllability version of the
Maximum Principle}:

\bigbreak\noindent(LCMP) $\,\,$ 
\fbox{\begin{minipage}{92mm}
\noindent{\sl Let $\Sigma=(Q,U,I,{\cal U},f)$ be a control system in
$\rr^n$. Assume that $a\le b$, $a,b\in I$, and $\xi_*:[a,b]\to Q$ is a
trajectory of $\Sigma$ corresponding to a control $\eta_*:[a,b]\to U$. Then a
necessary condition for ${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$ not to be a
neighborhood of $\xi_*(b)$ is that there exist a nonzero map $[a,b]\ni t\to
\psi(t)\in\rr_n$ such that (HMC) and (AE) hold.  }
\end{minipage}
}

\bigbreak\noindent From now on we will take (SSMP) to be the ``true''
statement of the Maximum Principle, and will seek to understand its
geometric meaning further. {\bf\em We will, however, insist on wanting
(LCMP) to be a special case of (SSMP)}.  This means that we need to use in
(SSMP) a concept of ``tangent cone'' to a set $S$ at a point $\hat q\in S$
having---in addition to (P1)---the property that:
\begin{itemize}
\item[(P2)] {\it if $S$ is given by (\ref{limdirs}), where 
$\{q_j\}$ is a sequence such that $q_j\to \hat q$,
$q_j\neq \hat q$, and the limit (\ref{limdir}) exists, then the half-line
$\{rv:r\ge 0\}$ is a tangent cone to $S$ at $\hat q$.}
\end{itemize}

\noindent
Property (P2) excludes many of the concepts of ``tangent cone to a set at a
point'' that appear in the literature, such as the Clarke tangent cone, or
Boltyanskii's ``approximating cones.''  The most natural concept of ``tangent
cone'' for which (P2) holds is the Bouligand (or ``contingent'') tangent
cone. It is well known, however, that this concept is too weak to be of use
for the Maximum Principle.  (For example, let us consider the control
system $\dot x=u$, $\dot y=0$ in $\rr^2$, the initial point $(0,0)$, and
the terminal set 
$S=\{(x,y):x\ge 0\,,\,|y|\ge x^2\}$. Take $a=0$, $b=1$, and consider the
trajectory $t\to (0,0)$, corresponding to the control $\eta(t)\equiv 0$.  Then
the Bouligand tangent cone to $S$ at $(0,0)$ is the half-space
$C=[0,+\infty[\times\rr$. So the hypotheses of (SSMP) hold in this case,
but the conclusion clearly does not.)  

\section{Weakly Approximating Cones and Transversality}
\label{WEAK}


It turns out that there is at least one concept of ``tangent cone'' that 
has all the desired properties.  Suppose $S\subseteq \rr^n$, $\hat q\in S$, and
$C\subseteq\rr^n$. We say that $C$ is a {\it weakly approximating cone}
to $S$ at $\hat q$ if


{\sl 
\begin{itemize}\begin{itemize}
\item[(WAC.1)] $C$ is a closed convex cone;

\item[(WAC.2)] there exist a closed subset $C^*$ of $C$, 
and a continuous map \mbox{$F:C^*\to S$}, such that 
\begin{itemize}
\item[(WAC.2.a)]
$ \lim_{\varepsilon \downarrow  0}\,\,
\widetilde{\sup}\Big\{\|\xi(1)\| : \xi  \in C^0([0,1], C\backslash
C^*),||\xi(0)|| \le \varepsilon\Big\} = 0\,, 
$
\item[(WAC.2.b)] $\quad\quad F(v)=\hat q+v+o(\|v\|)\quad\quad{\rm
as\quad\quad}v\to 0\,,\,v\in C^*\,.$
\end{itemize}\end{itemize}\end{itemize}
}
\noindent In this definition, $C^0([0,1], C\backslash C^*)$ is the set of
continuous maps from $[0,1]$ to $C\backslash C^*$. 
The expression ``$\widetilde{\sup}$'' stands for 
``supremum in the set $[0,+\infty]$'' so that 
$\widetilde{\sup}(\emptyset)=0$.\footnote{\baselineskip=7pt If we had used
``$\sup$'' instead of ``$\widetilde{\sup}$,''  then 
the supremum of the empty set would have been equal to $-\infty$.}
It then follows that {\sl (WAC.2.a) holds, in particular, if $C^*$ is a full
relative neighborhood of $0$ in $C$.}   

Notice that (WAC.2) implies in particular that $0\in C^*$ because, if 
$0\notin C^*$, then there would exist a convex neighborhood $U$ of $0$ such
that $U\cap C^*=\emptyset$, since $C^*$ is closed. But then 
there would be a curve $\xi\in C^0([0,1], C\backslash C^*)$ 
such that $\xi(0)=0$ and $\xi(1)\neq 0$, contradicting (WAC.2.a).)

Condition (WAC.2.a) says, roughly, that ``within $C$, it is not
possible to join points that are close to $0$ to points 
that are far from $0$ without hitting the set $C^*$.''

If the set $C^*$ can be taken to be a full relative neighborhood 
of $0$ in $C$, then we call $C$ an {\sl approximating cone}
to $S$ at $\hat q$. In other words, $C$ is an {\it approximating cone}
to $S$ at $\hat q$ if

{\sl 
\begin{itemize}\begin{itemize}
\item[(AC.1)] $C$ is a closed convex cone;

\item[(AC.2)] there exist a neighborhood $U$ of $0$
and a continuous map $F$ from $C\cap U$ to $S$, such that 
$F(v)=\hat q+v+o(\|v\|)$ as   $v\to 0$ via values in $C$.
\end{itemize}\end{itemize}
}

It turns out that (SSMP) is true if the words ``tangent 
cone'' are interpreted to mean ``weakly approximating cone'' as in
our definition, provided that appropriate technical assumptions are made 
on the system $\Sigma$. To understand how this works, let us first quote
the following {\bf\em separation theorem} proved recently, in a more
general setting, in Sussmann \cite{sussmulti}:

\begin{theorem}\label{theosepclas} 
Let $C_1$ be a closed convex cone in $\rr^m$, let  $U$ be a neighborhood
of $0$ in $\rr^m$, let $f:U\cap C_1\to\rr^n$ be a continuous map,
and let $L:\rr^m\to\rr^n$ be linear. Suppose that $f$ is differentiable at $0$
with differential $L$, i.e. that 
\begin{equation}
f(x)=f(0)+L.x+o(||x||)\quad\quad{\rm as}\quad\quad x\to_{C_1} 0\,.
\end{equation}
Let $S$ be a subset of $\rr^n$ having a cone $C_2$ as weakly approximating
cone at $f(0)$. Assume that $C_2$ is not a linear subspace of $\rr^n$. Then
a necessary condition for the sets $f(U\cap C_1)$ and $S$ to be locally
separated at $f(0)$ is that 
$(LC_1)^\dagger\cap (-C_2)^\dagger\neq\{0\}$. \proofend
\end{theorem}

\noindent In other words, if $f(U\cap C_1)\cap S\cap V=\{f(0)\}$ for some
neighborhood $V$ of $f(0)$, then there exists a nonzero row vector $\bar
z\in\rr_n$ such that $\bar z.L.v_1\le 0$ for all $v\in C_1$ and $\bar
z.v_2\ge 0$ for all $v\in C_2$.

\section{A Streamlined Version of the Classical Maximum Principle}
\label{STRLINED}
%\bigskip

As a first step in our discussion of recent formulations of the Maximum
Principle, we now present a rigorous version that resembles in spirit the
``classical'' version given in the 1962 book by Pontryagin {\it et al.}
\cite{pontrybook}.  Actually,
the theorem to be given here is much stronger than the classical result,
especially because of the stronger form of the Transversality Condition, as
will be explained below.  It is, however, a distillation of what lies
behind the method of proof of \cite{pontrybook}, whose main point is, in
our view, that it relies on the {\bf\em construction of needle variations}
\footnote{\baselineskip=7pt The idea of using needle variations goes back
to Weierstrass (1815-1897), who gave three series of lectures on the
calculus of variations, in 1875, 1879, and 1882.  Using needle variations,
Weierstrass proved the ``side condition,'' which amounts, in modern terms, 
to the Maximum Principle for the problem (SCV).},
and the {\bf\em differentiability in the classical sense
of the reference flow and the variations.}


The proof strategy consists of constructing needle variations, studying the
corresponding endpoint maps, computing their differentials, and then
applying a separation result. The ``classical'' case arises when the
differentials used are classical differentials. In that case, the
appropriate separation result is our Theorem \ref{theosepclas}.  In
Section \ref{BEYOND} we will show how, using essentially the same approach but
relying on more general concepts of differential, it is possible to derive
much stronger results that contain, extend, and strengthen various
``nonsmooth'' versions of the Maximum Principle.

First, let us define the {\em needle variation 
of a map $\eta:[a,b]\to U$ obtained by inserting a
control value $u\in U$ at  a time $s\in[a,b[$} to be the family 
$\{\nu_{s,u}(r)(\eta)\}_{0\le r\le b-s}$ of maps
$\nu_{s,u}(r)(\eta):[a,b]\to U$ defined by 
\begin{equation} 
\nu_{s,u}(r)(\eta)(t)=\left\{\begin{array}{ll}
\eta(t)&{\rm if}\quad t\in[a,b]\,\backslash\,\,]s,s+r]\,,\\
u&{\rm if}\quad t\in\,\,]s,s+r]\,.\end{array}\right.
\end{equation}
With this definition, each $\nu_{s,u}(r)\,\,$---for 
$0\le r\le b-s$---is a map from the set $MAP([a,b],U)$ of all maps
$\eta:[a,b]\to U$ into itself, and $\nu_{s,u}(0)(\eta)=\eta$. 

Clearly, we can consider composites of the maps  $\nu_{s_j,u_j}(r_j)$. 
Precisely, if $a\!\le\! s_1\!<\! s_2\!<\! \cdots\!<\! s_m\!<\!b$,
${\bf u}\!=\!(u_1,\cdots,u_m)\!\in\! U^m$ is an $m$-tuple of control
values, and ${\bf s}=(s_1,\ldots,s_m)$, we define maps
$$
\nu_{{\bf s},{\bf u}}(r_1,\ldots,r_m):
MAP([a,b],U)\to MAP([a,b],U)\,,
$$
if \mbox{$0\le r_i\le b-s_i$ for $i=1,\cdots,m$}, by letting
\begin{equation} 
\nu_{{\bf{s}},{\bf{u}}}(r_1,\ldots,r_m)\equaldef\nu_{s_1,u_1}(r_1)
\compo\nu_{s_2,u_2}(r_2)\compo\cdots\compo\nu_{s_m,u_m}(r_m)\,.
\end{equation}
We then define a {\bf\em needle-variational neighborhood} of $\eta$ to be a
set ${\cal N}$ such 
that, for every $m\in\nn$, every choice of ${\bf s}=(s_1,\ldots,s_m)$
such that $a\le s_1<s_2<\cdots<s_m<b$,
and every choice of ${\bf u}=(u_1,\ldots,u_m)$ in $U^m$, there exists an
$\bar{r}>0$ such that 
$\nu_{{\bf s},{\bf u}}({\bf r})(\eta)\in{\cal
N}$ for every $m$-tuple ${\bf r}=(r_1,\ldots,r_m)\in[0,\bar r]^m$,  such that
${\bf s}+{\bf r}\in[a,b]^m$.

To every map $\eta:[a,b]\to U$, we associate a time-varying vector 
field $f_\eta:Q\times[a,b]\to\rr^n$ by letting
$$
f_{\eta}(q,t)\equaldef f(q,\eta(t),t)\,.
$$
We say that a time-varying $\rr^n$-valued vector field $F$, defined on 
some subset ${\rm Dom}(F)$ of $\rr^n\times\rr$, satisfies 
a {\bf\em $C^1$-Carath\'eodory condition} near a curve $\xi:[a,b]\to\rr^n$ 
if there exists a tube 
\begin{equation}\label{tubedef}
{\cal T}(\xi,\varepsilon)\equaldef\{(q,t):a\le t\le b, 
\|q-\xi(t)\|\le\varepsilon\}\,, 
\end{equation}
such that 
{\sl 
\begin{itemize}\begin{itemize}
\item[(M-Car)] $F(q,t)$ $\,$is $\,$measurable $\,$with $\,$respect 
$\,$to $\,t$---on $\,$the  $\,$compact $\,$set
$\{t:(q,t)\in{\cal T}(\xi,\varepsilon)\}$---for each fixed $q$, and
of class $C^1$ with respect to  $q$---on the closed ball
$\{q\in\rr:\|q-\xi(t)\|\le\varepsilon\}$---for each  fixed $t\in[a,b]$, 
\item[($C^1$-Car)] there is a function $\varphi\in
L^1([a,b],\rr)$ such that 
$$
\|F(q,t)\|+\Big\Vert\frac{\partial F}{\partial q}(q,t)\Big\Vert\le\varphi(t)
$$
for all $(q,t)\in {\cal T}(\xi,\varepsilon)$.
\end{itemize}\end{itemize}
}

We then assume that 
{\sl \medbreak 
\begin{itemize} \begin{itemize}
\item[(CTH.1)] The class ${\cal U}$ of admissible
controls is a needle-variational neighborhood of $\eta_*$.
\item[(CTH.2)] For every control $\eta$ which is either 
constant or equal to the reference control
$\eta_*$, the corresponding time-varying vector field $f_\eta$ satisfies a 
$C^1$-Carath\'eodory condition near $\xi_*$.
\item[(CTH.3)] A {\bf\em trajectory} for a control $\eta\in{\cal U}$ 
is a locally absolutely continuous map $\xi:[a_\eta,b_\eta]\to Q$ 
such that $\dot\xi(t)=f(\xi(t),\eta(t),t)$ for almost 
all $t\in[a_\eta,b_\eta]$. A {\bf\em trajectory-control pair} 
is a pair $(\xi,\eta)$ such that $\eta\in{\cal U}$ and 
$\xi$ is a trajectory for $\eta$.
\item[(CTH.4)] A {\bf\em solution of the adjoint equation (AE) along
the reference trajectory-control pair $\gamma_*=(\xi_*,\eta_*)$} is an
absolutely continuous map $\psi:[a,b]\to \rr_n$ 
such that the equality
$$
-\dot\psi(t)=\psi(t).\frac{\partial f}{\partial q}(\xi_*(t),\eta_*(t),t)
$$
holds for almost all $t\in[a_\eta,b_\eta]$.
\item[(CTH.5)] The Hamiltonian minimization condition (HMC) is interpreted
{\bf\em weakly}, as the statement that 
\begin{itemize}\begin{itemize}
\item[(WHMC)]
for every $u\in U$,  the inequality 
$$
\psi(t).f(\xi_*(t),\eta_*(t),t)\ge\psi(t).f(\xi_*(t),u,t)
$$ 
holds for almost all $t\in[a,b]$.
\end{itemize}\end{itemize}
\item[(CTH.6)] ``Tangent cone'' means {\bf\em weakly approximating cone}.
\end{itemize}\end{itemize}
}
\noindent (For simplicity, we shall refer to (CTH) as the ``classical
technical hypotheses,'' because they are roughly those of \cite{pontrybook},
although they differ from them in some important technical aspects
that will be discussed later.)

Under (CTH), all the terms occurring in (SSMP)
are now precisely defined, and the following rigorous result holds:

\begin{theorem}\label{maxprinclas}
Under the classical technical hypotheses (CTH), 
the Maximum Principle (SSMP) is true. \proofend
\end{theorem}

We now outline the proof of Theorem \ref{maxprinclas}, following the ideas
of \cite{pontrybook}.  Let $\tilde{\cal U}$ be the union of $\{\eta_*\}$
and the set of all constant controls $\eta:[a,b]\to U$. Then the
$C^1$-Carath\'eodory condition implies that every control $\eta\in\tilde
{\cal U}$  gives rise to a {\sl flow of class $C^1$}, i.e., to a family
$\Phi^{\eta}=\{\Phi_{t,s}^\eta\}_{a\le s\le t\le b}$ of maps of class
$C^1$, each one of which is defined on an open subset ${\rm
Dom}(\Phi_{t,s}^\eta)$ of $Q$. The maps $\Phi_{t,s}^\eta$ are uniquely
characterized by the condition that, if $a\le s\le t\le b$, and $q\in Q$,
then $\Phi_{t,s}^\eta(q)$ is defined if and only if the solution
$\tau\to\xi_{q,\eta,s}(\tau)$ of the initial value problem
$\dot\xi(\tau)=f(\xi(\tau),\eta(\tau),\tau)$, $\xi(s)=q$, is defined on
$[s,t]$, and in that case $\Phi_{t,s}^\eta(q)=\xi_{q,\eta,s}(t)$.  The flow
maps $\Phi_{t,s}^\eta$ satisfy the obvious identities $\Phi_{t,t}^\eta={\rm
identity}$ and
$\Phi_{t_3,t_2}^\eta\compo\Phi_{t_2,t_1}^\eta=\Phi_{t_3,t_1}^\eta$ whenever
$a\le t_1\le t_2\le t_3\le b$.

\def\gammast{\gamma_*}

The differentials $D\Phi_{t,s}^{\eta_*}$   of the maps 
$\Phi_{t,s}^{\eta_*}$  satisfy the {\sl variational equation}:
\begin{equation}\label{flafordphi}
D\Phi_{t,s}^{\eta_*}(q)
={\rm identity}+\int_s^t\frac{\partial f}{\partial q}\Big(
\Phi_{\tau,s}^{\eta_*}(q),\eta_*(\tau),\tau\Big).D\Phi_{\tau,s}^{\eta_*}(q)
\,d\tau\,.
\end{equation}
Use $\gamma_*$ to denote the reference trajectory-control pair
$(\xi_*,\eta_*)$.  We then define the {\it linearized flow}
${\bf{L}}^{\gammast}$ to be the family
${\bf{L}}^{\gammast}=\{L^{\gammast}(t,s)\}_{a\le s\le t\le b}$
of linear maps $L^{\gammast}(t,s):\rr^n\to\rr^n$ given by 
$$L^{\gammast}(t,s)=D\Phi_{t,s}^{\eta_*}(\xi_*(s))\,.
$$
Also, define 
$$
M^{\gammast}(t)=\frac{\partial f}{\partial q}\Big(\xi_*(t),\eta_*(t),t\Big)\,.
$$
Then the linear maps $L^{\gammast}(t,s)$ satisfy the identities 
$L^{\gammast}(t,t)={\rm identity}$  and 
$L^{\gammast}(t_3,t_2)\compo L^{\gammast}(t_2,t_1)=L^{\gammast}(t_3,t_1)$ whenever
$a\le t_1\le t_2\le t_3\le b$.  
Also, (\ref{flafordphi}) says that 
\begin{equation}\label{flaforl}
L^{\gammast}(t,s)={\rm identity}\!+\!\int_s^t M^{\gammast}(\tau)
L^{\gammast}(\tau,s)\,d\tau\,\,\,{\rm
if\,\,\,}a\!\le\!s\!\le\!t\!\le\!b. 
\end{equation}
It then follows easily that 
\begin{equation}\label{flaforlback}
L^{\gammast}(t,s)={\rm
identity}\!+\!\int_s^tL^{\gammast}(t,\sigma)M^{\gammast}(\sigma)\,d\sigma 
\,\,\,{\rm if}\,\,\,a\!\le\! s\!\le\! t\!\le\! b.
\end{equation}
This implies, in particular, that {\sl a map $[a,b]\ni t\to
\psi(t)\in\rr_n$ is a solution of the adjoint equation if and only if 
it satisfies the ``integrated adjoint equation''}
\begin{equation}\label{adjeqint}
\psi(t)=\psi(b).L^{\gammast}(b,t)\quad{\rm for \quad}a\le t\le b.
\end{equation}

Now, let $\hat{U}$ be a fixed finite subset of $U$.  We then construct needle
variations of the reference trajectory-control pair
$\gamma_*=(\xi_*,\eta_*)$ using control values in $\hat{U}$. Precisely, 
we pick an $m$-tuple ${\bf s}=(s_1,\ldots,s_m)$ 
such that $a\le s_1<s_2<\cdots<s_m<b$, and an
$m$-tuple ${\bf u}=(u_1,\ldots,u_m)$ of members of $\hat{U}$. Our hypotheses
then imply that there exists an $\bar r>0$ such that

\medbreak
\begin{itemize}
\item[1.] the controls $\nu_{{\bf s},{\bf u}}({\bf r})(\eta_*)$ are in ${\cal
{}U}$ whenever ${\bf{r}}=(r_1,\ldots,r_m)\in[0,\bar r]^m$, 
\item[2.] the {\bf\em endpoint map} ${\cal{E}}_{{\bf s},{\bf u},\eta_*}$,
given by 
\begin{equation}
{\cal{E}}_{{\bf s},{\bf u},\eta_*}(q,{\bf r})\equaldef
\Phi_{b,a}^{\nu_{{\bf s},{\bf u}}({\bf r})(\eta_*)}(q)\,,
\end{equation}
is defined and continuous on $B(\xi_*(a),\bar r)\times[0,\bar r]^m$,
where 
$$
B(\xi_*(a),\bar r)\equaldef\{q\in\rr^n: \Vert q-\xi_*(a)\Vert\le\bar r\}\,.
$$ 
\end{itemize}

\medbreak\noindent 
Moreover, one can prove that, if the times $s_1,\ldots,s_m$ are chosen in a
special way, then the map 
${\cal{E}}_{{\bf s},{\bf u},\eta_*}$ is differentiable at 
$(\xi_*(a),0)$, and the differential 
$D{\cal{E}}_{{\bf s},{\bf u},\eta_*}(\xi_*(a),0)$ is the linear map 
from $\rr^n\times\rr^m$ to $\rr^n$ given by 
\begin{eqnarray}
D{\cal{E}}_{{\bf s},{\bf u},\eta_*}(\delta q,\delta r_1,\ldots,\delta r_m)&\,\,&\nonumber\\
&\,\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!=
L^{\gamma_*}(b,a).\delta q+
\sum_{i=1}^m\delta r_i.L^{\gamma_*}(b,s_i).v^{\gamma_*}(s_i,u_i)\,,\label{diffofendpo}
\end{eqnarray}
where 
$$
v^{\gamma_*}(s,u)\equaldef f(\xi_*(s),u,s)-f(\xi_*(s),\eta_*(s),s)\,.
$$      
Precisely, what is required for (\ref{diffofendpo}) to hold
is to choose the $s_i$ belonging to the 
``good set''   $G(\eta_*,\hat{U})$ of all times $s\in[a,b[$ 
such that 

{\sl 
\begin{itemize}
\item[(G)] there exist a compact subset $K$ of $[a,b]$
and an integrable function $\varphi:[a,b]\to[0,\infty]$,  such 
that
\begin{equation}
\lim_{r\downarrow 0}\frac{1}{r}\Big({\rm meas}(K\cap[s,s+r])\Big)=1\,,
% GNU Original was
% \lim_{r\downarrow 0}\frac{1}{r}\Big({\rm meas}(K\cap[s,s+r])\Big)=1
% Not corrected for book. 
\end{equation}
\begin{equation}
\lim_{r\downarrow 0}\frac{1}{r}\int_s^{s+r}|\varphi(t)-\varphi(s)|\,dt=0\,,
\end{equation}
the inequality
\begin{equation}
\|f(q,u,t)\|\le\varphi(t)
\end{equation}
holds whenever $t\in[s,s+r]$ and $u\in \hat{U}\cup\{\eta_*(t)\}$, 
\begin{equation}
\lim_{q\to \xi_*(s),r\downarrow 0,s+r\in K}
f(q,u,s\!+\!r)\!=\!f(\xi_*(s),u,s)\,\,{\rm for\quad all}\,\,u\in \hat{U}\,,
\end{equation}
and 
\begin{equation}
\lim_{q\to \xi_*(s),r\downarrow 0,s+r\in K}
f(q,\eta_*(s+r),s\!+\!r)\!=\!f(\xi_*(s),\eta_*(s),s)\,.
\end{equation}
\end{itemize}
}

\medbreak\noindent
It follows from the Scorza-Dragoni theorem that, under our assumptions,
the set  $[a,b]\backslash G(\eta_*,\hat{U})$ has measure zero. 

Now suppose that ${\cal R}_{[a,b]}^\Sigma(\xi_*(a))$ and $S$ are locally
separated at $\xi_*(b)$.  Then {\sl a fortiori} the sets ${\cal E}_{{\bf
s},{\bf u},\eta_*}\Big(\{\xi_*(a)\}\times[0,\bar r]^m\Big)$ and $S$ are locally
separated at $\xi_*(b)$, if $\bar r>0$ is small enough. Applying Theorem
\ref{theosepclas}, we find that there exists a covector $\bar
p\in\rr_n$ such that $\|\bar p\|=1$, $\bar p.v\ge 0$ for all $v\in C$, and 
\begin{equation}
\bar p.L^{\gamma_*}(b,s_i).v^{\gamma_*}(s_i,u_i)\le 0\quad{\rm for\quad}i=1,\ldots,m\,.
\end{equation}
If we define 
$$
\psi(t)=\bar p.L^{\gamma_*}(b,t)\quad{\rm for}\quad t\in[a,b]\,,
$$ 
we have 
$\psi(b).v\ge 0$ for all $v\in C$, and 
\begin{equation}
\psi(t).v^{\gamma_*}(s_i,u_i)\le 0\quad{\rm for\quad}i=1,\ldots,m\,.
\end{equation}

The conclusion of the Maximum Principle follows from this by an elementary
compactness argument. Indeed, using Lusin's Theorem, we can write
\mbox{$G(\eta_*,\hat{U})=\bigcup_{k=0}^\infty E_k$}, where 
${\rm meas}(E_0)=0$ and, for $k>0$, the set $E_k$ is compact and such that 
the restrictions to $E_k$ of the maps \mbox{$t\to v^{\gamma*}(t,u)$} 
are continuous for every $u\in \hat{U}$. For $k>0$, let $D_k$ be the set of
points of density of $E_k$.    Let \mbox{$D=\bigcup_{k=1}^\infty D_k$},
so \mbox{${\rm meas}(D)=b-a$}.  If $F$ is an arbitrary subset of
$D\times \hat{U}$, let $\Pi(F,\hat{U})$ be the set of 
all $\bar p\in\rr_n$ such that $\|\bar p\|=1$,
$-\bar p\in C^\dagger$, and 
\mbox{$\bar p.L^{\gamma_*}(b,t).v^{\gamma_*}(t,u)\le 0$}
for all $(t,u)\in F$. Then $\Pi(F,\hat{U})$ is 
clearly compact, and 
$$\Pi(F_1\cup F_2,\hat{U})=\Pi(F_1,\hat{U})\cap\Pi(F_2,\hat{U})\quad
{\rm for\quad all\quad}F_1,F_2\,.
$$
We already know that $\Pi(F,\hat{U})\neq\emptyset$ if
\mbox{$F=\{(s_1,u_1),\ldots,(s_m,u_m)\}$} with 
\mbox{$s_1<s_2<\cdots<s_m$}.     If $F$ is an arbitrary finite 
subset of \mbox{$D\times \hat{U}$}, we can write 
\mbox{$F=\{(s_1,u_1),\ldots,(s_m,u_m)\}$} with 
\mbox{$s_1\le s_2\le\cdots\le s_m$}.    
Each $s_i$ belongs to $D_{k(i)}$ for some
$k(i)$, so $s_i=\lim_{j\to\infty}s_{i,j}$, where 
$s_{i,j}\in D_{k(i)}$ and \mbox{$s_{1,j}<s_{2,j}<\cdots <s_{m,j}$} for each
$j$. If \mbox{$F_j=\{(s_{1,j},u_1),\ldots,(s_{m,j},u_m)\}$}, then 
\mbox{$\Pi(F_j,\hat{U})\neq\emptyset$}. Pick 
\mbox{$\bar p_j\in\Pi(F_j,\hat{U})$}. 
Then \mbox{$\|\bar p_j\|=1$}. Pick a subsequence $\{\bar p_{j(\ell)}\}$ of 
$\{\bar p_j\}$ that converges to a limit $\bar p$. Then 
$$\bar{p}_{j(\ell)}.L^{\gamma_*}(b,s_{i,j(\ell)}).
v^{\gamma_*}(s_{i,j(\ell)},u_i)\le 0  
\quad{\rm for\quad all}\quad i,\ell\,.$$
Since \mbox{$s_{i,j}\to s_i$}, \mbox{$s_{i,j}\in E_{k(i)}$,}
\mbox{$s_i\in E_{k(i)}$}, and the function 
\mbox{$s\to v^{\gamma_*}(s,u_i)$} is continuous on
$E_{k(i)}$, we have 
$$
\bar p.L^{\gamma_*}(b,s_{i}).v^{\gamma_*}(s_{i},u_i)\le 0
\quad{\rm for}\quad i=1,\ldots,m\,.
$$
Clearly, $\|\bar p\|=1$ and
$-\bar p\in C^\dagger$, so $\bar p\in\Pi(F,\hat{U})$. Therefore
$\Pi(F,\hat{U})\neq\emptyset$ if $F$ is an arbitrary finite subset 
of $D\times \hat{U}$.   So  
\mbox{$\{\Pi(F,\hat{U})\}_{F\subseteq D\times \hat{U},
{\rm card}(F)<\infty}$} is a family of compact subsets
having the finite intersection property. This implies that 
the intersection of all the $\Pi(F,\hat{U})$, as $F$ ranges over all finite
subsets of \mbox{$D\times \hat{U}$}, is nonempty.  Therefore 
$$
\Pi(D\times \hat{U})\neq\emptyset\,.
$$
Now, for an arbitrary subset $\hat U$ of $U$, let 
$\tilde\Pi(\hat{U})$ be the set of all 
$\bar p\in\rr_n$ such that $\|\bar p\|=1$,
$-\bar p\in C^\dagger$, and, for every $u\in \hat{U}$,
$\bar p.L^{\gamma_*}(b,t).v^{\gamma_*}(t,u)\le 0$ 
for almost all $t\in[a,b]$. Once again, 
$\tilde\Pi(\hat{U})$ is compact for every $\hat{U}$, and 
$\tilde\Pi(\hat{U}_1\cup\hat{U}_2)$ is equal to 
$\tilde\Pi(\hat{U}_1)\cap\tilde\Pi(\hat{U}_2)$ 
for all $\hat{U}_1,\hat{U}_2$.   (The compactness of 
$\tilde\Pi(\hat{U})$ follows because, if $\bar p_j\in 
\tilde\Pi(\hat{U})$ and $\bar p_j\to\bar p$, then 
$\int_\alpha^\beta
\bar p_j.L^{\gamma_*}(b,t).v^{\gamma_*}(t,u)\,dt\le 0$ 
whenever $a\le \alpha\le\beta\le b$ and $u\in\hat{U}$, so 
$\int_\alpha^\beta
\bar p.L^{\gamma_*}(b,t).v^{\gamma_*}(t,u)\,dt\le 0$ 
whenever $a\le \alpha\le\beta\le b$ and $u\in\hat{U}$, and then,
for each fixed $u$, the inequality 
$\bar p.L^{\gamma_*}(b,t).v^{\gamma_*}(t,u)\le 0$ must hold
for almost all $t\in[a,b]$.)  So 
$\{\tilde\Pi(\hat{U})\}_{\hat{U}\subseteq U,
{\rm card}(\hat{U})<\infty}$ is a family of compact subsets
having the finite intersection property. This implies that 
the intersection of all the 
$\tilde\Pi(\hat{U})$, as $\hat{U}$ ranges over all 
finite subsets of $U$, is nonempty.  Therefore 
$\tilde\Pi(U)\neq\emptyset$, which is precisely the desired
conclusion. \proofend

\medbreak
The proof of the Maximum Principle outlined above is essentially based on
the ideas of \cite{pontrybook}. There are, however, some major differences
between our technical assumptions and those of \cite{pontrybook}, which
make our version stronger.

First of all, in \cite{pontrybook} $f$ is assumed to depend on $q$ and
$u$ only.\footnote{\baselineskip=7pt{As explained in Chapter 1 of
\cite{pontrybook}, a result in which $f(q,u)$ is assumed to be of
class $C^1$ with respect to the state $q$ can be transformed
immediately into a result in which $f$ is allowed to be 
time-dependent---i.~e., $f=f(q,u,t)$---{\em provided that $f$ is of class
$C^1$ with 
respect to $q$ and $t$}. This is done by adding time as extra state
variable $q_0$, obeying the equation $\dot q_0=1$.  This procedure does
not, however, allow the extra generality of considering systems where
$f$ is only measurable with respect to $t$.}}

Second, \cite{pontrybook} assumes that $U$ is a subset of some Euclidean
space $\rr^m$,\footnote{\baselineskip=7pt The authors of \cite{pontrybook}
were obviously aware of the possibility of generalizing the Maximum
Principle to the case when $U$ is an arbitrary set. Indeed, they state in a
footnote, on page 75, that their arguments ``carry over without any change
to the case where $U$ is an arbitrary subset of some topological Hausdorff
space with a denumerable basis.''  They then add that ``an insignificant
modification of the proof even makes it possible to remove the requirement
that there exist a denumerable basis.''} and both $f(q,u)$ and
$\frac{\partial{f}}{\partial q}(q,u)$ are jointly continuous with respect
to $q$ and $u$ on the product $Q\times\bar U$.  It is assumed that every
admissible control is a bounded measurable $U$-valued map on $[a,b]$,
where ``bounded'' means ``with values in a compact subset of $U$.'' The
class ${\cal U}$ of admissible controls is assumed to contain the reference
control, and to be such that, if $\eta\in{\cal U}$ has domain
$[a_\eta,b_\eta]$, $t'$ and $t''$ are numbers such that $a_\eta\le t'\le
t''\le b_\eta$, and $u\in U$, then the control $[a_\eta,b_\eta]\ni t\to
\eta(t)+\chi_{_{[t',t'']}}(t)(u-\eta(t))$ also belongs to ${\cal U}$.

This is more than enough to guarantee that our technical hypotheses
hold.  Moreover, since $U$ is a subset of $\rr^m$, and $f(q,u)$ is
continuous with respect to $u$, the ``weak'' Hamiltonian minimization
condition of (CTH.5) implies the ``strong'' version of (HMC):
\begin {itemize}\begin {itemize}
\item[(SHMC)]
{\sl The inequality 
$$\psi(t).f(\xi_*(t),\eta_*(t))=\max\Big\{\psi(t).f(\xi_*(t),u):u\in
U\Big\}$$  
holds for almost all $t\in[a,b]$.}
\end{itemize}\end{itemize}
(Indeed, $U$ is a separable metric space.  If $\psi$ satisfies
(WHMC), then we can pick a countable dense subset $U_0$ of $U$, 
and a ``bad'' subset $B\subseteq[a,b]$  of measure zero such that 
$\psi(t).f(\xi_*(t),\eta_*(t))\ge\psi(t).f(\xi_*(t),u)$
whenever  $t\in[a,b]\backslash B$ and $u\in U_0$. The continuity of 
$f$ with respect to $u$ then implies that (SHMC) holds.)

Third, {\sl there is a fundamental difference between our version of the
Transversality Condition (TC) and that of 
\cite{pontrybook}}, arising from the fact that our version uses
a much weaker concept of ``tangent cone'' than that of \cite{pontrybook}.
Indeed, the concept of tangent cone implicitly used in \cite{pontrybook}
in the formulation of (TC) is the one that we have called
``approximating cone'' in this discussion.\footnote{\baselineskip=7pt
The definition of ``tangent cone'' as ''approximating cone'' is not given
explicitly in \cite{pontrybook}.  The crucial topological argument 
needed to establish the transversality  conditions corresponds to our
Theorem \ref{theosepclas}, with ``approximating cone''
instead of ``weakly approximating cone.'' The argument 
is only sketched in \cite{pontrybook}, and  the reader is referred for full
details to \cite{boltyanskii}.   Cf. Remark \ref{compaboltours} below
for a comparison of the two separation theorems.} 

The following example shows how this makes a difference.
\begin{example}{\rm 
Consider the optimal control problem in $\rr^2$---with coordinate point
$q=(x,y)$---in which it is desired to minimize the integral
% GNU Original was
% $q=(x,y)$---in which is it desired to minimize the integral
% Not corrected for book. 
$$J=-\int_0^1u(t)dt\,,
$$
subject to the dynamical law $\dot x=u$, $\dot y=v$ and 
the endpoint constraints $\xi(0)=(0,0)$, $\xi(1)\in S$, where 
$$
S=\Big\{(x,y):x>0\wedge y=x\sin{\frac{1}{x}}\Big\}\cup\{(0,0)\}\,.
$$
Then the trajectory $[0,1]\ni t\to (0,0)$, corresponding to the controls
\mbox{$u(t)\equiv v(t)\equiv 0$}, obviously fails to be a minimizer, since 
the choice of controls \mbox{$v(t)\equiv 0$}, $u(t)\equiv\frac{\pi}{k}$,
for any  
positive integer $k$, will yield a trajectory satisfying our endpoint
constraints and having a lower cost.  

On the other hand, {\sl the version of the Maximum Principle given in
\cite{pontrybook} will not exclude this trajectory as a candidate for a
minimum}.  This can be seen by explicitly finding an adjoint vector that
satisfies all the conditions, which is quite easy to do.
There is, however, an even simpler argument.  It suffices
to notice that the only approximating cone for the set $S$ at $(0,0)$ is
$\{(0,0)\}$.  Hence the conclusion of the Maximum Principle for our
optimization problem is identical to that arising from any modified problem
with the same dynamics and cost functional, and with the set $S$ replaced
by another set $S'$ having $\{(0,0)\}$ as an approximating cone at
$(0,0)$. In particular, we can take $S'=\{(0,0)\}$. Since the cost of any
feasible trajectory $t\to\xi(t)=(x(t),y(t))$ is just $-x(1)$, we see that
all feasible trajectories of our original problem that end at $(0,0)$ are
actually minimizers for the modified problem, so they satisfy the
conditions of the Maximum Principle of \cite{pontrybook}.

It turns out, however, that {\sl our version of the transversality
condition excludes the trajectory
$[0,1]\ni t\to (0,0)$}. To see this, notice that the cone $C=\{(x,y):|y|\le
x\}$ is a ``tangent cone'' to $S$ at $(0,0)$ in our sense. To satisfy the
necessary condition of the Maximum Principle we need a vector
$(\psi_1,\psi_2)\in\rr_2$ and a nonnegative $\psi_0$ such that, to begin with,
$\psi_1u+\psi_2v+\psi_0u$ is maximized by $u=v=0$, for which we must have
$\psi_2=0$ and 
$\psi_1+\psi_0=0$. Moreover, since $(\psi_0,\psi_1,\psi_2)\neq(0,0,0)$, we
need $\psi_0>0$ and $\psi_1<0$.   But the transversality condition yields
$-(\psi_1,\psi_2)\in C^\dagger$, so $\psi_1\ge 0$, since $(1,0)\in C$. 
So we have reached a contradiction.}  \proofend
\end{example}

For a second illustration of the extra power of our version of the
``classical'' Maximum Principle, we observe that, {\sl if ``tangent cone''
is taken to mean ``approximating cone,'' then
Condition (P2) does not hold, so the local controllability result is
{\bf\em not} a special case of the general separation theorem, and must be
handled separately.}  On the other hand, {\sl if ``tangent cone'' is
% GNU Original was
% handled separately.}  On the other hand, if {\sl if ``tangent cone'' is
% Not corrected for book. 
taken to mean ``weakly approximating cone,'' then (P2) holds, so {\bf\em the
local controllability result becomes a special case of the general maximum
principle, and does not require a separate analysis}.}

\begin{remark}\label{compaboltours}{\rm 
The most important technical difference between our version of
the classical Maximum Principle and that of \cite{pontrybook} is our use of
weakly approximating cones.   This, in our view, solves
a puzzling question regarding the proof given in \cite{pontrybook}. The
crucial point of that proof is found on page 97 of \cite{pontrybook},
where the authors---assuming that the necessary condition of the Maximum
Principle is violated---are trying to prove that the intersection of two
sets contains more than one point. They do this by proving the existence
of a {\em one-parameter family} of points $q_\varepsilon$, one for each 
sufficiently small positive value of $\varepsilon$.  This argument strongly
suggests that one ought to be able to prove the existence of a full 
curve of intersection points and get a result stronger than what is
actually proved in \cite{pontrybook}. It turns out that this is not quite
true, but is nearly true.  What actually follows when the argument 
is carried out with care, is the existence of a nontrivial {\bf\em
connected} set of intersection points. The set may, however,
fail to be path-connected.
The key result is the following theorem (cf. Leray-Schauder
\cite{leray}, Browder \cite{browder}),
in which we use $\bb^n$ to denote the closed unit ball in $\rr^n$: 

\begin{theorem}\label{theobrowder}
Let $F:[0,1]\times\bb^n\to\rr^n$ be a continuous map, 
such that 
\begin{itemize}
\item[1.] $0\notin F\Big([0,1]\times\partial\bb^n\Big)$,
\item[2.] the map $\partial\bb^n\ni x\to F(0,x)\in\rr^n\backslash\{0\}$ 
is homotopic (in the class of continuous maps from 
$\partial\bb^n$ to $\rr^n\backslash\{0\}$)  to the inclusion map 
$x\to x$. 
\end{itemize}
Then there exists a compact connected
subset $Z$ of $[0,1]\times\bb^n\to\rr^n$ such that 
$Z\cap(\{0\}\times\bb^n)\neq\emptyset$, 
$Z\cap(\{1\}\times\bb^n)\neq\emptyset$, and $F(t,x)=0$ whenever 
$(t,x)\in Z$.
\end{theorem}
In other words: if $\{F_t\}_{0\le t\le 1}$ is a homotopy of maps 
from $\bb^n$ to $\rr^n$ that never take the value $0$ on the boundary 
of $\bb^n$, and $F_0$, restricted to the boundary of $\bb^n$, is homotopic
to the inclusion map, then not only is there a zero of $F$ at each level of
the homotopy---as implied by standard degree theory---but in addition
there must exist a connected set of zeros of the map 
\mbox{$(t,x)\to F_t(x)$} that contains a zero at each end of the homotopy.

Theorem \ref{theobrowder} is the basic tool used in \cite{sussmulti} to
prove Theorem \ref{theosepclas} and its generalizations.  Our definition of
``weakly approximating cone'' is precisely what is needed to make this
work.}\proofend
\end{remark}

\section{Clarke's Nonsmooth Version and the \L{}ojasiewicz Improvement}
\label{BEYOND}
%\bigskip

Theorem \ref{maxprinclas} and its proof, as sketched in Section \ref{STRLINED},
admit an obvious generalization, obtained by observing that, in the proof, 
the $C^1$-Carath\'eodory condition for the controls other than the reference
control is never used. In fact, the formula for the differential 
of the endpoint map is valid without any change if the ``constant control''
time-varying vector fields  $(q,t)\to f(q,u,t)$ are only assumed to
satisfy the following ``\L{}ojasiewicz technical hypothesis'':
\begin{itemize}\begin{itemize}
\item[(LTH)] {\sl There exists an $\varepsilon_u>0$ such 
that $f(q,u,t)$ is continuous in $q$ for each fixed $t$
and measurable in $t$ for each fixed $q$ on the tube ${\cal
T}(\xi_*,\varepsilon_u)$,  and the bound
$\|f(q,u,t)\|\le\varphi(t)$ holds
for some integrable function $[a,b]\ni t\to\varphi(t)$,
}
\end{itemize}\end{itemize}
and to be such that  
\begin{itemize}\begin{itemize}
\item[(U)] {\sl the ordinary differential equation $\dot q=f(q,u,t)$ has
uniqueness of solutions.}
\end{itemize}\end{itemize}
Condition (U) is only needed if one insists on having a single-valued
continuous endpoint map that is differentiable in the classical sense, but
can be dispensed with if one drops this requirement. Indeed, it turns out
that there is a very nice class of {\sl set-valued} maps for which a
concept very similar to that of classical differentiability exists, and the
analogues of Theorems \ref{theobrowder} and \ref{theosepclas} are still
valid.  Precisely, let us call a set-valued map $F$ from a compact metric
space $X$ to a metric space $Y$ {\sl regular} if

\begin{itemize}\begin{itemize}
\item[(REG.1)] {\sl the graph $  GR(F)=\{(x,y): y\in F(x)\}$
is compact;}
\item[(REG.2)] {\sl there exists a sequence $\{F_j\}_{j=1}^\infty$ of
single-valued continuous maps from $X$ to $Y$ that converges to $F$ in the
graph sense, i.e. is such that}
$$
\lim_{j\to\infty}\sup\Biggl\{{\rm dist}\Big((x,y),GR(F)\Big):(x,y)\in GR(F_j)
\Biggr\}=0\,.
$$
\end{itemize}\end{itemize}
With this definition, it turns out that 
\begin{theorem}
Theorem \ref{theobrowder} also holds  if $F$ is assumed to be set-valued and 
regular, rather than single-valued and continuous. \proofend
\end{theorem}
\begin{remark}\label{rembrowder}{\rm 
Actually,
the ``regular'' version of  Theorem \ref{theobrowder} is more elegant and 
natural than the ``continuous'' version, because the  resulting 
set $Z$ can be taken to be of the form $\gamma([0,1])$, where 
$\gamma$ is a regular set-valued map from $[0,1]$ to 
$[0,1]\times\bb^n$.}\proofend
\end{remark}
Moreover, {\bf\em Theorem \ref{theosepclas} is true for regular maps}.  
(The definition of
``differential''  is still the obvious analogue of that of the classical
differential of a single-valued map: if $F$ is a set-valued map
from $\rr^n$ to $\rr^m$, $x\in\rr^n$,  $y\in\rr^m$, $F(x)=\{y\}$, $C$ is a
closed convex cone in $\rr^n$, and $L:\rr^n\to\rr^m$ is linear, we say that
$F$ is {\sl differentiable at $x$ in the direction of $C$, with
differential $L$}, if 
$$
\sup\Big\{\|y'-y-L(x'-x)\|:y'\in F(x')\Big\}=o(\|x'-x\|)
$$
as $x'-x\to_C0$.)

The flow maps $(t,s,q)\to \Phi_{t,s}^u(q)$ turn out to be regular near
$(t,t,q)$ for every $(t,q)\in[a,b]\times Q$ and every constant control
$u$, as long as the vector field $(q,t)\to f(q,u,t)$ satisfies
(LTH).  (The single-valued approximations 
required by (REG.2) are obtained by approximating the flow
$\Phi^u$ by the regularized flows $\Phi^{u,\rho}$, where 
$\Phi^{u,\rho}$ is the flow of the regularized vector field
$(q,t)\to f^\rho(q,u,t)$, with 
$$f^\rho(q,u,t)\equaldef\int_{\rr^n}f(q+\rho h,u,t)\sigma(h)dh$$
where $\sigma:\rr^n\to\rr$ is a smooth nonnegative function supported
in the ball $\{x:\|x\|\le 1\}$ and such that $\int\sigma=1$.)  Moreover, even
though 
the endpoint maps constructed in Section \ref{STRLINED} no longer need to be 
single-valued, they turn out to be regular near $(\xi_*(a),0)$
and, if the insertion times $s_i$ of the needle variations 
are chosen as explained in Section \ref{STRLINED}, then the differentiation
formula remains unchanged.  So the proof given in Section \ref{STRLINED}
is valid in the more general setting, and we get the 
{\sl classical Maximum Principle with the \L{}ojasiewicz 
improvement}\footnote{\baselineskip=7pt
Communicated to us by S. \L{}ojasiewicz, Jr., 
in a personal conversation that took place in May 1992. \L{}ojasiewicz's
brilliant idea was explained to us for the much more general setting of the
nonsmooth Maximum Principle, discussed below, and to our knowledge still
remains unpublished.}:
\begin{theorem}\label{maxprinclasloja}
Assume that the classical technical hypotheses (CTH) hold, except that only
the reference control $\eta_*$ is required to satisfy
a $C^1$-Carath\'eo\-dory condition near $\xi_*$, and for the constant
controls $u\in U$ we only  
assume that the weaker condition (LTH) holds. 
Then the Maximum Principle (SSMP) is true. \proofend
\end{theorem}
A different extension of the classical version of the Maximum Principle 
was proposed by F.H. Clarke in \cite{clarkeminimal} and \cite{clarkebook},
and has become known as the ``nonsmooth Maximum Principle.''  
In this version, Condition (CTH.2) is replaced by the weaker requirement
that the vector fields $(q,t)\to f(q,\eta(t),t)$ be Lipschitz continuous,
with an integral bound on the Lipschitz constant. 

Precisely, let us say that a time-varying vector field $F$, defined on 
some subset ${\rm Dom}(F)$ of $\rr^n\times\rr$, satisfies 
a {\bf\em Lipschitz-Carath\'eodory condition} near a curve
$\xi:[a,b]\to\rr^n$  if there exists a tube 
${\cal T}(\xi,\varepsilon)$ such that (M-Car) holds, and 
{\sl 
\begin{itemize}\begin{itemize}
\item[(Lip-Car)] there is a function $\varphi\in
L^1([a,b],\rr)$ such that 
$$
\|F(q,t)\|\le\varphi(t)\quad{\rm and}
\quad\Vert F(q',t)-F(q,t)\Vert\le\varphi(t).\Vert q'-q\Vert
$$
whenever $(q,t)\in {\cal T}(\xi,\varepsilon)$
and $(q',t)\in {\cal T}(\xi,\varepsilon)$.
\end{itemize}\end{itemize}
}

In the Clarke version of the Maximum Principle, Condition (CTH.2) is
replaced by the followin:
{\sl 
\begin{itemize}\begin{itemize}\begin{itemize}
\item[(CTH.2.Cl)] For every control $\eta$ that is either 
constant or equal to the reference control
$\eta_*$, the corresponding time-varying vector field $f_\eta$ satisfies a 
Lipschitz-Carath\'eodory condition near $\xi_*$.
\end{itemize}\end{itemize}\end{itemize}
}
\noindent Naturally, when this weaker condition holds, it is no longer
possible to interpret the adjoint equation in the classical sense, so
(CTH.4) has to be modified.  The modification proposed by Clarke was that
the adjoint equation be interpreted as a {\sl differential inclusion} with
the classical Jacobian matrix replaced by a Clarke generalized Jacobian. In
other words, Clarke suggested that, instead of (CTH.4), the following be
used: 
{\sl 
\begin{itemize}\begin{itemize}\begin{itemize}
\item[(CTH.4.Cl)] A {\bf\em solution of the adjoint equation (AE) along
the reference trajectory-control pair $\gamma_*=(\xi_*,\eta_*)$} is an
absolutely continuous map $\psi:[a,b]\to \rr_n$ 
such that
\begin{equation}\label{soladjeqlip}
\!\!\!\!-\dot\psi(t)\!\in\!\psi(t).\partial_1f(\xi_*(t),\eta_*(t),t)
\,\,for\,\,a.\,e.\,\,t\!\in\![a,b]\,.
\end{equation}
\end{itemize}\end{itemize}\end{itemize}
}
Here $\partial_1f(\bar q,u,t)$ is the {\sl Clarke generalized Jacobian}
(CGJ) at $\bar q$ of the map $q\to f(q,u,t)$.\footnote{\baselineskip=7pt 
For a Lipschitz continuous map $\mu:Q\to\rr^m$, where $Q$ is open in 
$\rr^n$, the CGJ of $\mu$ at $q\in Q$ is the convex hull $\partial\mu(q)$
of the set of all linear maps $L:\rr^n\to\rr^m$ such that 
$L=\lim_{j\to\infty}L_j$ for some sequence $\{L_j\}$ such that there are 
$q_j\in Q$ for which $q_j\to q$ and $\mu$ is differentiable at $q_j$ with 
differential $L_j$.  The existence of many such sequences $\{q_j\}$ follows
because $\mu$ is differentiable almost everywhere by Rademacher's Theorem.
The set $\partial\mu(q)$ is then nonempty, compact and convex.}

Clarke's own proof technique is not ``classical,'' in the sense that it
does not proceed as we did in  Section \ref{STRLINED}, i.~e.,
by constructing needle variations, differentiating the
endpoint maps, and then using a separation theorem such as Theorem
\ref{theosepclas}. It turns out, however, that a classical proof of
Clarke's result can be given, and this classical proof has a number of
advantages over the nonsmooth one, as we now show.

The link between Clarke's nonsmooth Maximum Principle and the classical
approach is provided by J. Warga's theory of ``derivate containers''
(cf. Warga [52]-[55]). A {\sl derivate container}
% GNU Original was
% (cf. Warga [52-55]) A {\sl derivate container}
% Not corrected for book. 
of a Lipschitz-continuous map $\mu:Q\to\rr^m$, where $Q$ is open in
$\rr^n$, at a point $q\in Q$ is a kind of ``generalized differential of
$\mu$ at $q$.''  The precise definition is as follows: 
\begin{definition}{\rm 
Let $Q$ be open in $\rr^n$, let $\mu:Q\to\rr^m$ be 
Lipschitz-continuous, and let $\bar q\in Q$. 
A compact set ${\bf L}$ of linear maps $\rr^n\to\rr^m$ is a {\sl derivate
container} of $\mu$ 
at $\bar q\in Q$ if for every $\varepsilon>0$ there exist a neighborhood
$V_\varepsilon$ of $\bar q$ and a sequence $\mu_j$ of maps of class $C^1$
from $V_\varepsilon$ to $\rr^m$, such that $\mu_j\to\mu$ uniformly on
$V_\varepsilon$ as $j\to\infty$ and the differential $D\mu_j(q)$ satisfies
${\rm dist}(D\mu_j(q),{\bf L})\le\varepsilon$ for every $j$ and every $q\in
V_\varepsilon$.}\proofend
\end{definition}
What makes this concept useful is, to begin with, that
it satisfies the {\sl chain rule}: if $Q_i$ are open in $\rr^{n_i}$
for $i=1,2$, $\mu_i:Q_i\to\rr^{n_{i+1}}$ are Lipschitz continuous,
$\mu_1(Q_1)\subseteq Q_2$, $q_1\in Q_1$, $q_2=\mu_1(q_1)$,
and ${\bf L}_i$ are, for 
$i=1,2$, derivate containers of $\mu_i$ at $q_i$, then 
$$
{\bf L}_2\compo{\bf L}_1\equaldef\{L_2\compo L_1:
L_2\in{\bf L}_2\,,\, L_1\in{\bf L}_1\}
$$
is a derivate container 
of $\mu_2\compo\mu_1$ at $q_1$.\footnote{\baselineskip=7pt 
A somewhat similar rule for the composition of maps 
holds for Clarke generalized Jacobians:
suppose that $Q_1,Q_2,q_1,q_2,\mu_1$, and $\mu_2$ are as above; then the
inclusion 
$\partial(\mu_2\compo\mu_1)(q_1)\subseteq
{\rm co}(\partial\mu_2(q_2)\compo\partial\mu_1(q_1))$ holds.  This, however,
is insufficient for the application to the Maximum Principle, since the
operation of taking convex hull can produce sets that are too large.
The advantage of the derivate containers is that the chain rule holds
{\it without having to take convex hulls}.}

In addition, Warga's derivate containers provide a natural extension to 
flows of Lipschitz vector fields of the differentiation results 
of Section \ref{STRLINED} for flows of vector fields of class $C^1$. 
Indeed, if (CTH.2.Cl) holds, then the reference flow 
$\Phi^{\eta_*}=\{\Phi_{t,s}^{\eta_*}\}_{a\le s\le t\le b}$ consists
of Lipschitz continuous maps, 
each one of which is defined on an open subset
${\rm Dom}(\Phi_{t,s}^{\eta_*})$ of $Q$. Moreover, derivate 
containers\footnote{\baselineskip=7pt We are choosing our language carefully.
Derivate containers are not unique, so one cannot
talk about {\it the} derivate container of a map at a point.   
The expression $D\Phi_{t,s}^{\eta_*}(\xi_*(s))$  does not denote 
``the derivate container of $\Phi_{t,s}^{\eta_*}$ at $\xi_*(s)$.''
It is just the name we choose to give to one particular derivate container
of $\Phi_{t,s}^{\eta_*}$ at $\xi_*(s)$, to emphasize the analogy with the
classical $C^1$ case.  Notice that ``$D\Phi_{t,s}^{\eta_*}(\xi_*(s))$''
should not  be read as  ``$D$ of the map $\Phi_{t,s}^{\eta_*}$ at $\xi_*(s)$,''
since $D\Phi_{t,s}^{\eta_*}(\xi_*(s))$ is not uniquely determined by the
map $\Phi_{t,s}^{\eta_*}$ and the point $\xi_*(s)$, because the same map
could arise from two different flows, and in that case the derivate container
defined here could depend on which flow is used.}
$D\Phi_{t,s}^{\eta_*}(\xi_*(s))$  of the maps $\Phi_{t,s}^{\eta_*}$  
at the points $\xi_*(s)$ can be
computed by  solving a {\sl set of variational equations}. Precisely, 
define ${\bf M}^{\gammast}(t)=\partial_1f(\xi_*(t),\eta_*(t),t)$
for $a\le t\le b$.  If $[a,b]\ni t\to M(t)\in {\bf M}^{\gammast}(t)$
is an arbitrary measurable selection of the set-valued map 
${\bf M}^{\gammast}$, 
we define the {\it linearized flow}  $L_M$ to be the family 
$\{L_M(t,s)\}_{a\le s\le t\le b}$ of linear maps 
from $\rr^n$ to $\rr^n$ such that 
\begin{equation}\label{flafordphilip}
L_M(t,s)=
{\rm identity}+\int_s^t M(\tau)L_M(\tau,s)\,d\tau\,.
\end{equation}
We then define ${\bf{L}}^{\gammast}(t,s)$ to be the set
$\{L_M(t,s):M\in\Gamma({\bf M}^{\gammast})\}$, where $\Gamma({\bf
M}^{\gammast})$ is the set of all measurable selections of ${\bf
M}^{\gammast}$.  It is clear that each ${\bf{L}}^{\gammast}(t,s)$ is compact and
nonempty.  (The compactness follows from the facts that
(i)~$\Gamma({\bf M}^{\gammast})$ is weakly compact in
$L^1([a,b],\rr^n)$, since ${\bf M}^{\gammast}$ is a measurable integrably
bounded set-valued map with compact convex values, and (ii)~the map $M\to
L_M$ is continuous with respect to the weak topology for $M$ and the
topology of uniform convergence for $L_M$.)

Moreover, it is easy to see that the set-valued
maps 
$$(t,s)\to{\bf L}^{\gammast}(t,s)\subseteq\rr^{n\times n}
$$ 
satisfy the identities 
\begin{equation}\label{linflow-a}
{\bf L}^{\gammast}(t,t)=\{{\rm identity}\}\quad{\rm for\quad}t\in[a,b]\,, 
\end{equation}
and 
\begin{equation}\label{linflow-b}
\!\!{\bf L}^{\gammast}(t_3,t_2)\compo{\bf L}^{\gammast}(t_2,t_1)=
{\bf L}^{\gammast}(t_3,t_1)\,\,{\rm whenever}\,\, a\!\le\! t_1\!\le\!t_2
\!\le\!t_3\!\le\!b\,.  
\end{equation}


It was proved by Warga that {\sl ${\bf L}^{\gammast}(t,s)$ is a derivate
container of  $\Phi_{t,s}^{\eta_*}$ at $\xi_*(s)$.}  So we choose
to define
$$
D\Phi_{t,s}^{\eta_*}(\xi_*(s))\equaldef{\bf L}^{\gammast}(t,s)\,.
$$
Moreover, it is easy
to see that  {\sl a map $[a,b]\ni t\to\psi(t)\in\rr_n$ is a solution of the
adjoint equation in the sense of (CTH.4.Cl) if and only if 
it satisfies the ``integrated adjoint equation''
\begin{equation}\label{adjeqintlip}
\psi(t)=\psi(b).L_M(b,t)\quad{\rm for \quad}a\le t\le b 
\end{equation}
for some $M\in\Gamma({\bf M}^{\gammast})$}. (Indeed, if (\ref{adjeqintlip})
holds,  then an argument similar to that used in Section \ref{STRLINED} shows
that  $\dot\psi(t)=-\psi(t).M(t)$ a.e., so (\ref{soladjeqlip}) holds.
Conversely, if (\ref{soladjeqlip}) holds, then one can show that there is a
measurable selection $M\in\Gamma({\bf M}^{\gammast})$ such that
$\dot\psi(t)=-\psi(t).M(t)$ for a.e. $t$, and then 
(\ref{adjeqintlip}) holds.)

Unfortunately, this is not enough yet to carry out the proof of
\S\ref{STRLINED} in the more general setting. The reason for this is 
that {\sl the class of Lipschitz maps is not large enough to contain 
the endpoint maps defined by the needle variations.}  

A class with the desired properties was introduced in Sussmann
[42]-[45]. The maps in this class are
called ``semidifferentiable,'' and their generalized differentials are
called ``semidifferentials.''  The precise definition is quite simple:
\begin{definition}{\rm 
Let $F$ be a set-valued map from $\rr^n$ to $\rr^m$. Let $\bar q\in\rr^n$,
and let $C$ be a closed convex cone in $\rr^n$.  Let ${\bf L}$ be a compact
set of linear maps from $\rr^n$ to $\rr^m$. We say that $F$ is {\em
semidifferentiable at $\bar q$ in the direction of $C$, with
semidifferential ${\bf L}$}---and write 
${\bf L}\in \sd_C(F;\bar q)$---if there exist a compact neighborhood $V$ of
$\bar q$ in $\rr^n$, and a 
Lipschitz map $H:V\to\rr^m$, such that (a)~the restriction of $F$
to $V\cap(\bar q+C)$ is a regular set-valued map 
from $V\cap(\bar q+C)$ to $\rr^m$, (b)~$H$ is a ``first-order
approximation to $F$ in the direction of $C$ near $\bar q$,'' in the sense
that 
$$
\sup\Big\{\|y-H(q)\|:y\in F(q)\Big\}=o(\|q-\bar q\|)\quad
{\rm as}\quad q-\bar q\to_C0\,,
$$
and (c)~${\bf L}$ is a derivate container of $H$ at $\bar q$.}\proofend
\end{definition}

Semidifferentials also satisfy the {\sl chain rule\/}: \quad if 
\begin{itemize}
\item[(a)] $F_i$ are
set-valued maps from $\rr^{n_i}$ to $\rr^{n_{i+1}}$ 
for $i=1,2$, 
\vspace{-2.7mm}
\item[(b)] $C_i$ are closed convex cones in $\rr^{n_i}$, 
\vspace{-2.7mm}
\item[(c)] \mbox{$q_i\in\rr^{n_i}$}, 
\vspace{-2.7mm}
\item[(d)] $F_1((q_1+C_1)\cap U_1)\subseteq q_2+C_2$
for some neighborhood $U_1$ of $q_1$,
\vspace{-2.7mm}
\item[(e)] ${\bf L}_i$ belongs to $\sd_{C_i}(F_i;q_i)$ for $i=1,2$,
\end{itemize}
then ${\bf L}_2\compo{\bf L}_1\in \sd_{C_1}(F_2\compo F_1;q_1)$.

\medbreak
Moreover, it is clear that a Lipschitz-continuous map is semidifferentiable
in the direction of any closed convex cone, and any derivate container 
of the map is also a semidifferential. So Warga's result on derivate
containers of flow maps arising from Lipschitz vector fields 
says in particular that these flow maps are semidifferentiable, and gives us
an explicit construction---if (CTH.2.Cl) holds---of a 
family $\{{\bf L}^{\gammast}(t,s)\}_{a\le s\le t\le b}$ 
of compact sets of linear maps from $\rr^n$ to $\rr^n$ such that 
\begin{equation}\label{islinsemi}
{\bf L}^{\gammast}(t,s)\in \sd_{\rr^n}(\Phi_{t,s}^{\eta_*};\xi_*(s))\quad
{\rm whenever}\quad a\le s\le t\le b\,, 
\end{equation}
and in addition (\ref{linflow-a}) and (\ref{linflow-b}) hold.

It follows trivially from the definition of semidifferentiability
that {\sl if a continuous single-valued or regular set-valued map
$F$ is differentiable at a point $\bar q$ in the classical 
sense in the direction of a cone $C$ with differential $L$,
then  $\{L\}\in \sd_{C}(F;\bar q)$}.  Thus the theory of semidifferentials 
contains both that of derivate containers of Lipschitz continuous maps and 
that of regular classically differentiable maps, and brings them both
into a unified framework where the chain rule holds.   

In addition, it turns out that the semidifferentiable 
analogue of Theorem \ref{theosepclas} holds: 

\begin{theorem}\label{theosepclassemi} 
Let $C_1$ be a closed convex cone in $\rr^m$, let $U$ be a neighborhood of
$0$ in $\rr^m$, let $f:U\cap C_1\to\rr^n$ be a set-valued map, and let
${\bf L}\in \sd_{C_1}(f;0)$.  Let $S$ be a subset of $\rr^n$ having a cone $C_2$
as weakly approximating cone at $f(0)$. Assume that $C_2$ is not a linear
subspace of $\rr^n$. Then a necessary condition for the sets $f(U\cap C_1)$
and $S$ to be locally separated at $f(0)$ is that $(LC_1)^\dagger\cap
(-C_2)^\dagger\neq\{0\}$ for some $L\in{\bf L}$, i.e. that there exist an
$L\in{\bf L}$ and a nonzero covector $\bar p\in\rr_n$ such that $p.L.v_1\le
0$ for all $v_1\in C_1$ and $p.v_2\ge 0$ for all $v_2\in C_2$. \proofend
\end{theorem}
(This is a special case of the more general theorem of 
Sussmann \cite{sussmulti}.) 

Using these results, the proof of Section \ref{STRLINED} carries over to the
current setting. It suffices to observe that, if the endpoint maps
${\cal E}_{{\bf s},{\bf u},\eta_*}$ are defined as in Section \ref{STRLINED}, 
then ${\cal E}_{{\bf s},{\bf u},\eta_*}$ is semidifferentiable at
$(\xi_*(a),0)$ in the direction of $\rr^n\times\rr_+^m$, and one 
member of $\sd_{\rr^n\times\rr_+^m}({\cal E}_{{\bf s},{\bf
u},\eta_*};(\xi_*(a),0))$ is the set
$$
{\bf D}_{{\bf s},{\bf u},\gamma_*}=\{E(L):L\in\Lambda^{\gamma_*}\}\,,
$$
where $\Lambda^{\gamma_*}$ is the set of all ``compatible selections of 
${\bf L}^{\gamma_*}$'' (i.e., the set of all families 
% GNU Original was
% ${\bf L}^{\gamma_*}$ (i.e., the set of all families 
% Not corrected for book. 
$L=\{L(t,s)\}_{a\le s\le t\le b}$ such that 
$L(t,s)\in{\bf L}^{\gamma_*}(t,s)$ for each $s,t$, and 
$L(t_3,t_2)\compo L(t_2,t_1)=
L(t_3,t_1)$ whenever $a\le t_1\le t_2\le t_3\le b$) and,
for $L\in\Lambda$, 
\begin{equation}
E(L)(\delta q,\delta r_1,\ldots,\delta r_m)\equaldef
L(b,a).\delta q+
\sum_{i=1}^m\delta
r_i.L(b,s_i).v^{\gamma_*}(s_i,u_i)\label{diffofendpolip}\,.
\end{equation}
It can be proved that $\Lambda^{\gamma_*}$ coincides with the set
of all maps $L_M^{\gamma_*}$ defined earlier.  Using this,
the proof of the Maximum Principle given in Section \ref{STRLINED} 
applies in this case, and we get the {\sl nonsmooth Maximum Principle 
with the \L{}ojasiewicz  improvement}:
\begin{theorem}\label{maxprinnonsmloja}
Assume that (i)~the classical technical hypotheses (CTH.1),
(CTH.3), (CTH.5), (CTH.6) hold; (ii)~the concept of ``solution of the
adjoint equation'' is interpreted in the sense of (CTH.4.Cl); 
(iii)~the reference vector field $f_{\eta_*}$  satisfies a
Lipschitz-Carath\'eodory condition near $\xi_*$; 
and (iv)~condition (LTH) holds for every $u\in U$. Then the Maximum
Principle (SSMP) is true. \proofend 
\end{theorem}
This result includes the nonsmooth Maximum Principle as a special 
case, but is more general because of the \L{}ojasiewicz improvement. 
(The result of \cite{clarkeminimal} requires the stronger assumption
that $f(q,u,t)$ be Lipschitz  with respect to $q$ for all $u$, $t$.) 

\section{Multidifferentials, Flows, and a General Version of the  Maximum
Principle} 
\label{MULTI}

Proving the nonsmooth version of the maximum principle by classical
needle-variation techniques has several advantages, two of which should
already be apparent from the previous discussion: (i)~the proof is
conceptually close in spirit to the elegant classical idea of geometrizing
the argument by making everything revolve around the construction of a
suitable ``tangent cone'' to the reachable set, and (ii)~the proof yields a
stronger result because of the \L{}ojasiewicz improvement.  

These, however, are only the first two of a long list of reasons that make
it worthwhile to pursue the classical approach.  It suffices to look at the
basic structure of the proof to see that, with practically no change, one
can go much farther and prove a more general theorem.

To begin with, the Lipschitz-Carath\'eodory conditions for the reference
control are only needed to establish the semidifferentiability of the
reference flow $\Phi^{\eta_*}$ and to show that ${\bf L}^{\gamma_*}$ is a
semidifferential of the flow.  Once this is understood, one can do away
with the technical assumptions for the reference vector field and impose
instead the much weaker requirement that the reference trajectory be part
of a general {\sl semidifferentiable flow} $\Phi$.  It is then no longer
necessary to assume that the state space $Q$ is fixed, and we can allow it
to depend on $t$.  The ``reachable set'' will just be a subset ${\cal R}$
of the terminal state space $Q_b$, and the Maximum Principle will give a
necessary condition for ${\cal R}$ to be separated from some other set.
This condition will involve the other basic ingredient of our general
situation, namely, a set ${\cal V}$ of {\sl variations}. The variations should
then give rise to {\sl endpoint maps}, and should be such that these
endpoint maps, when applied starting at $\xi_*(a)$, always yield points
of ${\cal R}$.

Finally, and most importantly, the proofs of Sections \ref{STRLINED} and
\ref{BEYOND} have essentially the same structure, and differ mainly in
the specific differentiation theory they use. This strongly suggests that
everything might work equally well if, instead of the semidifferentials,
we use an even more general differentiation theory, provided only that this
theory has some basic properties, such as the chain rule and a
transversality theorem such as Theorems \ref{theosepclas} and
\ref{theosepclassemi}.   

This suggestion turns out to be true.  At least one such theory, that of
the {\sl multidifferentials}, was introduced in Sussmann \cite{sussmulti}.
Using multidifferentials, one can prove an even more general version of the
maximum principle, which applies to {\sl systems of differential inclusions}
as well as to control systems of the usual kind.

We now make all this precise, and begin by listing the basic definitions. 
\def\lsl#1{\medbreak\noindent {\bf\em #1.\enspace}}

\lsl{Multidifferentials} If $X$, $Y$ are finite-dimensional real linear
spaces,  we write $L(X,Y)$ to denote the set of all linear maps
from $X$ to $Y$. If $X$ is normed, we write
$B_X\equaldef\{x\in X:\|x\|\le 1\}$. 
If $S$ is a convex subset of $X$, ${\bf D}\subseteq
L(X,Y)$, and $k=1,2,\ldots\,,$ we define
$C^k_{{\bf D}}(S,Y)$ to be the set of all maps $H:S\to Y$ that admit an
extension to a $C^k$ map $\tilde H:\Omega\to Y$, defined on an open subset
$\Omega$ of $X$ such that ${\rm Clos}(S)\subseteq\Omega$, and having the
property that 
\begin{equation}\label{c1d} 
D\tilde H(x)\in{\bf D}\,\,\,\,{\rm for\,\,\,\, all}\,\,\,\, x\in S\,.  
\end{equation}
(If ${\bf D}$ is closed in $L(X,Y)$, then of course (\ref{c1d}) is
equivalent to the requirement that $D\tilde H(x)\in{\bf D}$ for all
$x\in{\rm Clos}(S)$.)

\begin{definition}{\rm 
Let $X$, $Y$ be finite-dimensional normed real linear spaces, and let
$X$ be normed.  Let $F:X\to 2^Y$ be a set-valued map. Let
$(\bar{x},\bar{y})$ be a point in $X\times Y$, and let ${\bf D}$ be a
compact subset of $L(X,Y)$. Let  
$C$ be a closed convex cone in $X$.  We say that ${\bf D}$ is a {\em
multidifferential of $F$ at $(\bar x,\bar y)$ in the direction of $C$}, and
write
$$
{\bf D}\in \md_C(F;\bar x,\bar y)\,,
$$
if the following is true:
\medbreak\noindent(MD) for every neighborhood ${\bf D'}$ of 
${\bf D}$ in $L(X,Y)$ there exists a pair $(\rho,\Theta)$ such that
\begin{itemize}\begin{itemize}
\item[(MMD.1)] $\rho>0$, and $\Theta$ is a function on the interval
$]0,\rho]$ with values in $[0,\infty[$ and such that 
$\lim_{s\downarrow 0}\Theta(s)=0$,
\item[(MMD.2)] for every $\varepsilon>0$ there exist  $f,H$  such that
\begin{itemize}
\item[(MMD.2.1)] $f$ is a regular set-valued map
from $\bar x+(C\cap\rho B_X)$ to $Y$;
\item[(MMD.2.2)] ${\rm Graph}(f)\subseteq{\rm Graph}(F)$;
\item[(MMD.2.3)] $H\in C^1_{{\bf D'}}(\bar x+(C\cap\rho B_X),Y)$;
\item[(MMD.2.4)]  $H(\bar x)=\bar y$;
\item[(MMD.2.5)] the inequality  
\begin{equation}\label{issat}
\!\!\!\!\!\!\sup\Big\{|\!|y\!-\!H(x)|\!|:y\!\in\!
f(x)\Big\}\!\le\!\varepsilon\!+\!\Theta(|\!|x\!-\!\bar{x}|\!|)
|\!|x\!-\!\bar x|\!|
\end{equation}
holds whenever   $x-\bar x\in C\cap\rho B_X$. \proofend
\end{itemize}\end{itemize}\end{itemize}}
\end{definition}

\lsl{Time interval} A {\it time interval} is a totally ordered set.  In
what follows, a time interval $I$ will be fixed, and we will use ``$\le$''
to denote the order relation. The reader may assume that $I$ is a
subinterval of $\rr$, but this fact will never be used. We prefer the more
general setting of a totally ordered set both because it is mathematically
more natural and because in several applications this extra generality 
is actually useful. 

\lsl{State bundles} If $I$ is a time interval, a {\it state space 
bundle}---or {\it state bundle}, for short---{\it over $I$} is a family
$Q=\{Q_t\}_{t\in I}$ of nonempty sets, indexed by $t\in I$.  If $J$ is a
subset of $I$, then $J$ is a time-interval (i.~e., a totally ordered set)
as well, so the family $Q\restr J\equaldef\{Q_t\}_{t\in J}$ 
is a state bundle, called the {\sl restriction} of $Q$ to $J$. 

\lsl{Sections} Given a state bundle $Q$ over a time interval $I$, a {\it
section of $Q$} is a map $I\ni t\to\xi(t)\in Q_t$.  {\it No requirement
such as smoothness, or continuity, or even measurability, is made in the
definition of section}. In fact, such a requirement would be meaningless in
this general setting, since the $Q_t$ are just sets, and could be all
different.  We use $\Xi(Q)$ to denote the set of all sections of
$Q$. We should think of the members of $\Xi(Q)$ as ``curves'' on the
time-varying state space $Q_t$, at least in some formal sense.

\lsl{Flows} If $I$ is a time interval, and $Q$ is a state bundle over $I$,
then a {\sl flow} on $Q$ is a family $\Phi=\{\Phi_{t,s}:s,t\in{}I,{}s\le
t\}$ of set-valued maps, such that
\begin{itemize}
{\sl \item[a.] for each $s,t\in I$ such that $s\le t$, 
$\Phi_{t,s}$ is a map from $Q_s$ to subsets of $Q_t$,
\item[b.] for each $t\in I$,  $\Phi_{t,t}$ is the identity map of $Q_t$,
}
\end{itemize}
and 
\begin{itemize}
\item[c.]
{\sl the identity 
\begin{equation}
\Phi_{t_3,t_2}\compo\Phi_{t_2,t_1}=\Phi_{t_3,t_1}
\end{equation}
holds whenever $t_1\le t_2\le t_3$ and the $t_i$ are in $I$.}
\end{itemize}
\xdef\flow{F\!L\!O\!W}
\xdef\var{V\!A\!R}
\xdef\svm{S\!V\!M}
\xdef\lflow{L\!F\!L\!O\!W}
\xdef\lmflow{L\!M\!F\!L\!O\!W}
\xdef\traj{T\!R\!A\!J}
\xdef\iiv{I\!I\!V}
\xdef\cs{C\!S}
We use $\flow(Q)$ to denote the set of all flows on $Q$.

If $\Phi\in \flow(Q)$ and $J$ is a subset of the time interval $I$ 
of $Q$, then the family $\Phi\restr J=\{\Phi_{t,s}\}_{s,t\in{}J,s\le t}$
is a flow on $Q\restr J$, called the {\sl restriction} of $\Phi$ to $J$.

\lsl{Trajectories} A {\sl trajectory} of a flow $\Phi$ on $Q$ is a section
$\xi\in\Xi(Q)$ such that $\xi(t)\in\Phi_{t,s}(\xi(s))$ whenever $s,t\in I$,
$s\le t$.  We write $TRAJ(\Phi)$ to denote the set of all trajectories of
$\Phi$.

\lsl{Variations} If $k\in\nn$, then a {\sl $k$-parameter variation} in a
set $S$ is, simply, a map $\nu$ from the nonnegative orthant
$\rr_+^k$ of $\rr^k$ into $S$.   If
$\nu(0)=s$, $s\in S$, then $s$ is the {\sl base point} of $\nu$, and
$\nu$ is said to be a {\sl ($k$-parameter) variation of $s$ in $S$}.  We
write $\var^k(S)$ to denote the set of all $k$-parameter variations in $S$,
and $\var^k(S;s)$ to denote the set of all $\nu\in \var^k(S)$ whose base
point is $s$.

Given a flow $\Phi$ on a state bundle $Q$ over a time interval $I$,
$a,b\in{}I$, and $a\le b$, a {\sl $k$-parameter variation of $\Phi$ carried
by the interval} $[a,b]$ is a $k$-parameter variation of
$\Phi_{b,a}$ in the set $\svm(Q_a,Q_b)$ of all set-valued maps from $Q_a$ to
$Q_b$.  We use $\var^k(\Phi;a,b)$ to denote the set of all $k$-parameter
variations of $\Phi$ carried by $[a,b]$.

\lsl{The endpoint map of a variation} If $k\in\nn$, $\Phi$, $Q$, $I$,
$a,b$ are as above, and $\nu\in \var^k(\Phi;a,b)$, the {\sl endpoint map}
${\cal E}(\nu;\Phi,a,b)$ is the set-valued map from $Q_a\times\rr_+^k$ 
to $Q_b$ such that
\begin{equation}
{\cal E}(\nu;\Phi,a,b)(q,r_1,\ldots,r_k)=\nu(r_1,\ldots,r_k)(q)
% GNU Original was
% {\cal E}(\nu;\Phi,a,b)(q,r_1,\ldots,r_k)=\nu(r_1,\ldots,r_k)(\Phi_{b,a})(q)
% Not corrected for book. 
\end{equation}
for $q\in Q_a$, $r_1,\ldots,r_k\in[0,\infty[$. 

\lsl{Linear bundles and flows} If $Q$ is a state bundle and the sets $Q_t$
are linear spaces, then $Q$ is called a {\sl linear
bundle}.\footnote{\baselineskip=7pt Throughout this discussion, the field of
scalars is always $\rr$. No other field is ever considered.}  If all the
linear spaces are finite-dimensional then $Q$ is a {\sl finite-dimensional
linear state space bundle.}

Naturally, when $Q$ is a real linear state space bundle we can talk about
{\it linear flows}. We use $\lflow(Q)$ to denote the set of all linear flows
on $Q$. 

\lsl{Linear multiflows} A {\it linear multiflow} on a linear state bundle
$X$ over the time interval $I$
is a family $\Lambda=\{\Lambda_{t,s}:s,t\in I,s\le t\}$ such that each
$\Lambda_{t,s}$ is a nonvoid set of linear maps from $X_s$ to $X_t$, having
the property that $\Lambda_{t,t}=\{{\rm identity}_{X_t}\}$ for each $t$,
and
\begin{equation}\label{compati}
\Lambda_{t_3,t_2}\compo\Lambda_{t_2,t_1}=\Lambda_{t_3,t_1}
\end{equation}
whenever $t_1\le t_2\le t_3$ and the $t_i$ are in $I$.  If $X$ is
finite-dimensional, we call a linear multiflow $\Lambda$ on 
$X$ {\sl compact} if all the sets $\Lambda_{t,s}$ are compact.
We use $\lmflow(X)$, $\lmflow_c(X)$ to denote, respectively, 
the set of all linear multiflows on $X$, and that of all
$\Lambda\in \lmflow(X)$ that are compact.

\lsl{OFD bundles} An {\sl open finite-dimensional (OFD) state space
bundle} is a state bundle $Q$ such that each $Q_t$ is a nonempty open
subset of a finite-dimensional linear space $X_t$.  In that case, the
bundle $X=\{X_t\}_{t\in I}$ is the {\sl tangent bundle} of $Q$.

\lsl{Differentiable flows} If $Q$ is an OFD state bundle over a time
interval $I$, with tangent bundle $X$, we define a flow $\Phi$ to be
{\it{}differentiable} along a trajectory $\xi$ of $\Phi$ if $\Phi_{t,s}$ is
continuous near $\xi(s)$ and differentiable at $\xi(s)$ whenever $s,t\in
I$, $s\le t$. If $\Phi$ is differentiable along $\xi$, then we can define a
flow $D\Phi(\xi)\in \lflow(X)$---called the {\sl differential}, or the
{\sl{}linearization}, of $\Phi$ along $\xi$---by letting
$D\Phi(\xi)_{t,s}$ be the differential $D\Phi_{t,s}(\xi(s))$, so
$D\Phi(\xi)_{t,s}$ is a linear map from $X_s$ to $X_t$.  The fact that
$D\Phi(\xi)$ is a linear flow follows from the chain rule for classical
differentials.

\lsl{Multidifferentials of flows} Let $Q$ be an OFD a state space
bundle over a time interval $I$, and let $X$ be its 
tangent bundle. We then define a {\sl multidifferential of a flow
$\Phi\in \flow(Q)$ along a trajectory $\xi\in TRAJ(\Phi)$} to be a linear
multiflow \mbox{$\Lambda=\{\Lambda_{t,s}\}_{s,t\in I,s\le t}\in \lmflow(X)$}
such that $\Lambda_{t,s}$ belongs to $\md_{X_s}(\Phi_{t,s};\xi(s),\xi(t))$
whenever $s,t\in{}I$, $s\le t$.  (This implies, in particular, that
$\Lambda\in{}\lmflow_c(X)$.)  We use $\md(\Phi;\xi)$ to denote the set of all
multidifferentials of $\Phi$ along $\xi$, and call $\Phi$
{\sl{}multidifferentiable along} $\xi$ if $\md(\Phi;\xi)\neq\emptyset$.
\begin{remark}{\rm 
It is possible to give counterexamples showing that if
each map  $\Phi_{t,s}$ is multidifferentiable at $\xi(s)$, then a family
$\Lambda$ of multidifferentials satisfying the compatibility condition
(\ref{compati}) may fail to exist. For this reason, we impose the
requirement that such a family exist as an extra condition in our
definition of multidifferentiability of a flow along a trajectory.}\proofend
\end{remark}

\lsl{Compatible selections} If $X$ is a finite-dimensional linear state
space bundle over $I$, and $\Lambda$ is a linear multiflow over $X$, a
{\sl{}compatible selection} of $\Lambda$ is a linear flow
$L=\{L_{t,s}:s,t\in I,s\le t\}\in \lflow(X)$ such that
$L_{t,s}\in\Lambda_{t,s}$ whenever $s\le t$ and $s,t\in I$.
We use $\cs(\Lambda)$ to denote the set of all compatible selections
$L$ of $\Lambda$. 

It is not hard to prove, using Zorn's Lemma together with the compactness
of the sets $\Lambda_{t,s}$, that compatible selections always exist if
$\Lambda$ is compact. Moreover, {\sl if $\Lambda$ belongs to $\lmflow_c(X)$,
and $J$ is a subset of $I$, then every compatible selection $L\in
\cs(\Lambda\restr J)$ of the restriction $\Lambda\restr
J=\{\Lambda_{t,s}:s\in J,t\in J,s\le t\}$ can be extended to a compatible
selection $L'\in \cs(\Lambda)$}.

\lsl{Impulse infinitesimal variations} Given a finite-dimensional linear
state bundle $X$ over the time interval $I$, a {\sl $1$-parameter impulse
infinitesimal variation in $X$} is a pair $V=(t,v)$ such that $t\in I$ and
$v\in X_t$. If $a\in I$, $b\in I$ are such that $a\le b$ and $a\le t\le b$,
then $V$ is said to be {\sl carried by the interval $[a,b]$.}  If
$k\in\nn$, then a {\sl $k$-parameter impulse infinitesimal variation in $X$}
is a $k$-tuple $V=(V_1,\ldots,V_k)$ of $1$-parameter infinitesimal
variations in $X$.  If $a\in I$, $b\in I$ are such that each $V_j$ is
carried by $[a,b]$, then we say that $V$ is {\sl carried by $[a,b]$.}  We
use $\iiv^k(X)$ to denote the set of all $k$-parameter impulse
infinitesimal variations in $X$, and $\iiv^k(X;a,b)$ to denote the set
of all $V\in \iiv^k(X)$ that are carried by the interval $[a,b]$.  

Given a flow $\Phi$ on an OFD state bundle $Q$ over the time interval $I$
with tangent bundle $X$, we will also refer to the members of
$\iiv^k(X)$ as {\sl $k$-parameter impulse infinitesimal variations of
$\Phi$}, and we will write $\iiv^k(\Phi)$, $\iiv^k(\Phi;a,b)$ 
instead of $\iiv^k(X)$, $\iiv^k(X;a,b)$. 

\lsl{The infinitesimal endpoint map determined by an impulse infinitesimal
variation} Suppose we are given a compact linear multiflow $X$ on a
finite-dimensional linear state bundle $X$ over the time interval $I$.  Let
$k\in\nn$, and assume that $a,b\in I$, $a\le b$. Let $V\in \iiv^k(X;a,b)$,
and suppose that $V=(V_1,\ldots,V_k)$, $V_j=(t_j,v_j)$ for
$j=1,\ldots,k$.

For each compatible selection $L\!\in\!\cs(\Lambda)$, define a
linear map $E(L;V,a,b)$ from $X_a\times\rr^k$ to $X_b$ by
\begin{equation}
E(L;V,a,b)(x,r_1,\ldots,r_k)=L_{b,a}x+\sum_{j=1}^kr_jL_{b,t_j}v_j\,.
\end{equation}
The set ${\bf E}(\Lambda;V,a,b)=\{E(L;V,a,b):L\in \cs(\Lambda)\}$ is the
{\sl infinitesimal endpoint map associated to $V$ and $\Lambda$}. It is clear
that ${\bf E}(\Lambda;V,a,b)$ is a nonempty compact set of linear maps from
$X_a\times\rr^k$ to $X_b$.  

\lsl{Impulse infinitesimal variations generated by a variation} 
Suppose we are given $I$, $X$, $a$, $b$, $\Lambda$, $k$, 
$V=((t_1,v_1),\ldots,(t_k,v_k))$, $Q$, $\Phi$, $\xi$ as above. 
Let $\nu\in \var^k(\Phi_{b,a})$. We say that $V$ is 
{\sl generated by} $(\nu,\Phi,\xi,\Lambda,a,b)$ if
\begin{equation}
{\bf E}(\Lambda;V,a,b)\in \md_{X_a\times\rr_+^k}({\cal E}(\nu;\Phi,a,b)\,;\,
(\xi(a),0),\xi(b))\,.
\end{equation}

\lsl{Adjoint vectors} Assume that $X$ is a finite-dimensional linear state
bundle over the time interval $I$, and $\Lambda\in \lmflow(X)$.  We then
define a {\sl $\Lambda$-adjoint vector}
to be a map $I\ni t\to\psi(t)\in X_t^{*}$---where $X_t^*$ is the dual space
of $X_t$---having the property  that
\footnote{\baselineskip=7pt Recall that $\Lambda_{t,s}$ 
is a set of linear maps from $X_s$ to $X_t$, and $\psi(t)$ is a linear map
from $X_t$ to $\rr$, so the composite 
$\psi(t)\compo\Lambda_{t,s}=\{\psi(t)\compo L:L\in\Lambda_{t,s}\}$ is well 
defined.  The composite $\psi(t)\compo L$ is the usual 
pullback of the functional $\psi(t):X_t\to\rr$ to a functional
from $X_s$ to $\rr$ via $L$. Often, one uses $L^*$ to denote the 
pullback map, so what we call $\psi(t)\compo L$
would be called $L^*(\psi(t))$ instead, but we prefer to write 
$\psi(t)\compo L$. This notation has the advantage of giving the usual 
matrix formula $\psi(s)=\psi(t).L_{t,s}$ if the $X_t$ are Euclidean spaces,
thought of as spaces 
of column vectors, and then the $X_t^*$ are regarded as spaces of row
vectors.}  
\begin{equation}\label{advectorsemi-a}
\psi(s)\in\psi(t)\compo\Lambda_{t,s}\quad{\rm whenever}\quad s\in I,t\in I, 
s\le t\,.
\end{equation}
An equivalent characterization is the following: 
a {\sl $\Lambda$-adjoint vector} is a map
$I\ni t\to\psi(t)\in X_t^{*}$ such that there is a compatible selection
$L\in \cs(\Lambda)$ for which
\begin{equation}\label{advectorsemi-b}
\psi(s)=\psi(t)\compo L_{t,s}\quad{\rm whenever}\quad s\in I,t\in I, s\le t\,.
\end{equation}
We use $ADJ(\Lambda)$ to denote the set of all $\Lambda$-adjoint
vectors. 

\lsl{Strong admissibility} Let $Q$ be an OFD state bundle with time
interval $I$, and assume $I$ has a minimum $a$ and a maximum $b$. Let
$\Phi\in \flow(Q)$, $\xi\in TRAJ(\Phi)$, $\Lambda\in \md(\Phi;\xi)$.  Let
% GNU Original was
% $\Phi\in \flow(Q)$, $\xi\in TRAJ(\Phi)$, $\Lambda\in \sd(\Phi;\xi)$.  Let
% Not corrected for book. 
${\bf V}\subseteq \iiv^1(\Phi)$ be a finite set of one-parameter
infinitesimal variations of $\Phi$, and let ${\cal R}$ be a subset of
$Q_b$.  We say that ${\bf V}$ is {\sl strongly admissible for 
$({\cal R},\Phi,\xi,\Lambda)$} if:
\begin{itemize} \begin{itemize} 
\item[(SADM)] {\sl There $\,$exist $\,$an $\,$integer $\,k\in\nn$, $\,$a 
$\,$variation $\,\nu\in\var^k(\Phi)$, a $V=(V_1,\ldots,V_k)\in \iiv^k(\Phi)$, 
and an $\bar r>0$, such that
\begin{itemize}
\item[(1)] ${\cal E}(\nu;\Phi,a,b)(\xi(a),r)\!\subseteq\!{\cal R}$
whenever $r\!=\!(r_1,\ldots,r_k)\!\in\![0,\bar r]^k$,
\item[(2)] $V$ is generated by $(\nu,\Phi,\xi,\Lambda,a,b)$,
\end{itemize}
and
\begin{itemize}
\item[(3)]${\bf V}\subseteq\{V_j:j=1,\ldots,k\}$.
\end{itemize}
}
\end{itemize}\end{itemize} 

\xdef\adj{A\!D\!J}
\lsl{Convergence of impulse infinitesimal variations} Given a time interval
$I$ having a minimum $a$ and a maximum $b$, a linear finite-dimensional
state bundle $X$ over $I$, and a compact linear multiflow $\Lambda$ on 
$I$, we can define a topology ${\cal T}(X,\Lambda)$ on $\iiv^1(X)$ as
follows. We pick, in an arbitrary fashion, a compact neighborhood $K$ of $0$
in $X_b^*$. We then let $\adj(\Lambda;K)$ denote the set of all adjoint
vectors $\psi\in \adj(\Lambda)$ such that $\psi(b)\in K$, equipped with the
topology of pointwise convergence.  Then $\adj(\Lambda;K)$ is a topological
subspace of the product space $\Pi_{t\in I}\Lambda_{b,t}^*(K)$, which is
compact by Tikhonov's theorem. It is easy to prove that $\adj(\Lambda;K)$ is
itself compact.  Each $V=(t,v)\in \iiv^1(X)$ gives rise to a continuous map
$\mu_V: \adj(\Lambda;K)\to\rr$, defined by letting $\mu_V(\psi)=\psi(t).v$.
Then ${\cal T}(\Lambda,X)$ is the topology of uniform convergence of the
functions $\mu_V$ on $\adj(\Lambda;K)$.  (This topology is easily seen to be
independent of the choice of $K$. Notice, however, that ${\cal T}(\Lambda,X)$ 
need not be Hausdorff, since it may happen, for example, that
$\Lambda_{b,t}=\{0\}$ for some $t<b$ but ${\rm dim}(X_t)>0$, in which case,
if $V=(t,v)$, with $v\in X_t$ and $v\neq 0$, and $V'=(t,0)$, then the
functions $\mu_V$ and $\mu_{V'}$ will coincide, even though $V\neq V'$.)

\lsl{Admissibility} Let $Q$ be an OFD state bundle with time
interval $I$ and tangent bundle $X$. Assume that $I$ has a minimum $a$ and
a maximum $b$. Let $\Phi\in \flow(Q)$, $\xi\in TRAJ(\Phi)$, 
$\Lambda\in \md(\Phi;\xi)$. Let ${\bf V}$ be a set of $1$-parameter
% GNU Original was
$\Lambda\in \sd(\Phi;\xi)$. Let ${\bf V}$ be a set of $1$-parameter
% Not corrected for book. 
impulse infinitesimal variations of $\Phi$. Let ${\cal R}$ be a subset
of $Q_b$.  We call ${\bf V}$ {\sl admissible for}
% GNU Original was
% of $Q_b$.  We call $V$ {\sl admissible for}
% Not corrected for book. 
$({\cal R},\Phi,\xi,\Lambda)$ if  every finite subset of ${\bf V}$ can be
% GNU Original was
% $({\cal R},\Phi,\xi,\Lambda)$ if  every finite subset of $V$ can be
% Not corrected for book. 
approximated in the topology ${\cal T}(\Lambda,x)$ of $\iiv^1(\Phi)$ by 
strongly admissible finite subsets of $\iiv^1(\Phi)$.  The precise meaning
of this condition is as follows: 
\begin{itemize}\begin{itemize} 
\item[(ADM)] {\sl Let $K$ be a fixed compact neighborhood of 
$0$ in $X_b$. Then for every finite subset ${\bf W}$ of ${\bf V}$ and every
$\varepsilon>0$ there exists a finite subset ${\bf W}_{\varepsilon}$ of
$\iiv^1(\Phi)$ which is strongly admissible for
$({\cal{}R},\Phi,\xi,\Lambda)$ and such that
\begin{equation}
\max_{W\in{\bf W}}\,\,\,\min_{W'\in{\bf W}_{\varepsilon}}
\,\,\max_{\psi\in \adj(\Lambda;K)}
|\mu_{W}(\psi)-\mu_{W'}(\psi)|\le\varepsilon\,.
\end{equation}
}
\end{itemize}\end{itemize} 
(It is easy to see that the validity of (ADM) does not depend on the choice
of the compact set $K$.) 

\medbreak We are now ready to state the {\sl flows, variations, and
multidifferentials version of the Maximum Principle}:

\begin{theorem}\label{sfinal}
Let $I$ be a time interval having a minimum $a$ and a maximum $b$.  Let $Q$
be an OFD state bundle over $I$, and let $X$ be its tangent bundle.  Let
$\Phi\in \flow(Q)$, $\xi\in TRAJ(\Phi)$, $\Lambda\in \md(\Phi;\xi)$.  Let
${\cal R}$, $S$ be subsets of $Q_b$.  Let $C$ be a closed convex cone in
$X_b$ which is a weakly approximating cone to $S$ at $\xi(b)$ and is not a
linear subspace of $X_b$. Let ${\bf V}$ be a subset of $\iiv^1(\Phi)$ which
is admissible for $({\cal R},\Phi,\xi,\Lambda)$.  Assume that 
${\cal R}$ and $S$ are locally separated at $\xi(b)$. Then 
there exists a $\psi\in \adj(\Lambda)$ such that $\psi(b)\neq 0$, 
$-\psi(b)\in C^{\dagger}$ and $\psi(t).v\le 0$ for all
$(t,v)\in{\bf V}$. \proofend
\end{theorem}

This theorem contains as special cases all the versions stated earlier,
but is much more general in a number of ways. For example: 

\medbreak\noindent{{\bf 1. Non-Lipschitz and discontinuous reference vector
fields.}} For control systems of the classical form $\dot q=f(q,u,t)$,
Theorem \ref{sfinal} applies in many cases when the time-varying vector
field $f_{\eta_*}$ arising from the reference control $\eta_*$ fails to
be Lipschitz or even continuous with respect to $q$.  Two simple
illustrations of this are the {\sl reflected brachystochrone problem} and
the {\sl light refraction problem}.  
\begin{example}{\bf\em (The reflected brachystochrone problem.)}
{\rm We consider 
the minimum-time optimal control problem in $\rr^2$---with coordinates
$x$, $y$---whose dynamical law is $\dot x=u\sqrt{|y|}$, $\dot
y=v\sqrt{|y|}$, the controls $u$, $v$ being subject to the constraint
$u^2+v^2\le 1$.  (It is easy to reformulate the problem as one involving a
fixed interval such as $I=[0,1]$.  It suffices to rewrite the dynamical
equations as $\dot x=wu\sqrt{|y|}$, $\dot y=wv\sqrt{|y|}$ and seek to
minimize $\int_0^1w(t)dt$, where $w$ is a new control with values in
$[0,\infty[$.)  In this case, the right-hand side of the dynamical
equations is continuous but not Lipschitz continuous. Moreover, if one
wants to join optimally a point $A$ in the upper half-plane to a point $B$
in the lower half-plane, then the optimal trajectory will obviously go from
$A$ to a point $C$ in the $x$-axis and then from $C$ to $B$.  The parts
from $A$ to $C$ and from $C$ to $B$ are solutions of the classical
brachystochrone problem, so they are arcs of cycloids. This still leaves
one free parameter, namely the choice of $C$. For a given $C$, the
corresponding curve $\xi_C$ and control $\eta_C$ are such that the
time-varying vector field $f_{\eta_C}$ is not Lipschitz continuous with
respect to the state. This means that we cannot even apply the nonsmooth
maximum principle with \L{}ojasiewicz modification to the pair
$(\xi_C,\eta_C)$, because the reference vector field is not Lipschitz.  It
turns out that Theorem \ref{sfinal} does apply, because the flow determined
by $\eta_C$ is multidifferentiable---and, in fact, differentiable in the
classical sense---along $\xi_C$, provided that we restrict it to the
totally ordered set $I-\{\tau\}$, where $\tau$ is the unique point such
that $\xi_C(\tau)$ lies in the $x$ axis.  

An application of  Theorem \ref{sfinal} gives the extra junction condition.
Precisely,  the cycloids satisfy a 
differential equation $(y')^2|y|=K-|y|$. The extra conditions is that the
constants $K_+$ and $K_-$ corresponding to the two arcs of cycloid that
make up $\xi_C$ be equal.} \proofend
\end{example}

\begin{example}\label{refra}
 {\bf\em (The refraction problem.)} {\rm This is similar to the 
reflected brachystochrone problem, except that the dynamical law is now
$||\dot q||=c_+$ if $y>0$, $||\dot q||=c_-$ if $y<0$, where $c_-$, $c_+$
are positive constants such that $c_-\neq c_+$. Now the dynamical law has a
{\sl discontinuous} right-hand side.  Once again, Theorem \ref{sfinal}
applies.  The answer is, of course, that of the well known Snell law of
refraction, but this time the derivation of the result does not require an
extra {\sl ad hoc} argument and fits directly within the framework of the
necessary conditions for optimality.} \proofend
\end{example}
\begin{remark}
{\rm Example \ref{refra} is also covered by the theory of ``optimal 
multiprocesses'' of Clarke and Vinter, cf. \cite{clarkevin1}.} \proofend
\end{remark}

\medbreak\noindent{{\bf 2.\ Stronger conclusions for nonsmooth problems.}}
Even in situations where the nonsmooth version applies, it can happen that
Theorem \ref{sfinal} gives strictly stronger results, as the following
example shows.
\begin{example}\label{withsine} {\bf\em (A $\,$problem $\,$with 
$\,$Lipschitz-continuous $\,$data, $\,$for which the usual
nonsmooth maximum principle does not give the best possible result.)}
{\rm  We
consider the one-dimensional optimal control problem in which 
the dynamical law is 
\begin{equation}\label{eqsine}
\dot x=u+x^2\sin{\frac{1}{x}}\,,
\end{equation}
%GNU Change next line
%the control constraint is $|u|\le 1$, and the objective is 
the control constraint is $u\in\{-1,0,1\}$, and the objective is 
to minimize the integral $\int_0^1e^tu(t)dt$ subject to the  
endpoint constraints $x(0)=x(1)=0$.  Suppose we want to test the
trajectory $[0,1]\ni t\to x(t)=0$ and corresponding control $u(t)\equiv 0$
for optimality. The nonsmooth maximum principle will give the adjoint
equation $\dot\psi\in[-|\psi|,|\psi|]$, one of whose solutions is
$\tilde\psi(t)=e^t$. The Hamiltonian (assuming that $\psi_0=1$, which is
easily justified for this problem) is $H=\psi.u+
\psi.x^2\sin{\frac{1}{x}}-e^tu$. So the Hamiltonian maximization condition
is satisfied for $\tilde\psi$. Therefore the nonsmooth maximum principle
does {\sl not} exclude the zero trajectory as a candidate for optimality.
On the other hand, Theorem \ref{sfinal} does, because the reference flow
$\Phi$ is also differentiable in the classical sense, since
$\Phi_{t,s}(x)=x+o(|x|)$ as $x\to 0$ for each $t,s$, as can easily be seen,
for example, using Gronwall's inequality.  So, using the classical
differential in the role of $\Lambda$, we get the adjoint equation
$\dot\psi=0$, $\,$which $\,$is {\sl $\,$not} $\,$satisfied $\,$by 
$\,$any $\,$Hamiltonian-maximizing
% GNU Original was
% $\dot\psi=0$, which is {\sl not} satisfied by any Hamiltonian maximizing
% Not corrected for book. 
$\,$choice $\,$of $\,$$\psi$.} \proofend
\end{example}
The preceding example shows the advantage of a theory with a nonunique
``generalized differential'':  our version of 
the maximum principle gives a nontrivial
Hamiltonian-maximizing adjoint vector for {\sl every} multidifferential
$\Lambda\in \md(\Phi,\xi_*)$; when the conditions of the nonsmooth maximum
principle (NSMP) hold, the usual statement of the NSMP only gives an adjoint
vector corresponding to one of these multidifferentials, whereas our
version allows for the possibility of selecting other multidifferentials 
that may yield stronger conclusions. 

\begin{example} {\bf\em (A non-Lipschitz analogue of Example
\ref{withsine}.)} {\rm It is easy to modify Example \ref{withsine} to get a problem where the 
conditions of the maximum principle are not satisfied but those of our
Theorem \ref{sfinal} are.   It suffices to substitute for $x^2$
in (\ref{eqsine}) a factor of the form $|x|^\alpha$, where 
$1<\alpha<2$.  Then the right-hand side of the dynamical equation is not
Lipschitz continuous, but Theorem \ref{sfinal} still applies.} \proofend
\end{example}

\medbreak\noindent{{\bf 3.\ High-order variations.}}  Theorem \ref{sfinal}
is also stronger, for a different reason, in situations where the dynamics
is very smooth, and the classical version applies.  This is because the
theorem uses a general, abstract definition of ``variation,'' so that the
usual needle variations are special cases, but other, ``high-order''
variations, expressed in terms of Lie brackets of the vector fields of the
system under consideration, are also allowed.  (For some simple examples of
such variations, cf.  Kawski and Sussmann \cite{withkawski}.  For a more
general discussion of high-order conditions, cf.  Gabasov-Kirillova
\cite{gabakir}, Kelley, Kopp and Moyer \cite{kelley}, Knobloch
\cite{knobloch}, Krener \cite{krener}.)

\medbreak\noindent{{\bf 4. Jumps.}} Theorem \ref{sfinal} allows {\sl jump}
maps. For example, it could happen that at some time $\bar t$ the state
jumps from $q$ to $J(q)$, where $J$ is a map. It could even happen that the
state spaces $Q_{\bar t_-}$, $Q_{\bar t_+}$ before and after the jump are
different.  (To include such jumps in our framework it is convenient to
change the time-interval $I$ and treat $\bar t_-$ and $\bar t_+$ as
different points, which, incidentally, is one of several reasons why it is
preferable to 
work with general totally ordered sets rather than ordinary intervals.)  If
the jump time $\bar t$ and the map $J$ are fixed in advance, and $J$ is
multidifferentiable at $(\xi_*(\bar t_-),\xi(\bar t_+))$ with
multidifferential ${\bf D}$, then the corresponding condition on the
adjoint covector $\psi$ is $\psi(\bar t_-)\in\psi(\bar t_+)\compo{\bf D}$.
If $\bar t$ and $J$ are not fixed, then one gets extra inequality 
conditions involving the variations of the pair $(\bar t,J)$. 

\medbreak\noindent{{\bf 5. Differential inclusions.}} Our result
also applies to {\sl systems of differential inclusions}.  For example, the
following is a special case of Theorem \ref{sfinal}.

\begin{theorem}\label{diffeq}
Let $I=[a,b]$ be a compact subinterval of $\rr$. Let $n\in\nn$, let $Q$ be
an open subset of $\rr^n$, and let ${\bf F}=\{F_\alpha:\alpha\in A\}$ be a
family of set-valued maps
$Q\times I\ni(q,t)\to{}F_\alpha(q,t)\subseteq\rr^n$, such that each
$F_\alpha$ is almost lower semicontinuous and locally integrably lower
bounded.  Let $\alpha_*\in A$, and let $\xi_*:[a,b]\to Q$ be a trajectory
of $F_{\alpha_*}$ such that $F_{\alpha_*}$ is integrably pseudo-Lipschitz
along $\xi_*$.  Let ${\cal{R}}\subseteq Q$ be such that, whenever
$k\in\nn$, $a\le\tau_1<\ldots<\tau_k<b$, and $\alpha_1,\ldots,\alpha_k\in
A$, there exists an $\bar r>0$ such that, if
$(r_1,\ldots,r_k)\in[0,\bar r]^k$, then  every solution $\xi$ of the
inclusion
$$
\dot\xi(t)\in \sum_{j=1}^k\chi_{_{[\tau_j,\tau_j+r]}}(t)F_{\alpha_j}(\xi(t),t)
+\biggl(1-\sum_{j=1}^k\chi_{_{[\tau_j,\tau_j+r]}}(t)\biggr)F_{\alpha_*}(\xi(t),t)
$$
for which $\xi(a)=\xi_*(a)$ satisfies $\xi(b)\in{\cal R}$. 
Let $S\subseteq Q$, and let $C$ be a closed convex cone in
$\rr^n$ which is a weakly approximating cone to $S$ at $\xi_*(b)$ and is not a
linear subspace of $\rr^n$. Assume that 
${\cal R}$ and $S$ are locally separated at $\xi_*(b)$. Then:
\begin{itemize}
%GNU: new line: 
\item[(i)] 
%GNU: two new lines:
If $\lim_{\delta\downarrow 0}\sup\Big\{{\rm dist}(\dot\xi_*(t),
F_{\alpha_*}(x,t)):a\le t\le b,|\!|x-\xi_*(t)|\!|\le\delta\Big\}=0$,
then there exists a time-varying vector field
$(q,t)\to f(q,t)$, defined on some tube
${\cal T}(\xi_*,\varepsilon)$ (cf. Eq. (\ref{tubedef})), that 
% GNU Original was
% ${\cal T}(\xi_*,\varepsilon)$ (cf. Eq. (\ref{tubedef}), that 
% Not corrected for book. 
satisfies a Lipschitz-Cara\-th\'eo\-dory condition near $\xi_*$ and 
is such that $\dot\xi(t)=f(\xi(t),t)$
for almost every $t$, and $f(q,t)\in{\rm co}(F(q,t))$ for 
all $(q,t)\in{\cal T}(\xi_*,\varepsilon)$, 
\item[(ii)] For every $f$ that satisfies the conditions of 
(i) there exists an absolutely continuous solution 
$\psi:[a,b]\to\rr_n$ of the differential inclusion
$-\dot\psi(t)\in\psi(t)\cdot\partial f_t(\xi_*(t))$---where 
$f_t(q)\equaldef f(q,t)$---such that $\psi(b)\neq 0$,
$-\psi(b)\in C^\dagger$, and, for every $\alpha\in A$, 
$$
\psi(t).\dot\xi_*(t)\ge \max\{\psi(t).v:v\in F_\alpha(\xi_*(t),t)\}
$$
for almost all $t\in I$. \proofend
\end{itemize}
\end{theorem}
(The definitions of ``almost lower semicontinuous,'' ``locally integrably
lower bounded,'' and ``integrably pseudo-Lipschitz'' can be found, for
example, in Sussmann \cite{susskobe}.) 

\begin{remark}{\rm Theorem \ref{diffeq} gives a necessary 
condition for separation {\sl in terms of the adjoint
equation for a selection of the convexified inclusion.}  So this condition is
also the necessary condition for the relaxed problem. The observation that
the condition arising from the relaxed problem is also necessary for the
nonrelaxed one is known as the ``Kaskosz maximum principle'' (cf. Kaskosz 
\cite{kaskosz}, Warga \cite{wargae}).}\proofend
\end{remark} 
\begin{remark}{\rm  A result similar to
Theorem \ref{diffeq} but for a single Lipschitz nonconvex inclusion 
was proved by Q.~J.~Zhu in \cite{zhu} and by H.~D.~Tuan in
\cite{tuan}.}\proofend
\end{remark} 

\medbreak\noindent{{\bf 6. Hybrid problems.}} Theorem \ref{sfinal} applies
to problems that are ``hybrid'' in the sense that the technical assumptions
$T_j$ of various existing necessary conditions for optimality (such as the
classical smooth maximum principle with high-order conditions, the
nonsmooth maximum principle, and the maximum principle for differential
inclusions) hold near the reference trajectory on various subintervals
$I_j$ of its time interval $I$, but none of these technical conditions
holds all the time.  In this case, the other theorems will give for each
$j$ an adjoint vector $\psi_j$ defined on $I_j$ and satisfying the
conclusions $C_j$ of the theorem that applies on $I_j$, but it will not
follow that these results can be combined into one that yields the
existence of a global $\psi$ satisfying $C_j$ on $I_j$ for each $j$.  
\begin{example}{\bf\em (High-order conditions for a problem whose dynamics 
is not everywhere smooth.)} 
{\rm 
Consider a minimum-time problem with dynamical law 
$$\dot q=f(q)+ug(q)$$ 
and control constraint $|u|\le 1$. The high-order maximum principle gives an
extra inequality---the Legendre-Clebsch condition---valid when $f$ and
$g$ are sufficiently smooth, e.g. of class $C^3$.  On the other hand, if
$f$ and $g$ are only Lipschitz continuous, the nonsmooth maximum principle
applies. It is natural to ask what happens if $f$ and $g$ are smooth in
some open region $Q_S$ of the state space, but just Lipschitz continuous in
some other open region $Q_L$. Suppose $\xi:[a,b]\to Q_S\cup Q_L$ is a
trajectory that is not entirely contained in either of the two regions and
corresponds to a control $\eta:[a,b]\to[-1,1]$.  A reasonable conjecture is
that, if $\xi$ is time-optimal, then there will exist a nontrivial field of
covectors $\psi$ that satisfies the conclusion of the nonsmooth principle
while $\xi(t)$ is in $Q_L$, and those of the smooth high-order principle
when $\xi(t)\in{}Q_S$. In other words, {\sl $\psi$ will satisfy the adjoint
differential inclusion and the Hamiltonian maximization condition on the
whole interval $[a,b]$, and in addition the Legendre-Clebsch condition
$$
\langle\psi(t),[g,[f,g]](\xi(t))\rangle\ge 0
$$ at every time $t$ such that
$|\eta(t)|<1$, $t$ is a Lebesgue point of $\eta$, and $f$ and $g$ are of
class $C^3$ near $\xi(t)$.} (For details of how to construct the 
variation that gives rise to the impulse infinitesimal variation
$(t,-[g,[f,g]](\xi(t)))$, cf. \cite{withkawski}.)

Neither the classical high-order maximum principle nor the nonsmooth
maximum principle can be used  to prove that the conjecture is 
true, but Theorem \ref{sfinal} applies.} \proofend
\end{example}

\section{Three Ways to Make the Maximum Principle Intrinsic on Manifolds}
\label{INTRI}
%\bigskip

So far, the state spaces have been open subsets of Euclidean spaces, and
the formulations of the Maximum Principle have depended strongly on this 
fact. Naturally, it is desirable to have a formulation that works
intrinsically on manifolds. This is essentially equivalent to having 
a formulation in Euclidean spaces that is invariant under arbitrary
nonlinear changes of coordinates.  Such a formulation would make it
possible, even when we are working in $\rr^n$, to follow
Lagrange's strategy of expressing ``the equations of every problem in the
most simple form relative to each set of variables,''
and seeing ``beforehand which variables one should use in order to
facilitate the  integration as much as possible.''   

It turns out that this can be done in several ways, all of which are useful
and enhance the power of the Maximum Principle as a technical tool. 

Let us first consider, for simplicity, the classical maximum principle. 
Since this case is discussed in great detail in Sussmann \cite{sussfree}, 
we will only review it briefly.   

Although the adjoint equation
$-\dot\psi(t)=\frac{\partial{H}}{\partial{q}}(\xi_*(t),\eta_*(t),\psi(t),t)$
is not 
manifestly invariant under nonlinear coordinate changes, it is quite easy
to prove its invariance, as we now show.     

\medbreak\noindent{{\bf 1.$\,$The Hamiltonian formulation.}}
One way to prove invariance of the adjoint equation
is to use the canonical symplectic structure of the
cotangent bundle $T^*Q$ of a manifold $Q$. This structure gives rise to a
canonical fiber-preserving isomorphism $J:T^*(T^*Q)\to T(T^*Q)$, which
enables us to asign to every covector $\omega\in T_z^*(T^*Q)$ a tangent
vector $J_z(\omega)\in T_z(T^*Q)$.  In particular, if $H:T^*Q\to\rr$ is a
function, and $z\in T^*Q$ is a point where $H$ is differentiable, then
there is a well defined tangent vector $\vec{H}(z)\in T_zT^*Q$, given by
$\vec{H}(z)=J_z(dH(z))$.  

Now suppose that we are given a control system
$\dot q=f(q,u,t)$ on a manifold $Q$, and a reference trajectory-control 
pair $\gamma_*=(\xi_*,\eta_*)$ such that $f_{\eta_*}$ satisfies  
a $C^1$-Carath\'eodory condition near $\xi_*$. For each $u,t$, 
define 
$${\cal H}_{u,t}(q,p)\equaldef H^{\Sigma}(q,u,p,t)\,(\equaldef
p.f(q,u,t))\,.
$$
Then the function ${\cal H}_{\eta_*(t),t}$ is of class $C^1$ 
for almost every $t$. The system 
\begin{eqnarray}
\dot\xi(t)&=&f(\xi(t),\eta_*(t),t)\,, \label{ham1}\\
-\dot\psi(t)&=&\psi(t).\frac{\partial f}{\partial q}(\xi(t),\eta_*(t),t)
\label{ham2}
\end{eqnarray}
is then easily seen to be equivalent to 
\begin{equation}
\dot\zeta(t)=\vec{{\cal H}}_{\eta_*(t),t}(\zeta(t))\,,
\end{equation}
if we write $\zeta(t)=(\xi(t),\psi(t))$.   So the system
(\ref{ham1},\ref{ham2}) is
clearly intrinsically defined in a coordinate-free way.  

At this point, it is worth noticing that, if $\Sigma=(Q,U,I,{\cal U},f)$ is 
a control system on a manifold $Q$, such that for every $\eta\in{\cal U}$
the corresponding vector field $f_{\eta}$ satisfies a $C^1$-Carath\'eodory 
condition, then we can define 
$$f^*(z,u,t)=\vec{{\cal H}}_{u,t}(z)\,,\quad{\rm for}\quad
(z,u,t)\in T^*Q\times U\times I\,, 
$$
and obtain a new control system $\Sigma^*=(T^*Q,U,I,{\cal U},f^*)$ 
with state space $T^*Q$, called the {\it Hamiltonian lift} of $\Sigma$. 
Then the necessary condition given by the Maximum Principle can be viewed
as stating that {\bf\em the reference trajectory-control pair 
$\gamma_*=(\xi_*,\eta_*)$ must be the projection of a 
reference trajectory-control pair $\Gamma_*=(\Xi_*,\eta_*)$ of the
Hamiltonian lift such that $\Gamma_*$ has some special properties}, namely,
nontriviality, 
Hamiltonian maximization, and the transversality condition. This seemingly
trivial remark has turned out to be useful because, once it is understood
that the Maximum Principle relates certain trajectories of a control system
$\Sigma$ to special trajectories of $\Sigma^*$, then it is at least
conceivable in principle that this construction may be iterated.   

We now present an example---based on Sussmann
\cite{sussiber,sussathens,susscrac}---showing how the idea of iterating
Hamiltonian lifts can be used to prove a ``weak regularity theorem'' for
trajectories:

\begin{example}\label{examregu}
{\rm The following result was announced in \cite{sussiber} for the case
when $U$ is an interval and in \cite{sussathens} for the general case. A
detailed proof appears in \cite{susscrac}:

\begin{theorem}\label{regularity}
Let $Q$ be a real-analytic manifold, let $U$ be a compact subanalytic
subset of a real-analytic manifold $\hat U$, and let $f:Q\times U\to TQ$ be
an analytic map such that $f(q,u)\in T_qQ$ for every $q\in Q$, $u\in U$.
Let $\xi:[a,b]\to Q$ be an absolutely continuous curve that satisfies the
equation $\dot\xi(t)=f(\xi(t),\eta(t))$ a.e. for some measurable control
$\eta:[a,b]\to U$.  Then there exists a trajectory-control pair
$(\xi^{\#},\eta^{\#})$, defined on some interval $[a^{\#},b^{\#}]$, such
that $\xi^{\#}(a^{\#})=\xi(a)$, $\xi^{\#}(b^{\#})=\xi(b)$, and $\eta^{\#}$
is real-analytic on an open dense subset of $[a^{\#},b^{\#}]$.
\end{theorem}
Theorem \ref{regularity} says, roughly, that for an analytic system,
whenever a point $\bar q$ can be steered to some other point $\hat q$ by
means of some measurable control it follows that this can also be done
using a control that has an extra regularity property.  This theorem can be
applied to optimal control problems of the general form (FEOCP)---as long
as $f$ and $L$ do not depend on $t$ and are analytic with respect to $q$
and $u$---by considering an augmented system in which the running cost is
added as a new state variable. In that case, Theorem \ref{regularity}
implies that, {\sl if $\bar q$ can be steered to $\hat q$ with cost $c$ by
means of some measurable control, then $\bar q$ can be steered to $\hat q$
with cost $c$ by means of a control that is real-analytic on an open dense
subset of its domain.}  In particular, we can take $c$ to be the optimal
cost, and conclude---under the analyticity hypothesis---that if (FEOCP)
has a solution then it has a solution that is real-analytic on an open
dense subset of its domain. For problems having a unique solution, this
result gives a {\sl regularity property} of the solution.


The proof of Theorem \ref{regularity} is quite long, and uses highly
nontrivial facts about real-analytic maps, such as resolution of
singularities. Without going into any details, we just single out one
noteworthy fact: {\sl the proof uses in a crucial way the idea of iterating
the Hamiltonian lifting.}  The way this idea enters the proof is roughly as
follows.  If $\xi(b)$ is an interior point of the reachable set ${\cal R}$
from $\xi(a)$, then it follows from general results about real-analytic
systems that $\xi(b)$ is reachable from $\xi(a)$ by means of a piecewise
constant control.\footnote{\baselineskip=7pt It is not necessarily true
that $\xi(b)$ is reachable from $\xi(a)$ by means of a piecewise constant
control {\sl in the same time} $b-a$.}
If $\xi(b)$ belongs to the boundary of ${\cal R}$, then the
Maximum Principle says that $\xi$ is the projection of a ``special''
trajectory $\Xi$ of the lifted Hamiltonian system. What makes $\Xi$ special
is the Hamiltonian maximization condition, according to which $\eta(t)$
cannot be an arbitrary point of $U$, but must belong to $U_{\rm{max}}(\Xi(t))$,
where}
$$
U_{\rm{max}}(q,p)=\Big\{u\in U:p.f(q,u)=\max\{p.f(q,v):v\in U\}\Big\}\,.
$$
{\rm 
So $\Xi$ is a trajectory of a system $\Sigma^{*,\rm{max}}$ which is of the same
kind as $\Sigma$, but has a ``smaller'' control space.  The trouble with
this is that the new control space $U_{\rm{max}}(q,p)$ depends on the point
$(q,p)$ in the state space $T^*Q$ of the new system, so $\Sigma^{*,\rm{max}}$ is
not the kind of control system to which the Maximum Principle can be
applied. At this point, the machinery of real-analytic stratifications and
desingularization turns out to be helpful, by enabling us to construct a
partition ${\cal P}$ of $T^*Q$ into connected embedded real-analytic
submanifolds such that (a)~for every $t$ in an open dense subset $J$ of
$[a,b]$, the restriction of the curve $\Xi$ to a neighborhood $V(t)$ of
$t$ is entirely contained in one of the members of ${\cal P}$, and
(b)~for every $P\in{\cal P}$, the set 
$U_{{\rm max},P}(z)=\{u\in U_{\rm{max}}(z):\vec{{\cal H}}_u(z)\in T_zP\}$
is ``the same for all $z\in P$'' and is ``smaller than $U$.'' 
The expressions  ``the same'' and ``smaller'' are, of course, vague,
but they can be given a precise definition, using desingularization theory.
It then turns out that locally, on an open  dense subset of $[a,b]$,
the pair $(\Xi,\eta)$ is ``exactly like'' the pair $(\xi,\eta)$, but with a
smaller control set.   One can then iterate the construction, and keep
making the control set smaller and smaller. Moreover, this process can be
proved to terminate in a finite number of steps, and Theorem 
\ref{regularity} follows.} \proofend
\end{example}

\medbreak\noindent{{\bf 2.$\,$Connections along curves.}}
A second way to reformulate the adjoint equation invariantly is to use {\sl
connections along curves}, as explained in Sussmann \cite{sussfree}.
Precisely, recall that, if $Q$ is a smooth manifold, then a {\sl vector
field along a curve} $\xi:[a,b]\to Q$ is a map $[a,b]\ni t\to \theta(t)\in
T_{\xi(t)}Q$ that assigns to each $t\in[a,b]$ a tangent vector $\theta(t)$
to $Q$ at $\xi(t)$.  (The definition of a {\sl field of covectors} is
similar, except that in this case the value at each $t$ must belong to the
cotangent space $T_{\xi(t)}^*Q$.)  
\begin{definition}{\rm 
Let $Q$ be a smooth manifold. A {\em connection along an absolutely
continuous curve} $\xi:[a,b]\to Q$ is a map $\nabla$ that assigns to every
absolutely continuous vector field $\theta$ along $\xi$ an integrable
vector field $\nabla\theta$ along $\xi$, in such a way that
$\nabla(\theta_1+\theta_2)=\nabla(\theta_1)+\nabla(\theta_2)$ for all
$\theta_1,\theta_2$, and $\nabla(\varphi\theta)=\varphi\nabla\theta
+\dot\varphi\theta$ whenever $\theta$ is an absolutely continuous vector
field along $\xi$ and $\varphi:[a,b]\to\rr$ is an absolutely continuous
function.}\proofend
\end{definition}
It is easy to see that, if $\xi$ is an integral curve of a
time-varying vector field $(q,t)\to f(q,t)$ that satisfies a
$C^1$-Carath\'eodory condition near $\xi$, then the pair $(\xi,f)$ gives
rise in a canonical way to a connection $\nabla_{\xi,f}$ along $\xi$,
characterized by the fact that, if $X$ is a smooth vector field on $Q$, and
$X\compo\xi$ is the vector field along $\xi$ defined by
$(X\compo\xi)(t)=X(\xi(t))$, then
$$
\nabla_{\xi,f}(X\compo\xi)(t)=[f_t,X](\xi(t))
$$
for $t\in[a,b]$, where $f_t$ is the vector field $q\to f_t(q)\equaldef
f(q,t)$. 

A connection along a curve induces an operator of ``parallel translation''
along the curve.  Precisely, if $\xi:[a,b]\to Q$ is a curve and $\nabla$ is a
% GNU Original was
% along a curve.  Precisely, if $\xi:[a,b]\to Q$ is a curve and $\nabla$ is a
% Not corrected for book. 
connection along $\xi$, then a vector field $\theta$ along $\xi$ is 
$\nabla$-{\it parallel} if $\nabla\theta\equiv 0$.   It is easy to see that 
if $s\in[a,b]$ and $v\in T_{\xi(s)}Q$, then there is a unique 
vector field $\theta$ along $\xi$ which is $\nabla$-parallel and satisfies
$\theta(s)=v$.   If we define $\tau_{\xi,\nabla,t,s}(v)=\theta(t)$, then 
the map $\tau_{\xi,\nabla,t,s}$ is a linear isomorphism from $T_{\xi(s)}Q$ to 
$T_{\xi(t)}Q$ for each $s,t\in[a,b]$. The maps 
$\tau_{\xi,\nabla,t,s}$ satisfy the obvious flow identities
(i.e., $\tau_{\xi,\nabla,t,t}={\rm identity}$, and
$\tau_{\xi,\nabla,t_1,t_2}\compo\tau_{\xi,\nabla,t_2,t_3}=
\tau_{\xi,\nabla,t_1,t_3}$).   

Although a connection $\nabla$ along a curve $\xi$ is defined as a
differential operator on vector fields along $\xi$,  such an object induces
in a standard way differential operators on fields of covectors and on
fields of higher-order tensors. If $\theta$ is a field of
vectors and $\omega$ is a field of covectors along $\xi$, then the
``product rule''
$$
\frac{d\,\,}{dt}\Big(\omega(t).\theta(t)\Big)=
(\nabla\omega)(t).\theta(t)+\omega(t).(\nabla\theta)(t)
$$
holds. In particular, if $\theta$ and $\omega$ are parallel (i.e. 
if $\nabla\omega\equiv 0$ and $\nabla\theta\equiv 0$) then the function
$t\to \omega(t).\theta(t)$ is a constant.   

If $\xi$ is an integral curve of a time-varying vector field 
$f$, and $\nabla=\nabla_{\xi,f}$, then the equations
$\nabla\theta=0$ and $\nabla\omega=0$ are, respectively, the {\sl
variational equation} and the {\sl adjoint equation} along $(\xi,f)$.
In the setting of the maximum principle,
when $f$ is the vector field $f_{\eta_*}$ 
corresponding to the reference control $\eta_*$, the
adjoint equation that appears in the maximum principle is precisely 
the equation 
$$
\nabla_{\xi_*,f_{\eta_*}}\psi\equiv 0\,.
$$

We now give a few examples---borrowed from Sussmann \cite{sussfree}, where
the reader can find a much more detailed discussion---to illustrate the
advantages of the connection point of view.
\begin{example}\label{exageo}
{\bf\em (The geodesic equation for Riemannian metrics.)}
{\rm The maximum principle, formulated in terms of connections along
curves, can be used to give a very elegant, completely coordinate-free
derivation of the geodesic equation for Riemannian metrics that elucidates
the special role of the Levi-Civita connection without ever having to write
formulas for the Christoffel symbols. 

Suppose $Q$ is a Riemannian manifold. Let $U$ be the set of all smooth
vector fields $u$ on $M$ such that $||u(q)||\le 1$ for all $q$. Let
$\Sigma$ be the control system $\dot q=f(q,u)$, $u\in U$, where
$f(q,u)\equaldef u(q)$, and the class of admissible controls is the 
set ${\cal U}$ of all maps $\eta:[a_{\eta},b_{\eta}]\to U$ such that the map
$(q,t)\to f(q,\eta(t))$ satisfies a $C^1$-Carath\'eodory condition on every
compact subset of $Q$.   Then the minimum-time trajectories of $\Sigma$ are
% GNU Original was
% Let ${\cal U}$ be the set of all maps $\eta:[a_{\eta},b_{\eta}]\to U$ such
% that the map $(q,t)\to f(q,\eta(t))$ satisfies a $C^1$-Carath\'eodory
% condition on every compact subset of $Q$.  
% Let $\Sigma$ be the control system $\dot q=f(q,u)$,
% $u\in U$, where $f(q,u)\equaldef u(q)$, and the class of admissible
% controls is ${\cal U}$.  Then the minimum-time trajectories of $\Sigma$ are
% Not corrected for book. 
exactly the length-minimizers parametrized by arc-length.  Let $\xi$ be a
minimum-time trajectory of $\Sigma$, and let $\xi$ satisfy
$\dot\xi(t)=f(\xi(t),\eta(t))$ for some control $\eta$.
Let $\psi$ be a field of covectors given by the maximum principle. 
Then the Hamiltonian maximization condition implies that
$\psi=\rho.G(\dot\xi)$, 
where $G$ is the metric, regarded as a vector bundle homomorphism 
from $TQ$ to $T^*Q$, and $\rho$ is a positive function.  The constancy of
the minimized Hamiltonian implies that $\rho$ is constant, so 
we may assume that $\rho\equiv 1$. The adjoint equation
says that 
$$\nabla_{\xi,f_{\eta}}\psi\equiv 0\,.$$
Let $\nabla^{LC}$ be the
Levi-Civita connection. Then, if $X$ is any smooth vector field on $Q$,
we have 
\begin{eqnarray*}
&\,\,&\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!\!
\Big\langle(\nabla_{\xi,f_{\eta}}\psi)(t),X(\xi(t))\Big\rangle\\
&=&\frac{d\,\,}{dt}\Big\langle\psi(t),X(\xi(t))\Big\rangle-
\Big\langle\psi(t),\Big(\nabla_{\xi,f_{\eta}}(X\compo\xi)\Big)(t)\Big\rangle\\
&=&
\frac{d\,\,}{dt}\Big\langle\psi(t),X(\xi(t))\Big\rangle-
\Big\langle\psi(t),[f_{\eta(t)},X](\xi(t))\Big\rangle\\
&=&
\frac{d\,\,}{dt}\Big\langle\psi(t),X(\xi(t))\Big\rangle-
\Big\langle\psi(t),(\nabla_{f_{\eta(t)}}^{LC}X-\nabla_X^{LC}{f_{\eta(t)}})
(\xi(t))\Big\rangle\,,
\end{eqnarray*}
using the fact that $\nabla^{LC}$ is torsion-free. 

For each $t$, the function 
$q\to ||f_{\eta(t)}(q)||^2$ has a local maximum at the point $q=\xi(t)$, 
so its derivative in any direction vanishes at $\xi(t)$. But
$$X.||f_{\eta(t)}||^2=2\Big\langle
G(f_{\eta(t)}),\nabla_X^{LC}f_{\eta(t)}\Big\rangle\,,$$
since $\nabla^{LC}G=0$. So 
$\Big\langle \psi(t),(\nabla_X^{LC}f_{\eta(t)})(\xi(t))\Big\rangle=0$.
Then 
\begin{eqnarray*}
\Big\langle\nabla_{\xi,f_{\eta}}\psi(t),X(\xi(t))\Big\rangle&=&
\frac{d\,\,}{dt}\Big\langle\psi(t),X(\xi(t))\Big\rangle-
\Big\langle\psi(t),(\nabla_{f_{\eta(t)}}^{LC}X)(\xi(t))\Big\rangle\\
&=&\Big\langle\nabla_{f_{\eta}}^{LC}\psi(t),X(\xi(t))\Big\rangle\,.
\end{eqnarray*}
Since $X$ is arbitrary, we conclude that 
$\nabla_{\xi,f_{\eta}}\psi\equiv \nabla_{f_{\eta}}^{LC}\psi$.
Therefore the adjoint equation  $\nabla_{\xi,f_{\eta}}\psi=0$ implies
$(\nabla_{f_{\eta(t)}}^{LC}\psi)(t)=0$ for all $t$. 

Since $\psi=G(\dot\xi)$ and $\nabla^{LC}G=0$, we get
$\nabla_{f_{\eta(t)}}^{LC}\dot\xi\equiv 0$, i.e.,
$\nabla_{\dot\xi(t)}^{LC}\dot\xi\equiv 0$, which is the usual geodesic
equation.} \proofend
\end{example}

\begin{example}\label{coord-dem} {\bf\em (How the adjoint equation treats all
vector fields equally.)} {\rm 
The adjoint equation $\nabla_{\xi_*,f_{\eta_*}}\psi=0$ implies, for any smooth vector
field  $X$ on $Q$, the equation 
\begin{equation}\label{adjeqvecf}
\frac{d\,\,}{dt}\Big(\psi(t).X(\xi_*(t))\Big)=
\psi(t).[f_{\eta_*(t),t},X](\xi_*(t))\,,
\end{equation} 
where we write $f_{u,t}(q)\equaldef f(q,u,t)$.  In fact, it is easy to
prove that {\sl the adjoint equation holds if and only if (\ref{adjeqvecf}) 
holds for all smooth vector fields $X$.}  The usual way of writing 
the adjoint equation in coordinates corresponds to singling out 
some special vector fields $X$, namely, the coordinate vector fields. 
Indeed, if $q\to (x^1(q),\ldots,x^n(q))$ is a chart, and we write
% GNU Original was
% (Indeed, if $q\to (x^1(q),\ldots,x^n(q))$ is a chart, and we write
% Not corrected for book. 
$\psi(t)=\sum_j\psi_j(t)dx^j$, $f(q,u,t)=\sum_jf^j(q,u,t)\partial_j$, where
$\partial_j$ is the vector 
field usually called $\frac{\partial\,\,\,\,}{\partial x^j}$, then 
$\psi(t).\partial_j=\psi_j(t)$, and 
$$
[f_{\eta_*(t),t},\partial_j](\xi_*(t))=-
\sum_i\partial_jf^i(\xi_*(t),\eta_*(t),t)\partial_i\,,
$$
so (\ref{adjeqvecf}), for $X=\partial_j$,  says that 
\begin{equation}\label{adjeqcoor}
\dot\psi_j=-\sum_i\psi_i.\partial_jf^i(\xi_*(t),\eta_*(t),t)\,.
\end{equation}
Naturally, the usual form of the adjoint equation says that
(\ref{adjeqcoor}) holds for $j=1,\ldots,n$.   Comparing with 
(\ref{adjeqvecf}), we see that 
{\bf\em (\ref{adjeqvecf}) says that something happens for all 
vector fields, whereas (\ref{adjeqcoor}) says that the same thing happens
for the special vector fields arising from a coordinate chart.}  In other
words,  (\ref{adjeqvecf}) treats all vector fields equally, whereas 
(\ref{adjeqcoor}) singles out some vector fields attached to a chart.

Paraphrasing the statement by Lagrange quoted earlier, one may expect that 
the more general formulation (\ref{adjeqvecf})   might make it possible 
to 
\begin{center}\begin{minipage}{99mm}\baselineskip=9pt\noindent 
\noindent express the equations of every optimal control problem in the
most simple form relative to each set of variables and enable us
to see beforehand which vector fields one should use in order to facilitate
the analysis as much as possible.
\end{minipage}\end{center}

It may happen that, to analyze a particular problem, the functions
that play the most important role are the ``momentum functions'' 
$\psi.X_i$ arising from certain vector fields $X_i$ that are not the 
$\partial_j$ corresponding to a given chart. (Even more strongly, if the
$X_i$ do not commute then they cannot arise as coordinate vector fields
of {\sl any} chart.) }\proofend
\end{example}

\begin{example}{\bf\em (A simple example of trajectory analysis using Lie
brackets and the intrinsic Maximum Principle.)} {\rm 
In Sussmann \cite{sussfree}, we discuss many examples of how the 
qualitative analysis of the optimal trajectories becomes much simpler
by using (\ref{adjeqvecf}).  The simplest of them is given by the proof 
of the following theorem:
\begin{theorem}
Consider a control system $\Sigma:\,\,\,\dot q=f(q)+ug(q)$ in an open 
subset $Q$ of $\rr^2$, with control constraint $|u|\le 1$. 
Suppose that 
\begin{itemize}
\item[(I)] $g(q)$ and $[f,g](q)$ are linearly independent at every 
$q\in Q$. 
\end{itemize}
Then every time-optimal trajectory of $\Sigma$ is bang-bang with
finitely many switchings. If, in addition, 
\begin{itemize}
\item[(II)] $f(q)$ and $g(q)$ are linearly independent at every 
$q\in Q$, 
\end{itemize}
then every time-optimal trajectory of $\Sigma$ is bang-bang with at most
one switching.
\end{theorem}
\noindent {\bf\em Proof}.  Let $\xi:[a,b]\to Q$ be time-optimal, and let
$\psi:[a,b]\to\rr_2$ be an adjoint vector given by the Maximum Principle. 
Let 
$$\varphi_1(t)=\psi(t).g(\xi(t))\,,\quad\varphi_2(t)=\psi(t).[f,g](\xi(t))\,,
\quad\varphi_3(t)=\psi(t).f(\xi(t))\,.$$
Then (\ref{adjeqvecf}) tells us that 
$\dot\varphi_1=\varphi_2$, and (I) implies that 
$\varphi_1$ and $\varphi_2$ never vanish simultaneously. So 
all zeros of $\varphi_1$ are isolated, and then $\xi$ is bang-bang, as
stated. Now suppose that (II) holds as well. Then 
we can write $[f,g]=\alpha.f+\beta.g$, where $\alpha$ and $\beta$ are
smooth functions on $Q$.   Condition (II) implies that
$\alpha$ never vanishes along
$\xi$, so the function $t\to\alpha(\xi(t))$ has constant sign. Clearly,
$\varphi_2=\alpha.\varphi_3+\beta.\varphi_1$ along $\xi$. Suppose 
$\varphi_1$ vanishes more than once. Let $t_1$, $t_2$ be two
consecutive zeros. Then
$\dot\varphi_1(t_1).\dot\varphi_1(t_2)<0$.
So $\varphi_2(t_1).\varphi_2(t_2)<0$. But 
$$
\varphi_2(t_1).\varphi_2(t_2)=
\alpha(\xi(t_1)).\alpha(\xi(t_2)).\varphi_3(t_1).\varphi_3(t_2)\,.
$$
Since $\alpha$ has constant sign along $\xi$, we find 
$\varphi_3(t_1).\varphi_3(t_2)<0$. On the other hand, 
the Hamiltonian $H$ is equal to 
$\varphi_3+u\varphi_1$, and this function must be constant along
$(\xi,\psi)$.  In particular, this implies that 
$\varphi_3(t_1)=\varphi_3(t_2)$, so 
$\varphi_3(t_1).\varphi_3(t_2)\ge 0$. This contradiction 
proves our statement.}\proofend
\end{example}

\medbreak\noindent{{\bf 3.$\,$The intrinsic form of the general ``flows,
variations, and multidifferentials'' version of the Maximum Principle.}} So
far, we have concentrated on seeking intrinsic formulations of the classical
smooth Maximum Principle. The next natural step would be to write
intrinsic formulations of Clarke's nonsmooth principle, and other versions
that generalize the classical one.  

This is not hard to do.  For example,
Clarke's nonsmooth principle can be formulated in Hamiltonian form, by
observing that the Clarke generalized gradient $\partial h(z)$ of a
Lipschitz-continuous function $h:T^*Q\to\rr$ at a point $z\in T^*Q$ is a 
well-defined nonempty compact convex subset of $T_zT^*Q$, and then
rewriting the 
combined system (\ref{ham1},\ref{ham2}), which now becomes a differential
inclusion, as $\dot\zeta(t)\in J\Big(\partial{{\cal
H}}_{\eta_*(t),t}(\zeta(t))\Big)$.  Alternatively, we can use the
connections approach, and observe that the set $\Gamma({\bf M}^{\gammast})$
now becomes a set of connections along $\xi_*$, rather than a single
connection as in the classical case.

It is better, however, to jump directly to the most general version of the 
Maximum Principle, namely, Theorem \ref{sfinal}.  It turns out that 
in this case very little work needs to be done, because {\sl Theorem 
\ref{sfinal} is already almost intrinsic as it stands.}  To render it
completely intrinsic, it suffices to observe that 
\begin{itemize}
\item[1.] The concept of a {\sl multidifferential} at a point $(q,q')\in
Q\times Q'$ of a set-valued map $F:Q\to 2^{Q'}$ between manifolds of class 
$C^1$ is well defined, and the multidifferentials 
${\bf D}\in \md(F;q,q')$ are nonempty compact subsets of $L(T_qQ,T_{q'}Q')$,
the space of linear maps from $T_qQ$ to $T_{q'}Q'$. (To see this, choose
charts $x:U\to\rr^n$, $x':U'\to\rr^{n'}$ of $Q$, $Q'$, defined on
neighborhoods $U$, $U'$ of $q$, $q'$. Then, if we let $G=x'\compo F
\compo x^{-1}$, it follows that $G$ is a set-valued map from $\rr^n$ to
$\rr^{n'}$, 
and $\md(G;x(q),x'(q'))$ is a well-defined set of nonempty compact 
subsets  of $L(\rr^n,\rr^{n'})$. Then 
$$
\md(F;q,q')\equaldef\Big\{Dx'(q')\compo{\bf D}\compo(Dx(q))^{-1}:{\bf
D}\!\in\! \md(G;x(q),x'(q'))\Big\}
$$
\item[]is clearly a well defined set of nonempty compact subsets of 
the space $L(T_qQ,T_{q'}Q')$. Moreover, the Chain Rule for multidifferentials 
implies that $\md(F;q,q')$ is independent of the choice of the charts 
$x$, $x'$.)  More generally, if $C$ is a closed convex cone 
in $\rr^m$, then  the concept of a multidifferential at
$((q,0),q')$ of a set-valued map from $Q\times C$ to subsets of $Q'$
% GNU Original was
% $(q,0),q'$ of a set-valued map from $Q\times C$ to subsets of $Q'$
% Not corrected for book. 
is well defined. 
\item[2.] There is an obvious way to define what is meant by
a ``$C^1$-manifold 
bundle'' $Q=\{Q_t\}_{t\in I}$ over a time interval $I$, by just requiring
the $Q_t$ to be manifolds of class $C^1$. 
\item[3.] If  $Q=\{Q_t\}_{t\in I}$ is a $C^1$-manifold bundle, 
$\Phi$ is a flow on $Q$, and $\xi\in TRAJ(\Phi)$, then the {\sl tangent
bundle 
of $Q$ along $\xi$} is the linear space bundle 
\begin{equation}
T_{\xi}Q\,\,\equaldef\,\,\{T_{\xi(t)}Q_t\}_{t\in I}\,.
\end{equation}
Then the concept of a  {\sl multidifferential of $\Phi$ along $\xi$}
is well defined, and we can write $\md(\Phi;\xi)$ to denote the set of all
such multidifferentials.  The members of  $\md(\Phi;\xi)$ are compact
linear multiflows on the linear bundle $T_{\xi}Q$.   
\item[4.] The concepts of variation, impulse infinitesimal variation,
infinitesimal endpoint map, strong admissibility, admissibility, and
adjoint vector, 
carry over with no change to this more general setting.
\item[5.] The concept of a ``weakly approximating cone'' $C\subseteq 
T_qQ$ to a subset $S$ of a $C^1$ manifold $Q$ at point $q\in Q$ makes sense
intrinsically.   
\end{itemize}
We then get the following {\bf\em intrinsic verson of the general Maximum
Principle}: 

\begin{theorem}\label{sfinalmanif}
Let $I$ be a time interval having a minimum $a$ and a maximum $b$.  Assume
that $Q$ is a $C^1$-manifold state space bundle over $I$ and $\Phi$ is a 
flow on $Q$. Let 
$\xi\in TRAJ(\Phi)$, and let $X$ be the tangent bundle $T_{\xi}Q$, so
$X_t=T_{\xi(t)}Q_t$ for $t\in I$. Let $\Lambda\in \md(\Phi;\xi)$ be a
multidifferential of $\Phi$ along $\xi$.  Let ${\cal R}$, $S$ be subsets of
$Q_b$.  Let $C$ be a closed convex cone in $X_b$ which is a weakly
approximating cone to $S$ at $\xi(b)$ and is not a linear subspace of
$X_b$. Let ${\bf V}$ be a subset of $\iiv^1(\Phi)$ which is admissible for
$({\cal R},\Phi,\xi,\Lambda)$.  Assume that ${\cal R}$ and $S$ are locally
separated at $\xi(b)$. Then there exists a $\psi\in \adj(\Lambda)$ such that
$\psi(b)\neq 0$, $-\psi(b)\in C^{\dagger}$ and $\psi(t).v\le 0$ for all
$(t,v)\in{\bf V}$. \proofend
\end{theorem}

\section{Conclusion}
\label{CONCL}
%\bigskip

Optimal control, in its earlier form known as the calculus of variations,
has been the driving force behind some of the most important developments that
have taken place in geometry.  The systematic study of curves that minimize
``length'' turned out to be the key that unlocked the door opening up 
to modern differential geometry.  This study was made possible 
by the availability of techniques invented by the practitioners
of the calculus of variations for the analysis of even more general curve 
optimization problems.  Besides, it was in the context of the calculus 
of variations that the first important discoveries were made of 
results that are ``invariant under arbitrary nonlinear coordinate
changes,'' leading to the general notions of covariance and invariance
introduced in the nineteenth century. 

After the calculus of variations expanded its scope in the 1950s and took
on the new name of optimal control theory, it benefitted in many ways
from the incorporation of geometric insights. The necessary conditions for
optimality have been geometrized in two ways, namely, (a)~by reformulating
the maximum principle as a separation result for a reachable set, and
(b)~by making it intrinsically invariant on manifolds. The combined effect
of these two geometrizations, coupled with 
\begin{itemize}
\item[1.] the use of results from homotopy
theory (Leray-Schauder \cite{leray}, Browder \cite{browder}) to get a
stronger form of the transversality condition, 
\item[2.] the adaptation of the
needle-variations method of \cite{pontrybook} to a broader setting,
involving a theory of generalized differentials obtained by extending 
Warga's theory of derivate containers (Warga
\cite{wargaa}-\cite{wargad}), 
\item[3.] the use of selection
theorems for almost lower semicontinuous inclusions,
based on extending the
ideas of Bressan \cite{bressanondif}-\cite{bressandir}, 
\end{itemize}
and, finally,
\begin{itemize}
\item[4.] the use of uniform approximation theorems for relaxed
trajectories of pseudo-Lipschitz differential inclusions, based on extending
the ideas of Fryszkowski and coworkers \cite{colomboetal,fry},
\end{itemize}
has led to the formulation of a  nonsmooth,
intrinsic maximum principle of great generality and power. 

Much work remains to be done.  For example, the ideas of the previous
sections should be extended to problems with state space constraints.
Also, a more systematic analysis is needed of the high-order variations
that can occur. Finally, the precise relation between our version of the
maximum principle and other recent nonsmooth versions (e.g. Ioffe
\cite{ioffe}, Ioffe-Rockafellar \cite{iofferock}, Loewen-Rockafellar
\cite{loewenrock1,loewenrock2}, Rockafellar \cite{rocka}) is not yet
completely clear, and it is not known whether these other versions can be
combined with ours in a truly unified framework.


%%%%%%%%%%%%%%%bibl.tex%%%%%%%%%%%%%%%
\begin{thebibliography}{ZZ}

\baselineskip=9.5pt


{\mynine


\bibitem{agra1}{Agrachev, A. A. and R. V. Gamkrelidze,
``A second-order optimality principle for a time-optimal problem,''
{\mynineit Math. Sbornik} {\myninebf 100}, 142 (1976).}

\vspace{-2.6mm}\bibitem{agra2}{Agrachev, A. A. and R. V. Gamkrelidze,
``The exponential representation of flows and the chronological
calculus,''  {\mynineit Math. Sbornik} {\myninebf 109}, 149 (1978).}


\vspace{-2.6mm}\bibitem{boltyanskii}{Boltyanskii, V. G., {\mynineit Gomotopiceskaya teoriya
nepreryvnyh otobrazenii i vektornyh polei} (Russian). [Homotopy theory of
continuous mappings and of vector fields.] Trudy
Matem. Insttituta im. V.A. Steklova, Vol. XLVII. Izdat. Akad. Nauk SSSR,
Moscow, 1955.}

\vspace{-2.6mm}\bibitem{berkovitz}{Berkovitz, L. D., {\mynineit Optimal Control Theory},
Springer-Verlag, New York, 1974.}

\vspace{-2.6mm}\bibitem{bressanondif}{Bressan,~A., ``On differential relations
with lower continuous right-hand side. An existence theorem.''
{\mynineit J. Diff. Equations} {\myninebf 37} (1980), pp.~89-97.}

\vspace{-2.6mm}\bibitem{bressansolu}{Bressan,~A., ``Solutions of 
lower semicontinuous differential inclusions on closed sets,''
{\mynineit Rend. Sem. Mat. Univ. Padova} {\myninebf 69} (1983), pp.~99-107.}

\vspace{-2.6mm}\bibitem{bressandir}{Bressan,~A., ``Directionally continuous
selections and differential inclusions,'' 
{\mynineit Funk. Ekvac.} {\myninebf 31} (1988), pp.~459-470.}

\vspace{-2.6mm}\bibitem{bromilsus}{Brockett,~R.~W.,   R.~S.~Millman and H.~J.~Sussmann
(eds.), \ {\mynineit Differential Geometric Control Theory}, Progress in
Mathematics No. 27, Birkh\"auser, Boston, 1983.}

\vspace{-2.6mm}\bibitem{browder}{Browder,~F.~E., ``On the fixed point index for
continuous mappings of locally connected spaces,'' {\mynineit Summa Brazil. Math.}
{\myninebf 4} (1960), pp.~253-293.}

\vspace{-2.6mm}\bibitem{clarkeminimal}{Clarke, F.~H., ``The Maximum Principle 
under minimal hypotheses,'' {\mynineit S.I.A.M. J. Control Optim.} {\myninebf 14}
(1976), pp.~1078-1091.}

\vspace{-2.6mm}\bibitem{clarkebook}{Clarke, F.~H., {\mynineit Optimization and Nonsmooth
Analysis,} Wiley-Interscience, New York, 1983.}

\vspace{-2.6mm}\bibitem{clarkevin1}{Clarke, F.~ and R.~B.~ Vinter, ``Optimal
multiprocesses,'' {\mynineit SIAM J. Control Optim.} {\myninebf 27} (1989),
pp.~1072-1091.}

\vspace{-2.6mm}\bibitem{colomboetal}{Colombo,~R.M., A.~Fryszkowski,
T.~Rze$\check{{\rm z}}$uchowski, and V.~Staicu,~``Continuous selections of
solution sets of Lipschitzean differential inclusions,''  {\mynineit Funkcialaj
Ekvacioj} {\myninebf 34} (1991), pp.~321-330.}

\vspace{-2.6mm}\bibitem{docarmo}{Do Carmo, M., {\mynineit Riemannian Geometry}, Birkh\"auser,
Boston, 1992.}

%\vspace{-2.6mm}\bibitem{elliottcons}  {Elliott,~ D.~L., "A consequence of
%controllability,'' {\mynineit J. Diff. Equations} {\myninebf 10} (1971), pp. 364-370.}

\vspace{-2.6mm}\bibitem{amsbook}{Ferreyra, G., H.~Hermes, R.~ Gardner and H.~J.~Sussmann
(eds.) {\mynineit Differential-Geometric Control}, 
Proceedings of the July 1997  Summer Research Institute of the American
Mathematical Society (AMS), held in Boulder, CO, June-July 1997.
To appear in the AMS {\mynineit Symposia on Pure Mathematics} series.}

\vspace{-2.6mm}\bibitem{fry}{Fryszkowski,~A., and T.~Rze$\check{{\rm z}}$uchowski, 
``Continuous version
of the Filippov-Wa$\check{{\rm z}}$ewski relaxation theorem,'' {\mynineit J.~
Diff.  Equations} {\myninebf 94} (1993), pp.~254-265.}

\vspace{-2.6mm}\bibitem{gabakir}{Gabasov, V., and F. M. Kirillova,
``High-order necessary conditions for optimality,''
{\mynineit SIAM J. Control} {\myninebf 10} (1972), pp.~127-168.}

\vspace{-2.6mm}\bibitem{ioffe}{Ioffe, A., ``Euler-Lagrange and Hamiltonian formalisms for
dynamic optimization,'' {\mynineit Trans. Amer. Math. Society} {\myninebf 349} (1997),
pp.~2871-2900.}

\vspace{-2.6mm}\bibitem{iofferock}{Ioffe, A.~D. and R.~T. Rockafellar,
The Euler and Weierstrass conditions for nonsmooth variational problems,''
{\mynineit Calc. Var. Partial Differential Equations} {\myninebf 4}
(1996), pp.~59-87.}

\vspace{-2.6mm}\bibitem{isidori}{Isidori,~A., {\mynineit Nonlinear Control Systems}
Springer-Verlag, Berlin, 1995.}

\vspace{-2.6mm}\bibitem{jakures}{Jakubczyk, B., and W.~Respondek (eds.),
{\mynineit Geometry of Feedback and Optimal Control,} Marcel Dekker, New York,
1997.}

\vspace{-2.6mm}\bibitem{vel}{Jurdjevic, V., {\mynineit Geometric Control
Theory}, Cambridge University Press, 1997.}

\vspace{-2.6mm}\bibitem{kaskosz}{B. Kaskosz, ``A maximum principle in relaxed controls,''
{\mynineit Nonlinear Analysis, TMA} {\myninebf 14} (1990), pp.~357-367.}

\vspace{-2.6mm}\bibitem{kaskoloja}{Kaskosz,~B.,  and S.~\L{}ojasiewicz~ Jr.,
``A Maximum Principle for generalized control systems,'' {\mynineit
Nonlinear Anal. TMA} {\myninebf 9} (1985), pp.~109-130.}

\vspace{-2.6mm}\bibitem{withkawski}{Kawski, M., and H J. Sussmann, ``Noncommutative power
series and formal Lie-algebraic techniques in nonlinear control theory.''
In: {\mynineit Operators, Systems and Linear Algebra: Three Ddecades of Algebraic
Systems Theory,}\/ proceedings of the workshop held in Kaiserslautern,
Germany, Sept. 24-26, 1997, in honor of Professor Paul Fuhrmann; U.~Helmke,
D.~Praetzel-Wolters, E.~Zerz (eds.); B.~G.~Teubner Stuttgart, 1997,
pp~111-129.} 

\vspace{-2.6mm}\bibitem{kelley}{Kelley, H. J., R. E. Kopp and H. G. Moyer, ``Singular
extremals,'' in {\mynineit Topics in Optimization}, G. Leitman ed., Academic
Press, New York, 1967, pp.~63-101.}

\vspace{-2.6mm}\bibitem{knobloch}{Knobloch, H. W., {\mynineit High-Order Necessary Conditions in
Optimal Control}, Springer-Verlag, New York, 1975.}

\vspace{-2.6mm}\bibitem{krener}{Krener, A. J.,
``The higher order maximum principle and its application
to singular extremals,''
{\mynineit SIAM J. Control Opt.} {\myninebf 15} (1977), pp.~256-293.}

\vspace{-2.6mm}\bibitem{leray}{Leray, J., and J. Schauder, ``Topologie et \'equations
fonctionnelles,'' {\mynineit Ann. Sci. \'Ecole Norm. Sup.} {\myninebf 51} (1934),
pp.~45-78.}

\vspace{-2.6mm}\bibitem{loewenrock1}{Loewen, P.~D., and R.~T. Rockafellar,
``Optimal control of unbounded differential inclusions,''
{\mynineit SIAM J. Control and Optimization} {\myninebf 32} (1994), pp.~442-470.}

\vspace{-2.6mm}\bibitem{loewenrock2}{Loewen, P.~D., and R.~T. Rockafellar,
``New necessary conditions for the generalized problem of Bolza,''
{\mynineit SIAM J. Control and Optimization} {\myninebf 34} (1996), pp.~1496-1511.}

\vspace{-2.6mm}\bibitem{mordusus}{Mordukhovich,~B., and H.~J.~Sussmann (eds.), 
{\mynineit Nonsmooth Analysis and Geometric Methods in
Deterministic Optimal Control}, Springer-Verlag, New York, 1996.}

\vspace{-2.6mm}\bibitem{nijmeijer}{Nijmeijer, H., and A. van der Schaft, 
{\mynineit Nonlinear Dynamical Control Systems}, Springer-Verlag, 
New York, 1990.}

\vspace{-2.6mm}\bibitem{pars}{Pars, L.A., {\mynineit An Introduction to the Calculus of
Variations,} Heinemann, London, 1962.}

\vspace{-2.6mm}\bibitem{pontrybook}{Pontryagin, L.S., V.G. Boltyanskii, R.V. Gamkrelidze,
and E.F. Mischenko, {\mynineit The Mathematical Theory of Optimal Processes},
Wiley, New York, 1962.}

\vspace{-2.6mm}\bibitem{rabi1}{Rabinowitz, P.~H., {\mynineit Th\'eorie du d\'egr\'e topologique et
applications \`a des problemes aux
limites nonlin\'eaires.} Lect. Notes 
Universit\'e de  Paris Sud (Orsay), 1973.} 

\vspace{-2.6mm}\bibitem{rabi2}{Rabinowitz, P.~H., ``Some global results for nonlinear
eigenvalue problems,'' {\mynineit J. Functional Analysis} {\myninebf 7} (1971),
pp.~487-513.}

\vspace{-2.6mm}\bibitem{rocka}{Rockafellar, R.~T., ``Equivalent subgradient 
versions of Hamiltonian and Euler-Lagrange equations 
in variational analysis,'' 
{\mynineit SIAM J. Control and Optimization} {\myninebf 34} (1996), pp.~1300-1314.}

\vspace{-2.6mm}\bibitem{sussiber} {Sussmann, H.~J. ``A weak regularity theorem for
real analytic optimal control problems,''
{\mynineit Re\-vis\-ta Ma\-te\-m\'a\-ti\-ca Ibe\-ro\-ame\-ri\-cana} {\myninebf 2}, No. 3  
(1986) pp.~307-317.}

\vspace{-2.6mm}\bibitem{sussathens} {Sussmann, H.~J., ``Trajectory regularity and real
analyticity: some recent results,'' {\mynineit Proc. 25th IEEE Conference on
Decision and Control, Athens, Greece, 1986}, IEEE publications, New York,
1986, pp.~592-595.}

\vspace{-2.6mm}\bibitem{sussworkshop} {Sussmann, H.~J. (ed.), {\mynineit Nonlinear
Controllability and Optimal Control}, Marcel Dekker, New York, 1990.}

\vspace{-2.6mm}\bibitem{sussloja} {Sussmann, H.J., ``A strong version of the Maximum
Principle under weak hypotheses,'' {\mynineit Proc. 33rd IEEE Conference on
Decision and Control, Orlando, FL, 1994}, IEEE publications, New York,
pp.~1950-1956.}

\vspace{-2.6mm}\bibitem{susspavel} {Sussmann, H.J., ``A strong version of the Lojasiewicz 
Maximum Principle,'' in {\mynineit Optimal Control of Differential Equations,}
Nicolai H. Pavel Ed., M. Dekker Inc., New York, 1994.}

\vspace{-2.6mm}\bibitem{sussrecent} {Sussmann, H.J.,  
``Some recent results on the maximum principle of optimal control theory.''
In {\mynineit Systems and Control in the Twenty-First Century}, 
C. I. Byrnes, B. N. Datta, D. S. Gilliam,  and C. F. Martin, Eds.,
Birkh\"auser, Boston, 1997, pp.~351-372.}

\vspace{-2.6mm}\bibitem{susskobe} {Sussmann, H.J.,  
``A strong maximum principle for systems of differential inclusions.''
in {\mynineit Proceedings of the 35th IEEE Conference on Decision and Control},
Kobe, Japan, Dec. 1996.  IEEE publications, 1996, pp.~1809-1814 }

\vspace{-2.6mm}\bibitem{sussfree}{Sussmann, H. J., 
``An introduction to the coordinate-free maximum principle,''
In \cite{jakures} (1997), pp.~463-557.}

\vspace{-2.6mm}\bibitem{sussmulti}{Sussmann, H. J., 
``Multidifferential calculus: chain rule,  open mapping
and transversal intersection theorems,'' to appear in 
{\mynineit Optimal  control: theory, algorithms, and applications},
W. W. Hager and P. M. Pardalos, Editors, Kluwer Academic Publishers, 
1997.}

\vspace{-2.6mm}\bibitem{susscrac}{Sussmann, H.~ J., ``Some optimal control applications of
real analytic stratifications and desingularization.''  To appear in {\mynineit
Singularities Symposium---\L{}ojasiewiz 70} (proceedings of a conference
held in Krak\'ow, Poland, September 1996, in celebration of the 70th
birthday of Stanislaw \L{}ojasiewicz), B. Jakubczyk and W. Pawlucki, Eds.,
Banach Center Publications, Warsaw, Poland.}

\vspace{-2.6mm}\bibitem{sussjan1}{Sussmann, H. J., and J. C. Willems,
``300 years of optimal control: from the brachystochrone to the maximum 
principle,'' {\mynineit IEEE Control Systems Magazine}, 
June 1997.}

\vspace{-2.6mm}\bibitem{sussjan2}{Sussmann, H. J., and J. C. Willems,
``Curve optimization from the brachystochrone to the maximum 
principle and beyond,'' in preparation.}

\vspace{-2.6mm}\bibitem{tuan}{Tuan, H.~D., ``Controllability and extremality in nonconvex
differential inclusions,'' {\mynineit J. Optim. Theory Appl.}
{\myninebf 85} (1995), pp.~435-472.}

\vspace{-2.6mm}\bibitem{wargaa}{Warga, J., ``Fat homeomorphisms and unbounded derivate
containers,'' {\mynineit J. Math. Anal. Appl.} {\myninebf 81} (1981), pp.~545-560.}

\vspace{-2.6mm}\bibitem{wargab}{Warga, J., ``Controllability, extremality and
abnormality  
in nonsmooth optimal control,'' {\mynineit J. Optim. Theory Applic.} {\myninebf 41}
(1983), pp.~239-260.} 

\vspace{-2.6mm}\bibitem{wargac}{Warga, J., ``Optimization and controllability without
differentiability  assumptions,'' {\mynineit SIAM J. Control and Optimization}
{\myninebf 21} (1983), pp.~837-855.} 

\vspace{-2.6mm}\bibitem{wargad}{Warga, J., ``Homeomorphisms and local $C^1$
approximations,'' {\mynineit Nonlinear Anal. TMA} {\myninebf 12} (1988), pp.~ 593-597.}

\vspace{-2.6mm}\bibitem{wargae}{Warga, J., ``An extension of the Kaskosz maximum principle,''
{\mynineit Applied Math. Optim.} {\myninebf 22} (1990), pp.~61-74.}

\vspace{-2.6mm}\bibitem{yourgrau}{Yourgrau, W., and S. Mandelstam,
{\mynineit Variational Principles in Dynamics and Quantum Theory},
W. B. Saunders \& Co., Philadelphia, 1968.}

\vspace{-2.6mm}\bibitem{zhu}{Zhu. Q.~J., ``Necessary optimality conditions for nonconvex
differential inclusions with endpoint constraints,''  {\mynineit
J. Diff. Equations} {\myninebf 124} (1996), pp. 186-204.}

}

\end{thebibliography}

\end{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%  GNUSTYLE:   The file svcon2ehjs.sty starts here:
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Copyright (C) 1988, 1994 Springer-Verlag, Inc.
% 
% Everyone is granted permission to copy and redistribute this file, as
% long as it is unchanged. If any changes are made the file MUST be renamed.
%
%
% svcon.sty Dec 94
% Fixed lots & lots of bugs
% Made compatible with LaTeX2e
%
%Get rid of ae's
\catcode`\^^Z 10

\newif\if@multipleauthors
\@ifundefined{@svsa}{%
        \@multipleauthorstrue
	\typeout{LaTeX2e package "svcon2e" (Springer-Verlag Contributor).
		released December 1994}}%
        {\@multipleauthorsfalse}
\@twosidetrue
\headheight=9pt
\headsep=15pt   % 2pc-9pt
\textheight=45pc
%Let's put integer number of 12-pt \baselineskip's on page
\advance\textheight-2\p@
\textwidth=27pc
\parindent=1em
\parskip\z@
%
% Running head: 9 pt CM, c/lc, 2em# inside of fl. outside folio, base aligns
% with folio; recto: CN, period, word # to CT; verso: contributor's name(s).
% In single-author, verso: SN, period, word # to ST.
%
% If you want the running heads on the opposite pages, just put \flipheads
% after the documentstyle.  LaTeX isn't bright enough to allow one option
% file to define ds@flipheads.
%
\newif\if@flipheads \@flipheadsfalse
\def\flipheads{\@flipheadstrue}
\def\ps@myheadings{%
  \let\@oddfoot\@empty\let\@evenfoot\@empty
  \if@flipheads
    \def\@evenhead{\small\upshape\thepage\hskip2em\rightmark\hfill}%
    \def\@oddhead{\small\upshape\hfil\leftmark\hskip 2em minus 1em\thepage}%
  \else
    \def\@evenhead{\small\upshape\thepage\hskip2em\leftmark\hfill}%
    \def\@oddhead{\small\upshape\hfil\rightmark\hskip 2em minus 1em\thepage}%
  \fi
  \def\chaptermark##1{\markbothsame{\dochapternumber##1}}%
  \if@multipleauthors
    \def\sectionmark##1{{}}%
  \else
    \def\sectionmark##1{\markboth{{}%
      {\ifnum\c@secnumdepth>\z@\thesection \ \fi##1}}}
  \fi}

\pagestyle{myheadings}
\pagenumbering{roman}   % toc is in roman
\def\@chapterauthors{}
\def\markbothsame#1{\mark{{#1}{#1}}}
%
% All pages without running heads must carry corner marks.
%
\def\clap#1{\hbox to 0pt{\hss#1\hss}}
\newdimen\@crosshairrule \@crosshairrule=.24pt
\def\@crosshairs{%
\vbox to 0pt{\hsize=0pt\baselineskip=0pt\lineskip=0pt
  \vss
  \clap{\vrule height .125in width \@crosshairrule depth 0pt}
  \clap{\vrule width .25in height \@crosshairrule depth 0pt
   \rlap{\vbox to\z@{\vss%\hbox{This is page \thepage\strut}%
%      \hbox{Printer: Opaque this\strut}
\vss}}}
  \clap{\vrule height .125in width \@crosshairrule depth 0pt}
  \vss}}
\def\ps@empty{%
\def\@oddhead{\hfill\raise\headheight\@crosshairs}
\let\@evenhead\@oddhead
\def\@evenfoot{}\let\@oddfoot\@evenfoot}
\frenchspacing
\def\@chapapp{}
%8/10
\def\viiiONx{\@setfontsize\@footnotesize\@viiipt{10pt}}
%12/14
\def\xiiONxiv{\@setfontsize\large\@xiipt{14pt}}
%14/16
\def\xivONxvi{\@setfontsize\Large\@xivpt{16pt}}
%14/14
\def\xivONxiv{\@setfontsize\Large\@xivpt{14pt}}
%20/20
\def\xxONxx{\@setfontsize\huge\@xxpt{20pt}}
%
\renewcommand\footnotesize{%
   \viiiONx
   \abovedisplayskip 6\p@ \@plus2\p@ \@minus4\p@
   \abovedisplayshortskip \z@ \@plus\p@
   \belowdisplayshortskip 3\p@ \@plus\p@ \@minus2\p@
   \def\@listi{\leftmargin\leftmargini
               \topsep 3\p@ \@plus\p@ \@minus\p@
               \parsep 2\p@ \@plus\p@ \@minus\p@
               \itemsep \parsep}%
   \belowdisplayskip \abovedisplayskip
}
%
\def\chapter{\cleardoublepage      % Starts new recto.
   \thispagestyle{empty}     % Page style of chapter page is 'empty'
   \global\@topnum\z@        % Prevents figures from going at top of page.
   \@afterindentfalse        % Suppresses indent in first paragraph.  Change
   \secdef\@chapter\@schapter}   % to \@afterindenttrue to have indent.

%
% Chapter number: 20 CM bold, arabic numeral, fl.l, top text
% Chapter title: 20/20 CM bold x 27pc, c/lc, fl.l, sink 60 pt from top
%       type to base of CT, to's fl.l, do not hyphenate
%
\def\@makechapterhead#1{             % Heading for \chapter command
% We want the chapter number to be 20 pts from top text.  The simplest
% was is to set \topskip, making sure there's nothing between the number
% and the top of page, so \baselineskip doesn't come in.
  { \topskip=40pt \parindent 0pt \raggedright 
    \xxONxx\bf
    #1\par                      % Title.
    \nobreak                    % TeX penalty to prevent page break.
    \vskip-\prevdepth           % Fool TeX into ignoring the depth of
    \prevdepth=1sp              % the CT when spacing to the CA
    \if@multipleauthors
      \xdef\@chaptertitle{\dochapternumber#1}
      \vskip 16pt                % 2.5pi b/b between title and chapter authors
    \else
      \chaptermark{#1}
      \vskip 44pt        % want ~60pt b/b to abstract
    \fi
  } }
\def\dochapternumber{\ifnum\c@secnumdepth>\m@ne\@chapapp\ \thechapter. \ \fi}

\def\@makeschapterhead#1{             % Heading for \chapter* command
  { \topskip=80pt \parindent 0pt \raggedright 
    \xxONxx\bf
    #1\par                      % Title.
    \nobreak                    % TeX penalty to prevent page break.
    \vskip-\prevdepth           % Fool TeX into ignoring the depth of
    \prevdepth=1sp              % the CT when spacing to the CA
    \if@multipleauthors
      \xdef\@chaptertitle{\dochapternumber#1}
     \vskip 16pt                % 2.5pi b/b between title and chapter authors
    \else
      \chaptermark{#1}
      \vskip 44pt                % want ~60pt b/b to abstract
    \fi
  } }

\if@multipleauthors
\gdef\chapterauthors#1{%
        {\def\\{, }\def\footnote##1{}\def\footnotemark[##1]{}%
                \xdef\@chapterauthors{#1}}%
        \setsvmamark
        {\parindent0pt \raggedright\xivONxiv\bf #1\par
                \nobreak
}}\fi
\def\setsvmamark{\mark{{\@chapterauthors}{\@chaptertitle}}}
\def\shortchapname#1{\xdef\@chaptertitle{\dochapternumber#1}\setsvmamark}
\def\shortauthname#1{\xdef\@chapterauthors{#1}\setsvmamark}

%
% Restart the footnote counter each chapter.
%
%\newcounter{footnote}[chapter]
%
% FN    1pc space above footnotes to text.
%       5 pc 1/2 pt rule fl.l. above footnotes
%       leave 6pt # after rule followed by ftn
%
\skip\footins=18.5pt    % space above line and below line, 1pc+6pt+.5pt
\footskip=16pt
\footnotesep=7pt        % pretty arbitrary
\def\footnoterule{\kern-6.5pt\hrule width 5pc height .5pt\vskip 6pt}
%
% Footnote at end of chapter position below last text line (not at bottom
% of page).
%
\def\clearpage{\par\vfill\penalty -\@M \write\m@ne{}\vbox{}\penalty -\@Mi}
\def\newpage{\par\vfill\penalty -\@M}
\def\@makecol{\ifvoid\footins \setbox\@outputbox\box\@cclv
  \else\setbox\@outputbox
     \vbox{\unvbox\@cclv\vskip\skip\footins\footnoterule\unvbox\footins}\fi
     \xdef\@freelist{\@freelist\@midlist}\gdef\@midlist{}\@combinefloats
     \setbox\@outputbox\vbox to\@colht{\boxmaxdepth\maxdepth
        \@texttop
        \unvbox\@outputbox
        \@textbottom}\global\maxdepth\@maxdepth}
\flushbottom
%
% Abstract: 9/11 CMx25pi, sink 2.5pi b/b from CA
%
\def\abstract{\vskip 21pt % from CA
\rightskip=1pc\leftskip=1pc\parindent=\z@
\small\rm
{ABSTRACT }}
\def\endabstract{\par}  % If we don't do this, the paragraph won't be
                        % set until after rightskip and leftskip have been
                       % returned to their old values.

\def\tableofcontents{\@restonecolfalse\if@twocolumn\@restonecoltrue\onecolumn
  \fi\chapter*{Contents\@mkboth{Contents}{Contents}}
  \@starttoc{toc}\if@restonecol\twocolumn\fi}

%
% Don't indent the paragraph following each header.  Vertical distances
% showld have \baselineskip taken out, and the height of the next
% line, which should be text (10/12 CM).
%
\setcounter{secnumdepth}{2}     % number section and subsections
\setcounter{tocdepth}{2}        % and put them in the table of contents
% No 1 Head, 14/16 CM, 36pt b/b above, 24 tp b/b below
\def\section{\@startsection
    {section}{1}{\z@}{-24pt plus -6pt minus -3pt}{12pt}%
        {\xivONxvi\rm\raggedright}}
% No 2 Head, 12/14 CM, c/sc, 30pt above, 18 pt below
\def\subsection{\@startsection
    {subsection}{2}{\z@}{-18pt plus-4.8pt minus-2.4pt}{6pt}%
        {\xiiONxiv\it\raggedright}}
% No 3 Head, 10/12 CM, 24pt above, 18pt below
\def\subsubsection{\@startsection
    {subsubsection}{3}{\z@}{-12pt plus-4.2pt minus-2.1pt}{6pt}%
        {\xONxii\rm\raggedright}}
% No 4 Head, 10/12 CM italic, 20pt above, 18pt below
\def\paragraph{\@startsection
     {paragraph}{4}{\z@}{-8pt plus-3.8pt minus-1.7pt}{6pt}%
        {\xONxii\it\raggedright}}
% No 5 Head, 10/12 CM italic, period,  en space to run in text, 18pt above
\def\subparagraph#1{\@startsection
     {subparagraph}{5}{\z@}{-6pt plus-1.8pt minus-.9pt}{-.5em}{\xONxii\it}{#1.}}
% Ack: Acknowledgements same as No 5 head, 36pt above
\def\acknowledgments{\@startsection
     {subparagraph}{6}{\z@}{-24pt plus-3.6pt minus-1.8pt}{-.5em}%
        {\xONxii\it}*{Acknowledgments:\/}}
% R-1: ``References'', same as No.2 head
% unless it's single-author.
\def\mainbibliography#1{{\@multipleauthorsfalse\bibliography{#1}}}
\def\thebibliography#1{%
\if@multipleauthors
%  \@startsection
%    {section}{2}{\z@}{-16pt plus -5pt minus -1pt}{6pt}%

\vspace{1cm}\noindent{\sc References}
%       {\xiiONxiv\sc\raggedright}{References}%
\else
  \chapter*{References\@mkboth
  {References}{References}}%
  \addcontentsline{toc}{chapter}{\protect\numberline{}References}%
\fi
%endif
  \list
  {[\arabic{enumiv}]}{\settowidth\labelwidth{[#1]}\leftmargin\labelwidth
    \advance\leftmargin\labelsep
%Per zwart, enumi replaced by enumiv in this macro
    \usecounter{enumiv}}
    \def\newblock{\hskip .11em plus .33em minus -.07em}
    \sloppy
    \sfcode`\.=1000\relax}
% Put index in the toc
\def\theindex{\@restonecoltrue\if@twocolumn\@restonecolfalse\fi
\columnseprule \z@
\columnsep 35pt\twocolumn[\hbox{}\vskip 48pt\@makeschapterhead{Index}]
    \addcontentsline{toc}{chapter}{\protect\numberline{}Index}%
    \@mkboth{Index}{Index}\thispagestyle{empty}\parindent\z@
    \parskip\z@ plus .3pt\relax\let\item\@idxitem}

%
% lists: 10/12 CMx27pc, 1/2line# above/below
%
\topsep 6pt plus1.8pt minus.9pt    % Extra vertical space, in addition to 
                                   % \parskip, added above and below list and
                                   % paragraphing environments.
\itemsep 6pt plus1.8pt minus.9pt   % Extra vertical space, in addition to
                                   % \parskip, added between list items.
%
% Eq: Center on text width; equation numbers set flush right, align with
% last line of equation, 6pt # above/below.
%
\abovedisplayskip=6pt plus 1.8pt minus.9pt
\belowdisplayskip=\abovedisplayskip
\abovedisplayshortskip=\z@ plus 1.2pt
\belowdisplayshortskip=3pt plus 1.5pt minus .75pt
%
% FL: Figure legends ``FIGURE 0.0'' 9/11 CM, c/lc, period, word # to text.
\floatsep 12pt plus 2.4pt minus 1.2pt
\textfloatsep 12pt plus 4.8pt
\intextsep 12pt plus 2.4pt minus 1.2pt
%%\@maxsep 12pt
%
\long\def\@makecaption#1#2{{
   \vskip 6pt
   \small
   \setbox\@tempboxa\hbox{#1. #2}
   \ifdim \wd\@tempboxa >\hsize   % IF longer than one line:
       \unhbox\@tempboxa\hfil\par      %   THEN set as ordinary paragraph.
     \else                        %   ELSE  center.
       \hbox to\hsize{\hfil\box\@tempboxa\hfil}
   \fi}}
\def\fnum@figure{\small FIGURE \thefigure}%{\sc FIGURE~}\thefigure}
%
% TN+TT: ``TABLE 0.0'' 9/11 CM, c/lc. period, word # to table title:
% 3 pt # below to 1/2 pt rule x table measuer, 3 pt # below rule to
% TCH
%
\arrayrulewidth=.5pt
\def\fnum@table{\small TABLE~\thetable}
\def\TableSubtitleRule{\noalign{\vskip 3pt}\hline\noalign{\vskip 3pt}}
\long\def\TableFootnote#1{\gdef\TFN{#1}}

\def\@@startpbox#1{\vtop\bgroup \hsize #1\@arrayparboxrestore
\parindent=-1em\leftskip=1em}
\newsavebox{\@stablebox}
\def\stable#1#2{\@ifnextchar[{\@stable{#1}{#2}}{\@stable{#1}{#2}[h]}}
\def\@stable#1#2[#3]{\begin{table}[#3]
        \def\foo{#1}\def\TFN{}%
        \viiiONx
        \setbox\@stablebox=\hbox\bgroup
        \begin{tabular}{#2}%
        \noalign{\vskip 3pt}
        \hline
        \noalign{\vskip 3pt}}
\@namedef{endstable}{\crcr\noalign{\vskip 6pt}\hline\end{tabular}\egroup
$$\vbox{\hsize=\wd\@stablebox
\caption{\foo}
\box\@stablebox
\prevdepth=0pt  % last thing was a rule
\par
\TFN}$$
\end{table}}
%%%% GROT.  This is hacked to move a label definition outside.
%%%% I think the whole stable idea is broken.

\def\stable#1#2{\@ifnextchar[{\@stable{#1}{#2}}{\@stable{#1}{#2}[h]}}
\def\@stable#1#2[#3]{\table[#3]
        \def\foo{#1}\def\TFN{}\def\stablelabel{}%
        \viiiONx
        \setbox\@stablebox=\hbox\bgroup\def\label##1{\gdef\stablelabel{\label{##1}}}%
        \tabular{#2}%
        \noalign{\vskip 3pt}%
        \hline
        \noalign{\vskip 3pt}}
\@namedef{endstable}{\crcr\noalign{\vskip 6pt}\hline\endtabular\egroup
$$\vbox{\hsize=\wd\@stablebox
\caption{\foo}\stablelabel
\box\@stablebox
\prevdepth=0pt  % last thing was a rule
\par
\TFN}$$
\endtable}

\def\stablecont#1#2{\@ifnextchar[{\@stablecont{#1}{#2}}{\@stablecont{#1}{#2}[h]}}
\def\@stablecont#1#2[#3]{\table[#3]
        \def\foo{#1}\def\TFN{}
        \viiiONx
        \setbox\@stablebox=\hbox\bgroup\def\label{\error}
        \tabular{#2}%
        \noalign{\vskip 3pt}%
        \hline
        \noalign{\vskip 3pt}}
\@namedef{endstablecont}{\crcr\noalign{\vskip 6pt}\hline\endtabular\egroup
$$\vbox{\hsize=\wd\@stablebox
  \begingroup
    \@parboxrestore
        \vskip 10pt
        \small
           \setbox\@tempboxa\hbox{\ignorespaces \foo}
           \ifdim \wd\@tempboxa >\hsize   % IF longer than one line:
               \unhbox\@tempboxa\hfil\par      %   THEN set as ordinary paragraph.
             \else                        %   ELSE  center.
               \hbox to\hsize{\hfil\box\@tempboxa\hfil}
           \fi\par
  \endgroup
\box\@stablebox
\prevdepth=0pt  % last thing was a rule
\par
\TFN}$$
\endtable}

%%%%
\newdimen\stabularwidth

\def\stabular{\@ifnextchar[{\@stabular}{\@stabular[h]}}
\def\@stabular[#1]#2{
       \setbox\@stablebox=\hbox\bgroup\viiiONx
        \tabular[#1]{#2}%
        \noalign{\vskip 3pt}%
        \hline
        \noalign{\vskip 3pt}}
\def\endstabular{\crcr\noalign{\vskip 6pt}\hline\endtabular\egroup
\stabularwidth=\wd\@stablebox
\box\@stablebox}

\def\thesection{\arabic{section}}  %% deletes Chap No from Section.
\def\thefigure{\@arabic\c@figure}  %% deletes Chap No from Section.

\def\appendix{\par
 \setcounter{chapter}{0}
 \setcounter{section}{0}
 \def\@chapapp{}
 \def\thechapter{\Alph{chapter}}}

\usepackage{psfig}
\def\figput#1#2#3{\begin{figure}\hfil%
\psfig{figure=#1,height=#2}\hfil\caption{#3}%
                    \end{figure}}
\def\figpage#1#2#3{\begin{figure}[p]\hfil%
\psfig{figure=#1,height=#2}\hfil\caption{#3}%
                    \end{figure}}
% End of svcon2e.sty
\endinput







%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%  GNU:   The file svcon2ehjs.sty ends here.
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


