% def-lexical.tex
%
\def\name#1{\ttlbrace\emph{#1}\ttrbrace}
%
\section{Lexical Structure}
\seclab{lexical}

This section defines the lexical structure of RML.

\subsection{Reserved Words}
The following are the reserved words, including special symbols.
The word \verb@--@ represents
all words generated by the regular expression \verb@-(-)+@~.
\begin{verbatim}
and  as  axiom  datatype  default  end  eqtype  fail  let
module  not  of  relation  rule  type  val  with  withtype
&  (  )  *  ,  --  .  :  ::  =  =>  [  ]  _  |
\end{verbatim}

\subsection{Whitespace and Comments}
The blank, tab, linefeed, carriage return, and formfeed characters
are treated as whitespace characters.
Whitespace characters between tokens serve as separators, but are
otherwise ignored.

\begin{tabular}{rcl}
\emph{white} & = & \verb@[\ \t\n\013\012]@
\end{tabular}

A comment is any character sequence within comment brackets \verb@(*@ \verb@*)@
in which comment brackets are properly nested.
It is an error for a comment bracket to be unmatched.
A comment is equivalent to a single blank character.

\begin{tabular}{@{}r@{ }r@{ }l}
\emph{comment} & ::= & $\texttt{(*}~\emph{skipto}_{\texttt{*)}}$\\
$\emph{skipto}_{\texttt{*)}}$ & ::= & $\texttt{*}~\emph{after}_{\texttt{*}}$\\
& \BAR & $\texttt{(}~\emph{after}_{\texttt{(}}$\\
& \BAR & \verb@[^*(]@~$\emph{skipto}_{\texttt{*)}}$\\
$\emph{after}_{\texttt{*}}$ & ::= & $\texttt{)}$\\
& \BAR & $\texttt{*}~\emph{after}_{\texttt{*}}$\\
& \BAR & $\texttt{(}~\emph{after}_{\texttt{(}}$\\
& \BAR & \verb@[^)*(]@~$\emph{skipto}_{\texttt{*)}}$\\
$\emph{after}_{\texttt{(}}$ & ::= & $\texttt{*}~\emph{skipto}_{\texttt{*)}}~\emph{skipto}_{\texttt{*)}}$\\
& \BAR & $\texttt{(}~\emph{after}_{\texttt{(}}$\\
& \BAR & \verb@[^*(]@~$\emph{skipto}_{\texttt{*)}}$\\
\end{tabular}

\subsection{Integer constants}
The token class ICon, ranged over by \emph{icon}, denotes integer constants.
An integer constant may be written in decimal or hexadecimal base,
optionally preceded by a negation sign (\verb@-@ or \verb@~@).
The precision of integers is implementation-dependent.
Examples: \verb@34@ ~ \verb@0x22@ ~ \verb@-1@ ~.

\begin{tabular}{rcl}
\emph{ddigit} & = & \verb@[0-9]@ \\
\emph{decint} & = & \verb@[-~]?@\name{ddigit}\verb@+@ \\
\emph{xdigit} & = & \verb@[0-9a-fA-F]@ \\
\emph{hexint} & = & \verb@[-~]?"0x"@\name{xdigit}\verb@+@ \\
\emph{icon} & = & \name{decint}\verb@|@\name{hexint}
\end{tabular}

\subsection{Real constants}
The token class RCon, ranged over by \emph{rcon}, denotes real constants.
A real constant is written as a decimal integer constant,
followed by a fraction and an exponent.
Either the fraction or the exponent, but not both, may be omitted.
The precision of reals is implementation-dependent.
Examples: \verb@0.7@ ~ \verb@3.25E5@ ~ \verb@3E-7@ ~.

\begin{tabular}{rcl}
\emph{fraction} & = & \verb@"."@\name{ddigit}\verb@+@ \\
\emph{exponent} & = & \verb@[eE]@\name{decint} \\
\emph{rcon} & = & \name{decint}\verb@(@\name{fraction}\name{exponent}\verb@?@\verb@|@\name{exponent}\verb@)@
\end{tabular}

\subsection{Character constants}
The token class CCon, ranged over by \emph{ccon}, denotes character constants.
An underlying alphabet of 256 distinct characters numbered
0 to 255 is assumed, such that the characters numbered 0 to 127
coincide with the ASCII character set.
A character constant is written as \verb@#"@, followed by
a character descriptor, and terminated by \verb@"@.
A character descriptor is either a single printing character,
or \verb@\@ followed by one of these escape sequences:
\begin{itemize}

\item[\ttbackslash\textit{ddd}]
A sequence of three decimal digits,
denoting a character number in the interval $[0,255]$.

\item[\ttbackslash\ttcaret\textit{c}]
The control character $c$, where $c$ may be any character with number 63--95.
The number of \ttbackslash\ttcaret\textit{c} is 64 less
than the number of $c$, modulo 128,
mapping \verb@\^?@ to 127 (delete),
and \verb@\^A@--\verb@\^_@ to 0--31.

\item[\ttbackslash\textit{c}]
A single escape character $c$, with the following interpretations:

\begin{tabular}{lcrl}
\verb@\\@ & = & 92 & (backslash) \\
\verb@\"@ & = & 34 & (double quote) \\
\verb@\n@ & = & 10 & (linefeed) \\
\verb@\r@ & = & 13 & (carriage return) \\
\verb@\t@ & = & 9 & (tab) \\
\verb@\f@ & = & 12 & (formfeed) \\
\verb@\a@ & = & 7 & (alert) \\
\verb@\b@ & = & 8 & (backspace) \\
\verb@\v@ & = & 11 & (vertical tab)
\end{tabular}

\end{itemize}

\begin{tabular}{rcl}
\emph{echar} & = & \verb@[\\"nrtfabv]@ \\
\emph{cntrl} & = & \verb+[?-_]+ \\
\emph{escseq} & = & \name{ddigit}\verb@{3}|"^"@\name{cntrl}\verb@|@\name{echar} \\
\emph{pchar} & = & \verb@[\ -!#-[^-~\128-\255]|"]"@ \\
\emph{cdesc} & = & \name{pchar}\verb@|\\@\name{escseq} \\
\emph{ccon} & = & \verb@"#\""@\name{cdesc}\verb@\"@
\end{tabular}

Examples of character constants:
\verb@#"n"@ ~ \verb@#"\n"@ ~ \verb@#"\010"@ ~ \verb@#"\^J"@ ~.

\subsection{String constants}
The token class SCon, ranged over by \emph{scon}, denotes string constants.
A string constant is written as a sequence of string items
enclosed by a pair of double quotes \verb@"@.
A string item is either a character descriptor, denoting a single character,
or a \emph{gap}, denoting an empty sequence of characters.
Examples: \verb@"thirty-four is 3\ \4"@ ~ \verb@"TAB is \t"@ ~.

\begin{tabular}{rcl}
\emph{gap} & = & \verb@\\@\name{white}\verb@+\\@ \\
\emph{sitem} & = & \name{cdesc}\verb@|@\name{gap} \\
\emph{scon} & = & \verb@\"@\name{sitem}\verb@*\"@
\end{tabular}

\subsection{Identifiers}
The token class Id, ranged over by \emph{id}, denotes identifiers.
An identifier is written as a non-empty sequence of ASCII letters,
digits, primes, or underscores, starting with a letter.
An instance of the regular expression for \emph{id} that coincides with a
reserved word is interpreted as that reserved word, not as an identifier.
Examples: \verb@cons@ ~ \verb@g4711'@ ~.

\begin{tabular}{rcl}
\emph{alpha} & = & \verb@[A-Za-z]@ \\
\emph{alnum} & = & \name{alpha}\verb@|[_'0-9]@ \\
\emph{id} & = & \name{alpha}\name{alnum}\verb@*@
\end{tabular}

\subsection{Type Variables}
The token class TyVar, ranged over by \emph{tyvar}, denotes type variables.
A type variable is written as an identifier, prefixed by one or more primes.
The subclass EtyVar of TyVar, the \emph{equality} type variables,
consists of those which start with two or more primes.
Examples: \verb@'a@ ~ \verb@''key@ ~ .

\begin{tabular}{rcl}
\emph{tyvar} & = & \verb@"'"+@\name{alpha}\name{alnum}\verb@*@
\end{tabular}

\subsection{Lexical analysis}
Lexical analysis maps a program to a sequence of items, in left to right order.
Each item is either a reserved word, an integer, real,
character, or string constant, an identifier, or a type variable.
Whitespace and comments separate items but are otherwise ignored -- except
within character and string constants.
At each stage, the longest recognizable item is taken.
