Add generic Boot and Spad tokenizer.

19 files changed, 1334 insertions, 325 deletions

diff --git a/ChangeLog b/ChangeLog
index a42b12b3..288ae518 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2014-08-26  Gabriel Dos Reis  <gdr@integrable-solutions.net>
+
+	* config/open-axiom.m4 (OPENAXIOM_LINK_FILES): New.
+	* configure.ac: Call it.
+	Protect openaxiom-c-macros.h.in for multiple inclusion.
+	Now install it as <open-axiom/config>.
+
 2014-08-16  Gabriel Dos Reis  <gdr@integrable-solutions.net>
 
 	* config/open-axiom.m4: Do not waste time checking for standard
diff --git a/Makefile.in b/Makefile.in
index cb614405..06d89f4c 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -147,7 +147,26 @@ DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
 	$(top_srcdir)/src/graph/view3D/Makefile.in \
 	$(top_srcdir)/src/graph/viewAlone/Makefile.in \
 	$(top_srcdir)/src/graph/viewman/Makefile.in \
-	$(top_srcdir)/src/gui/gui.pro.in AUTHORS COPYING ChangeLog \
+	$(top_srcdir)/src/gui/gui.pro.in \
+	$(top_srcdir)/src/utils/hash-table.H \
+	$(top_srcdir)/src/utils/string-pool.H \
+	$(top_srcdir)/src/include/diagnostics.H \
+	$(top_srcdir)/src/include/dialect.H \
+	$(top_srcdir)/src/include/token-value.def \
+	$(top_srcdir)/src/include/token.H \
+	$(top_srcdir)/src/include/defaults.H \
+	$(top_srcdir)/src/include/structure.H \
+	$(top_srcdir)/src/include/iterator.H \
+	$(top_srcdir)/src/include/storage.H \
+	$(top_srcdir)/src/include/Charset.H \
+	$(top_srcdir)/src/include/FileMapping.H \
+	$(top_srcdir)/src/include/SourceFile.H \
+	$(top_srcdir)/src/include/Input.H \
+	$(top_srcdir)/src/include/vm.H \
+	$(top_srcdir)/src/include/sexpr.H \
+	$(top_srcdir)/src/include/Lisp.H \
+	$(top_srcdir)/src/include/Constructor.H \
+	$(top_srcdir)/src/include/Database.H AUTHORS COPYING ChangeLog \
 	INSTALL NEWS README TODO config/compile config/config.guess \
 	config/config.sub config/depcomp config/install-sh \
 	config/missing config/mkinstalldirs config/ltmain.sh \
diff --git a/config/open-axiom.m4 b/config/open-axiom.m4
index 6c374f66..d16ee27e 100644
--- a/config/open-axiom.m4
+++ b/config/open-axiom.m4
@@ -1120,3 +1120,33 @@ case $oa_gnu_compiler in
      ;;
 esac
 ])
+
+
+dnl --------------------------
+dnl -- OPENAXIOM_LINK_FILES --
+dnl --------------------------
+AC_DEFUN([OPENAXIOM_LINK_FILES],[
+oa_incdir=$target/include/open-axiom
+AC_CONFIG_LINKS([
+  $oa_incdir/hash-table:src/utils/hash-table.H
+  $oa_incdir/string-pool:src/utils/string-pool.H
+  $oa_incdir/diagnostics:src/include/diagnostics.H
+  $oa_incdir/dialect:src/include/dialect.H
+  $oa_incdir/token-value:src/include/token-value.def
+  $oa_incdir/token:src/include/token.H
+  $oa_incdir/defaults:src/include/defaults.H
+  $oa_incdir/structure:src/include/structure.H
+  $oa_incdir/iterator:src/include/iterator.H
+  $oa_incdir/storage:src/include/storage.H
+  $oa_incdir/Charset:src/include/Charset.H
+  $oa_incdir/FileMapping:src/include/FileMapping.H
+  $oa_incdir/SourceFile:src/include/SourceFile.H
+  $oa_incdir/Input:src/include/Input.H
+  $oa_incdir/vm:src/include/vm.H
+  $oa_incdir/sexpr:src/include/sexpr.H
+  $oa_incdir/Lisp:src/include/Lisp.H
+  $oa_incdir/Constructor:src/include/Constructor.H
+  $oa_incdir/Database:src/include/Database.H
+])
+
+])
diff --git a/config/openaxiom-c-macros.h.in b/config/openaxiom-c-macros.h.in
index c2503633..26fb89f8 100644
--- a/config/openaxiom-c-macros.h.in
+++ b/config/openaxiom-c-macros.h.in
@@ -1,5 +1,8 @@
 /* config/openaxiom-c-macros.h.in.  Generated from configure.ac by autoheader.  */
 
+#ifndef OPENAXIOM_CONFIG_included
+#define OPENAXIOM_CONFIG_included
+
 /* Define if building universal (internal helper macro) */
 #undef AC_APPLE_UNIVERSAL_BUILD
 
@@ -222,3 +225,5 @@
 
 /* Alignment query operator */
 #undef alignof
+
+#endif // OPENAXIOM_CONFIG_included
diff --git a/configure b/configure
index 702f736a..685b5836 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for OpenAxiom 1.5.0-2014-08-16.
+# Generated by GNU Autoconf 2.69 for OpenAxiom 1.5.0-2014-08-26.
 #
 # Report bugs to <open-axiom-bugs@lists.sf.net>.
 #
@@ -590,8 +590,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='OpenAxiom'
 PACKAGE_TARNAME='openaxiom'
-PACKAGE_VERSION='1.5.0-2014-08-16'
-PACKAGE_STRING='OpenAxiom 1.5.0-2014-08-16'
+PACKAGE_VERSION='1.5.0-2014-08-26'
+PACKAGE_STRING='OpenAxiom 1.5.0-2014-08-26'
 PACKAGE_BUGREPORT='open-axiom-bugs@lists.sf.net'
 PACKAGE_URL=''
 
@@ -1416,7 +1416,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures OpenAxiom 1.5.0-2014-08-16 to adapt to many kinds of systems.
+\`configure' configures OpenAxiom 1.5.0-2014-08-26 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1491,7 +1491,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of OpenAxiom 1.5.0-2014-08-16:";;
+     short | recursive ) echo "Configuration of OpenAxiom 1.5.0-2014-08-26:";;
    esac
   cat <<\_ACEOF
 
@@ -1608,7 +1608,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-OpenAxiom configure 1.5.0-2014-08-16
+OpenAxiom configure 1.5.0-2014-08-26
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2425,7 +2425,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by OpenAxiom $as_me 1.5.0-2014-08-16, which was
+It was created by OpenAxiom $as_me 1.5.0-2014-08-26, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2805,6 +2805,9 @@ ac_configure="$SHELL $ac_aux_dir/configure"  # Please don't use this var.
 
 
 
+
+
+
 ac_config_headers="$ac_config_headers config/openaxiom-c-macros.h"
 
 
@@ -3441,7 +3444,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='openaxiom'
- VERSION='1.5.0-2014-08-16'
+ VERSION='1.5.0-2014-08-26'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -19457,6 +19460,12 @@ case $oa_gnu_compiler in
 esac
 
 
+oa_incdir=$target/include/open-axiom
+ac_config_links="$ac_config_links $oa_incdir/hash-table:src/utils/hash-table.H $oa_incdir/string-pool:src/utils/string-pool.H $oa_incdir/diagnostics:src/include/diagnostics.H $oa_incdir/dialect:src/include/dialect.H $oa_incdir/token-value:src/include/token-value.def $oa_incdir/token:src/include/token.H $oa_incdir/defaults:src/include/defaults.H $oa_incdir/structure:src/include/structure.H $oa_incdir/iterator:src/include/iterator.H $oa_incdir/storage:src/include/storage.H $oa_incdir/Charset:src/include/Charset.H $oa_incdir/FileMapping:src/include/FileMapping.H $oa_incdir/SourceFile:src/include/SourceFile.H $oa_incdir/Input:src/include/Input.H $oa_incdir/vm:src/include/vm.H $oa_incdir/sexpr:src/include/sexpr.H $oa_incdir/Lisp:src/include/Lisp.H $oa_incdir/Constructor:src/include/Constructor.H $oa_incdir/Database:src/include/Database.H"
+
+
+
+
 ## We are ready to instantiate makefiles.
 ac_config_files="$ac_config_files Makefile src/Makefile src/io/Makefile src/syntax/Makefile src/rt/Makefile src/boot/Makefile src/algebra/Makefile"
 
@@ -19499,6 +19508,9 @@ ac_config_files="$ac_config_files src/graph/viewman/Makefile:config/var-def.mk:s
 ac_config_files="$ac_config_files src/gui/gui.pro"
 
 
+ac_config_commands="$ac_config_commands $target/include/open-axiom/config"
+
+
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
 # tests run on this system so they can be shared between configure
@@ -20070,7 +20082,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by OpenAxiom $as_me 1.5.0-2014-08-16, which was
+This file was extended by OpenAxiom $as_me 1.5.0-2014-08-26, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -20097,6 +20109,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 # Files that config.status was made for.
 config_files="$ac_config_files"
 config_headers="$ac_config_headers"
+config_links="$ac_config_links"
 config_commands="$ac_config_commands"
 
 _ACEOF
@@ -20127,6 +20140,9 @@ $config_files
 Configuration headers:
 $config_headers
 
+Configuration links:
+$config_links
+
 Configuration commands:
 $config_commands
 
@@ -20136,7 +20152,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-OpenAxiom config.status 1.5.0-2014-08-16
+OpenAxiom config.status 1.5.0-2014-08-26
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
@@ -20635,6 +20651,7 @@ fi
 
 
 
+target=$target
 
 _ACEOF
 
@@ -20647,6 +20664,25 @@ do
     "config/openaxiom-c-macros.h") CONFIG_HEADERS="$CONFIG_HEADERS config/openaxiom-c-macros.h" ;;
     "depfiles") CONFIG_COMMANDS="$CONFIG_COMMANDS depfiles" ;;
     "libtool") CONFIG_COMMANDS="$CONFIG_COMMANDS libtool" ;;
+    "$oa_incdir/hash-table") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/hash-table:src/utils/hash-table.H" ;;
+    "$oa_incdir/string-pool") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/string-pool:src/utils/string-pool.H" ;;
+    "$oa_incdir/diagnostics") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/diagnostics:src/include/diagnostics.H" ;;
+    "$oa_incdir/dialect") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/dialect:src/include/dialect.H" ;;
+    "$oa_incdir/token-value") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/token-value:src/include/token-value.def" ;;
+    "$oa_incdir/token") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/token:src/include/token.H" ;;
+    "$oa_incdir/defaults") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/defaults:src/include/defaults.H" ;;
+    "$oa_incdir/structure") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/structure:src/include/structure.H" ;;
+    "$oa_incdir/iterator") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/iterator:src/include/iterator.H" ;;
+    "$oa_incdir/storage") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/storage:src/include/storage.H" ;;
+    "$oa_incdir/Charset") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/Charset:src/include/Charset.H" ;;
+    "$oa_incdir/FileMapping") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/FileMapping:src/include/FileMapping.H" ;;
+    "$oa_incdir/SourceFile") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/SourceFile:src/include/SourceFile.H" ;;
+    "$oa_incdir/Input") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/Input:src/include/Input.H" ;;
+    "$oa_incdir/vm") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/vm:src/include/vm.H" ;;
+    "$oa_incdir/sexpr") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/sexpr:src/include/sexpr.H" ;;
+    "$oa_incdir/Lisp") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/Lisp:src/include/Lisp.H" ;;
+    "$oa_incdir/Constructor") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/Constructor:src/include/Constructor.H" ;;
+    "$oa_incdir/Database") CONFIG_LINKS="$CONFIG_LINKS $oa_incdir/Database:src/include/Database.H" ;;
     "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;;
     "src/Makefile") CONFIG_FILES="$CONFIG_FILES src/Makefile" ;;
     "src/io/Makefile") CONFIG_FILES="$CONFIG_FILES src/io/Makefile" ;;
@@ -20673,6 +20709,7 @@ do
     "src/graph/viewAlone/Makefile") CONFIG_FILES="$CONFIG_FILES src/graph/viewAlone/Makefile:config/var-def.mk:src/graph/viewAlone/Makefile.in:config/setup-dep.mk" ;;
     "src/graph/viewman/Makefile") CONFIG_FILES="$CONFIG_FILES src/graph/viewman/Makefile:config/var-def.mk:src/graph/viewman/Makefile.in:config/setup-dep.mk" ;;
     "src/gui/gui.pro") CONFIG_FILES="$CONFIG_FILES src/gui/gui.pro" ;;
+    "$target/include/open-axiom/config") CONFIG_COMMANDS="$CONFIG_COMMANDS $target/include/open-axiom/config" ;;
 
   *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
   esac
@@ -20686,6 +20723,7 @@ done
 if $ac_need_defaults; then
   test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
   test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers
+  test "${CONFIG_LINKS+set}" = set || CONFIG_LINKS=$config_links
   test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands
 fi
 
@@ -20983,7 +21021,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 fi # test -n "$CONFIG_HEADERS"
 
 
-eval set X "  :F $CONFIG_FILES  :H $CONFIG_HEADERS    :C $CONFIG_COMMANDS"
+eval set X "  :F $CONFIG_FILES  :H $CONFIG_HEADERS  :L $CONFIG_LINKS  :C $CONFIG_COMMANDS"
 shift
 for ac_tag
 do
@@ -21260,7 +21298,38 @@ $as_echo X"$_am_arg" |
 	  }
 	  s/.*/./; q'`/stamp-h$_am_stamp_count
  ;;
+  :L)
+  #
+  # CONFIG_LINK
+  #
+
+  if test "$ac_source" = "$ac_file" && test "$srcdir" = '.'; then
+    :
+  else
+    # Prefer the file from the source tree if names are identical.
+    if test "$ac_source" = "$ac_file" || test ! -r "$ac_source"; then
+      ac_source=$srcdir/$ac_source
+    fi
+
+    { $as_echo "$as_me:${as_lineno-$LINENO}: linking $ac_source to $ac_file" >&5
+$as_echo "$as_me: linking $ac_source to $ac_file" >&6;}
 
+    if test ! -r "$ac_source"; then
+      as_fn_error $? "$ac_source: file not found" "$LINENO" 5
+    fi
+    rm -f "$ac_file"
+
+    # Try a relative symlink, then a hard link, then a copy.
+    case $ac_source in
+    [\\/$]* | ?:[\\/]* ) ac_rel_source=$ac_source ;;
+	*) ac_rel_source=$ac_top_build_prefix$ac_source ;;
+    esac
+    ln -s "$ac_rel_source" "$ac_file" 2>/dev/null ||
+      ln "$ac_source" "$ac_file" 2>/dev/null ||
+      cp -p "$ac_source" "$ac_file" ||
+      as_fn_error $? "cannot link or copy $ac_source to $ac_file" "$LINENO" 5
+  fi
+ ;;
   :C)  { $as_echo "$as_me:${as_lineno-$LINENO}: executing $ac_file commands" >&5
 $as_echo "$as_me: executing $ac_file commands" >&6;}
  ;;
@@ -22166,6 +22235,7 @@ compiler_lib_search_path=$lt_compiler_lib_search_path_CXX
 _LT_EOF
 
  ;;
+    "$target/include/open-axiom/config":C) cp -p config/openaxiom-c-macros.h $target/include/open-axiom/config ;;
 
   esac
 done # for ac_tag
diff --git a/configure.ac b/configure.ac
index 035e7931..ffdec895 100644
--- a/configure.ac
+++ b/configure.ac
@@ -33,7 +33,7 @@ dnl  Makefiles for building OpenAxiom interpreter, compiler, libraries, and
 dnl  auxiliary tools where appropriate.
 dnl
 
-AC_INIT([OpenAxiom], [1.5.0-2014-08-16], 
+AC_INIT([OpenAxiom], [1.5.0-2014-08-26], 
         [open-axiom-bugs@lists.sf.net])
 
 dnl Most of the macros used in this configure.ac are defined in files
@@ -41,6 +41,10 @@ dnl located in the subdirectory config/
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([config])
 
+AH_TOP([#ifndef OPENAXIOM_CONFIG_included]
+[#define OPENAXIOM_CONFIG_included])
+AH_BOTTOM([#endif // OPENAXIOM_CONFIG_included])
+
 dnl Put all configuration macros here
 AC_CONFIG_HEADERS([config/openaxiom-c-macros.h])
 
@@ -109,6 +113,7 @@ OPENAXIOM_FFI_TYPE_TABLE
 OPENAXIOM_GCL_BUILD_OPTIONS
 OPENAXIOM_CHECK_MM
 OPENAXIOM_CHECK_MISC
+OPENAXIOM_LINK_FILES
 
 ## We are ready to instantiate makefiles.
 AC_CONFIG_FILES([
@@ -139,6 +144,10 @@ OPENAXIOM_MAKEFILE([src/graph/viewAlone/Makefile])
 OPENAXIOM_MAKEFILE([src/graph/viewman/Makefile])
 AC_CONFIG_FILES([src/gui/gui.pro])
 
+AC_CONFIG_COMMANDS([$target/include/open-axiom/config],
+  [cp -p config/openaxiom-c-macros.h $target/include/open-axiom/config],
+  [target=$target])
+
 AC_OUTPUT
 
 ## Set up Makefile for QT-based gui
diff --git a/src/ChangeLog b/src/ChangeLog
index 9630e63d..f3318a06 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,4 +1,20 @@
-2014-08-17  Gabriel Dos Reis  <gdr@axiomatics.org>
+2014-08-26  Gabriel Dos Reis  <gdr@axiomatics.org>
+
+	* Makefile.am (oa_src_include_headers): Remove.  These headers are
+	now linked by configure.
+	* boot/Makefile.am (noinst_PROGRAMS): Add bemol.
+	(bemol_SOURCES): New.
+	(bemol_LDADD): Likewise.
+	(AM_CXXFLAGS): Set.
+	* boot/bemol.cc: New file.  Initial tokenizer.
+	* include/sexpr.H: Tidy.
+	* include/token-value.def: New file.
+	* include/token.H: Expand.  Add generic tokenizer.
+	* syntax/token.cc: Rework.
+	* interp/simpbool.boot (list1): Escape leading question mark of ?ORDER.
+	(ordUnion): Likewise.
+
+2014-08-17  Gabriel Dos Reis  <gdr@integrable-solutions.net>
 
 	* boot/Makefile.am: New.
 	* boot/Makefile.in: Regenerate.
diff --git a/src/Makefile.am b/src/Makefile.am
index 5f40cf76..78ea4e62 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -44,27 +44,6 @@ subdirs = \
 
 oa_targetdir = $(top_builddir)/$(target)
 
-## Where do we put installed include header files?
-oa_incdir = $(oa_targetdir)/include/open-axiom
-
-oa_src_include_headers = \
-	defaults.H \
-	structure.H \
-	iterator.H \
-	storage.H \
-	Charset.H \
-	FileMapping.H \
-	SourceFile.H \
-	Input.H    \
-	diagnostics.H \
-	dialect.H \
-	vm.H \
-	token.H \
-	sexpr.H \
-	Lisp.H \
-	Constructor.H \
-	Database.H
-
 if OA_BUILD_SMAN
 OA_SMAN_TARGETS = all-sman all-clef
 endif
@@ -138,7 +117,7 @@ all-lib: stamp-subdirs all-utils
 all-lisp: all-lib all-driver
 	cd lisp && $(MAKE) $(AM_MAKEFLAGS) $@
 
-all-boot: all-lisp
+all-boot: all-syntax all-lisp
 	cd boot && $(MAKE) $(AM_MAKEFLAGS) $@
 
 all-interpsys: all-boot all-hyper-pre all-utils
@@ -183,22 +162,11 @@ all-hyper-post: all-algebra  all-doc
 all-graph: all-lib all-utils
 	cd graph && $(MAKE) $(AM_MAKEFLAGS) $@
 
-## Install include headers
-.PHONY: all-headers
-all-headers: $(patsubst %.H,$(oa_incdir)/%,$(oa_src_include_headers))
-
-$(oa_incdir)/%: $(srcdir)/include/%.H | $(oa_incdir)
-	$(RM) $@ && cp -p $< $@
-
-$(oa_incdir):
-	$(MKDIR_P) $@
-
 .PHONY: all-check
 all-check:
 	cd input && $(MAKE) $(AM_MAKEFLAGS) all-check
 
 mostlyclean-local:
-	rm -fr $(oa_incdir)
 	for d in $(subdirs); do \
 	  (cd $$d && $(MAKE) $(AM_MAKEFLAGS) mostlyclean); \
 	done
diff --git a/src/Makefile.in b/src/Makefile.in
index a09f81d7..bcea90a4 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -408,25 +408,6 @@ subdirs = \
 	etc graph hyper input interp \
 	lib io lisp share sman utils
 
-oa_incdir = $(oa_targetdir)/include/open-axiom
-oa_src_include_headers = \
-	defaults.H \
-	structure.H \
-	iterator.H \
-	storage.H \
-	Charset.H \
-	FileMapping.H \
-	SourceFile.H \
-	Input.H    \
-	diagnostics.H \
-	dialect.H \
-	vm.H \
-	token.H \
-	sexpr.H \
-	Lisp.H \
-	Constructor.H \
-	Database.H
-
 @OA_BUILD_SMAN_TRUE@OA_SMAN_TARGETS = all-sman all-clef
 @OA_BUILD_GRAPHICS_TRUE@OA_GRAPHICS_GRAPH_TARGET = all-graph
 @OA_BUILD_GUI_TRUE@OA_GRAPHICS_GUI_TARGET = all-gui
@@ -804,7 +785,7 @@ all-lib: stamp-subdirs all-utils
 all-lisp: all-lib all-driver
 	cd lisp && $(MAKE) $(AM_MAKEFLAGS) $@
 
-all-boot: all-lisp
+all-boot: all-syntax all-lisp
 	cd boot && $(MAKE) $(AM_MAKEFLAGS) $@
 
 all-interpsys: all-boot all-hyper-pre all-utils
@@ -849,21 +830,11 @@ all-hyper-post: all-algebra  all-doc
 all-graph: all-lib all-utils
 	cd graph && $(MAKE) $(AM_MAKEFLAGS) $@
 
-.PHONY: all-headers
-all-headers: $(patsubst %.H,$(oa_incdir)/%,$(oa_src_include_headers))
-
-$(oa_incdir)/%: $(srcdir)/include/%.H | $(oa_incdir)
-	$(RM) $@ && cp -p $< $@
-
-$(oa_incdir):
-	$(MKDIR_P) $@
-
 .PHONY: all-check
 all-check:
 	cd input && $(MAKE) $(AM_MAKEFLAGS) all-check
 
 mostlyclean-local:
-	rm -fr $(oa_incdir)
 	for d in $(subdirs); do \
 	  (cd $$d && $(MAKE) $(AM_MAKEFLAGS) mostlyclean); \
 	done
diff --git a/src/boot/Makefile.am b/src/boot/Makefile.am
index b24e923e..23af6c5d 100644
--- a/src/boot/Makefile.am
+++ b/src/boot/Makefile.am
@@ -58,7 +58,7 @@ LISP_LINK = \
 
 # We use a noinst_ primary because we take care of installation
 # procedure ourselves.
-noinst_PROGRAMS = bootsys
+noinst_PROGRAMS = bootsys bemol
 
 bootsys_SOURCES = \
 	utility.boot \
@@ -70,6 +70,14 @@ bootsys_SOURCES = \
 	parser.boot \
 	translator.boot
 
+bemol_SOURCES = \
+	bemol.cc
+
+bemol_LDADD = \
+	-L$(oa_target_libdir) -lOpenAxiom \
+	-L$(top_builddir)/src/syntax -lsyntax \
+	-L$(top_builddir)/src/io -lio
+
 oa_target_bootdir = $(oa_targetdir)/boot
 if OA_ECL_RT
 oa_bootsys_linkset = $(oa_target_bootdir)/linkset
@@ -89,11 +97,19 @@ bootsys_fn = $(bootsys_SOURCES:.boot=.fn)
 
 STAMP = touch
 
+AM_CXXFLAGS = \
+	-I$(top_srcdir)/src/include \
+	-I$(oa_target_includedir) \
+	-I$(top_builddir)/config \
+	-DOPENAXIOM_ROOT_DIRECTORY="\"$(open_axiom_installdir)\""
+
+
 # Make rule toplevel entry points.
 .PHONY: all-boot
 
 all-am: all-boot
 all-boot: $(oa_target_bindir)/bootsys$(EXEEXT) $(oa_bootsys_linkset)
+all-boot: bemol$(EXEEXT)
 
 # The final `bootsys' image.
 $(oa_target_bindir)/bootsys$(EXEEXT): stage2/bootsys$(EXEEXT)
diff --git a/src/boot/Makefile.in b/src/boot/Makefile.in
index d0c35a72..c55e08d3 100644
--- a/src/boot/Makefile.in
+++ b/src/boot/Makefile.in
@@ -128,10 +128,11 @@ POST_UNINSTALL = :
 build_triplet = @build@
 host_triplet = @host@
 target_triplet = @target@
-noinst_PROGRAMS = bootsys$(EXEEXT)
+noinst_PROGRAMS = bootsys$(EXEEXT) bemol$(EXEEXT)
 subdir = src/boot
 DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
-	$(top_srcdir)/config/mkinstalldirs
+	$(top_srcdir)/config/mkinstalldirs \
+	$(top_srcdir)/config/depcomp
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
 am__aclocal_m4_deps = $(top_srcdir)/config/libtool.m4 \
 	$(top_srcdir)/config/ltoptions.m4 \
@@ -146,13 +147,16 @@ CONFIG_HEADER = $(top_builddir)/config/openaxiom-c-macros.h
 CONFIG_CLEAN_FILES =
 CONFIG_CLEAN_VPATH_FILES =
 PROGRAMS = $(noinst_PROGRAMS)
-am_bootsys_OBJECTS =
-bootsys_OBJECTS = $(am_bootsys_OBJECTS)
-bootsys_LDADD = $(LDADD)
+am_bemol_OBJECTS = bemol.$(OBJEXT)
+bemol_OBJECTS = $(am_bemol_OBJECTS)
+bemol_DEPENDENCIES =
 AM_V_lt = $(am__v_lt_@AM_V@)
 am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
 am__v_lt_0 = --silent
 am__v_lt_1 = 
+am_bootsys_OBJECTS =
+bootsys_OBJECTS = $(am_bootsys_OBJECTS)
+bootsys_LDADD = $(LDADD)
 AM_V_P = $(am__v_P_@AM_V@)
 am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
 am__v_P_0 = false
@@ -166,6 +170,27 @@ am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
 am__v_at_0 = @
 am__v_at_1 = 
 DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)/config
+depcomp = $(SHELL) $(top_srcdir)/config/depcomp
+am__depfiles_maybe = depfiles
+am__mv = mv -f
+CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
+LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CXXFLAGS) $(CXXFLAGS)
+AM_V_CXX = $(am__v_CXX_@AM_V@)
+am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@)
+am__v_CXX_0 = @echo "  CXX     " $@;
+am__v_CXX_1 = 
+CXXLD = $(CXX)
+CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \
+	$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CXXLD = $(am__v_CXXLD_@AM_V@)
+am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@)
+am__v_CXXLD_0 = @echo "  CXXLD   " $@;
+am__v_CXXLD_1 = 
 COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
 	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
 LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
@@ -184,8 +209,8 @@ AM_V_CCLD = $(am__v_CCLD_@AM_V@)
 am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
 am__v_CCLD_0 = @echo "  CCLD    " $@;
 am__v_CCLD_1 = 
-SOURCES = $(bootsys_SOURCES)
-DIST_SOURCES = $(bootsys_SOURCES)
+SOURCES = $(bemol_SOURCES) $(bootsys_SOURCES)
+DIST_SOURCES = $(bemol_SOURCES) $(bootsys_SOURCES)
 am__can_run_installinfo = \
   case $$AM_UPDATE_INFO_DIR in \
     n|no|NO) false;; \
@@ -419,6 +444,14 @@ bootsys_SOURCES = \
 	parser.boot \
 	translator.boot
 
+bemol_SOURCES = \
+	bemol.cc
+
+bemol_LDADD = \
+	-L$(oa_target_libdir) -lOpenAxiom \
+	-L$(top_builddir)/src/syntax -lsyntax \
+	-L$(top_builddir)/src/io -lio
+
 oa_target_bootdir = $(oa_targetdir)/boot
 @OA_ECL_RT_FALSE@oa_bootsys_linkset = 
 @OA_ECL_RT_TRUE@oa_bootsys_linkset = $(oa_target_bootdir)/linkset
@@ -432,6 +465,12 @@ bootsys_objects = $(bootsys_SOURCES:.boot=.$(LNKEXT))
 bootsys_data = $(bootsys_SOURCES:.boot=.data)
 bootsys_fn = $(bootsys_SOURCES:.boot=.fn)
 STAMP = touch
+AM_CXXFLAGS = \
+	-I$(top_srcdir)/src/include \
+	-I$(oa_target_includedir) \
+	-I$(top_builddir)/config \
+	-DOPENAXIOM_ROOT_DIRECTORY="\"$(open_axiom_installdir)\""
+
 
 #
 # The bootstrapping `bootsys' image.
@@ -461,6 +500,7 @@ LISP_COMPILE = \
 all: all-am
 
 .SUFFIXES:
+.SUFFIXES: .cc .lo .o .obj
 $(srcdir)/Makefile.in:  $(srcdir)/Makefile.am  $(am__configure_deps)
 	@for dep in $?; do \
 	  case '$(am__configure_deps)' in \
@@ -500,6 +540,10 @@ clean-noinstPROGRAMS:
 	echo " rm -f" $$list; \
 	rm -f $$list
 
+bemol$(EXEEXT): $(bemol_OBJECTS) $(bemol_DEPENDENCIES) $(EXTRA_bemol_DEPENDENCIES) 
+	@rm -f bemol$(EXEEXT)
+	$(AM_V_CXXLD)$(CXXLINK) $(bemol_OBJECTS) $(bemol_LDADD) $(LIBS)
+
 bootsys$(EXEEXT): $(bootsys_OBJECTS) $(bootsys_DEPENDENCIES) $(EXTRA_bootsys_DEPENDENCIES) 
 	@rm -f bootsys$(EXEEXT)
 	$(AM_V_CCLD)$(LINK) $(bootsys_OBJECTS) $(bootsys_LDADD) $(LIBS)
@@ -510,6 +554,29 @@ mostlyclean-compile:
 distclean-compile:
 	-rm -f *.tab.c
 
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bemol.Po@am__quote@
+
+.cc.o:
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $<
+
+.cc.obj:
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.cc.lo:
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $<
+
 mostlyclean-libtool:
 	-rm -f *.lo
 
@@ -638,6 +705,7 @@ clean-am: clean-generic clean-libtool clean-local clean-noinstPROGRAMS \
 	mostlyclean-am
 
 distclean: distclean-am
+	-rm -rf ./$(DEPDIR)
 	-rm -f Makefile
 distclean-am: clean-am distclean-compile distclean-generic \
 	distclean-local distclean-tags
@@ -683,6 +751,7 @@ install-ps-am:
 installcheck-am:
 
 maintainer-clean: maintainer-clean-am
+	-rm -rf ./$(DEPDIR)
 	-rm -f Makefile
 maintainer-clean-am: distclean-am maintainer-clean-generic
 
@@ -723,6 +792,7 @@ uninstall-am:
 
 all-am: all-boot
 all-boot: $(oa_target_bindir)/bootsys$(EXEEXT) $(oa_bootsys_linkset)
+all-boot: bemol$(EXEEXT)
 
 # The final `bootsys' image.
 $(oa_target_bindir)/bootsys$(EXEEXT): stage2/bootsys$(EXEEXT)
diff --git a/src/boot/bemol.cc b/src/boot/bemol.cc
new file mode 100644
index 00000000..0399bb44
--- /dev/null
+++ b/src/boot/bemol.cc
@@ -0,0 +1,278 @@
+// -*- C++ -*-
+// Copyright (C) 2014, Gabriel Dos Reis.
+// All rights reserved.
+// Written by Gabriel Dos Reis.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     - Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//
+//     - Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in
+//       the documentation and/or other materials provided with the
+//       distribution.
+//
+//     - Neither the name of OpenAxiom. nor the names of its contributors
+//       may be used to endorse or promote products derived from this
+//       software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+// OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// --% Author: Gabriel Dos Reis
+// --% Description:
+
+#include <open-axiom/diagnostics>
+#include <open-axiom/token>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <string>
+#include <stack>
+#include <iterator>
+#include <ctype.h>
+
+using namespace OpenAxiom;
+
+// 
+// -- Reading input source files --
+// 
+
+// A physical line is just raw text, with coupled with location
+// information such as line and indentation column.
+struct Line : std::string {
+   LineNumber number;
+   ColumnIndex indent;
+   Line() : number(), indent() { }
+
+   std::string sub_string(ColumnIndex s, ColumnIndex e) const {
+      return substr(s, e - s);
+   }
+};
+
+// A program fragment is a logical line, composed of possibly
+// several physical lines subject to the off-side rule.  As a
+// special case, a line ending with the underbar character
+// continues to the next next with disregard to the off-side rule.
+struct Fragment : std::vector<Line> {
+   explicit operator bool() const { return not empty(); }
+   bool line_continuation() const {
+      return not empty() and back().back() == '_';
+   }
+   ColumnIndex last_indent() const {
+      return empty() ? 0 : back().indent;
+   }
+};
+
+// Formatting program fragments.
+static std::ostream&
+operator<<(std::ostream& os, const Fragment& f) {
+   std::copy(f.begin(), f.end(),
+             std::ostream_iterator<std::string>(std::cout, "\n"));
+   return os;
+}
+
+
+// A source input transform a character stream into a program fragment
+// stream, delivering a fragment one at a time.
+struct SourceInput {
+   SourceInput(std::istream& is) : input(is) { }
+   Fragment get();
+
+private:
+   std::istream& input;
+   Line line;
+};
+
+// Return the indentation level of a line.
+// FIXME: reject or expand tabs as appropriate.
+static ColumnIndex
+indentation(const Line& line) {
+   ColumnIndex idx { };
+   for (auto c : line) {
+      if (not isspace(c))
+         break;
+      ++idx;
+   }
+   return idx;
+}
+
+// Remove trailing white-space characters from the line.
+static Line&
+trim_right(Line& line) {
+   auto n = line.length();
+   while (n > 0 and isspace(line[n-1]))
+      --n;
+   line.resize(n);
+   return line;
+}
+
+// Clean up and dress up the line with indentation information.
+static Line&
+prop_up(Line& line) {
+   line.indent = indentation(trim_right(line));
+   return line;
+}
+
+// Return true if line is entirely a negative comment.
+static bool
+negative_comment(const Line& line) {
+   if (line.indent + 1 >= line.length())
+      return false;
+   return line[line.indent] == '-' and line[line.indent + 1] == '-';
+}
+
+// Return true if line is either empty or a negative comment.
+static bool
+blank(const Line& line) {
+   return line.empty() or negative_comment(line);
+}
+
+// Return true if line is entirely a positive comment, i.e. a description.
+static bool
+positive_comment(const Line& line) {
+   if (line.indent + 1 >= line.length())
+      return false;
+   return line[line.indent] == '+' and line[line.indent + 1] == '+';
+}
+
+// Decompose the input souce file into fragments, and return one
+// fragment at a time.
+Fragment
+SourceInput::get() {
+   Fragment fragment;
+   std::stack<ColumnIndex> indents;
+
+   if (not line.empty()) {
+      indents.push(line.indent);
+      fragment.push_back(line);
+   }
+
+   while (std::getline(input, line)) {
+      ++line.number;
+      if (blank(prop_up(line)))
+         continue;              // Don't bother with ignorable comments.
+      else if (fragment.line_continuation())
+         ;
+      else if (indents.empty()) {
+         if (fragment.empty() and line.indent != 0)
+            std::cout << "warning: white space at begining of fragment"
+                      << " on line " << line.number << '\n';
+         indents.push(line.indent);
+      }
+      else if (line.indent == 0 and not positive_comment(fragment.back()))
+         break;                 // A completely new line; save for later.
+      else if (line.indent > indents.top())
+         indents.push(line.indent);
+      else {
+         while (line.indent < indents.top())
+            indents.pop();
+      }
+      fragment.push_back(line);
+   }
+   return fragment;
+}
+
+// 
+// -- Decomposing source files into lexical units of information --
+// 
+
+struct Locus {
+   OpenAxiom::LineNumber line;
+   OpenAxiom::ColumnIndex column;
+};
+
+static std::ostream&
+operator<<(std::ostream& os, const Locus& l)
+{
+   os << '{' << l.line << ", " << l.column << '}';
+   return os;
+}
+
+struct BemolToken {
+   using Location = ::Locus;
+   OpenAxiom::TokenCategory category;
+   OpenAxiom::TokenValue value;
+   Locus start;
+   Locus end;
+
+   explicit operator bool() const { return category != TokenCategory::EOS; }
+};
+
+static std::ostream&
+operator<<(std::ostream& os, const BemolToken& t) {
+   os << t.category << '{' << t.start << '-' << t.end << '}';
+   return os;
+}
+
+static void
+translate_source_file(SourceInput& src, std::ostream& out, const char* path) {
+   while (auto f = src.get()) {
+      out << "================================================\n";
+      out << f;
+      OpenAxiom::TokenStream<Fragment, BemolToken> ts { f };
+      try {
+         while (auto t = ts.get(OpenAxiom::Language::Boot)) {
+            out << '\t' << t;
+            switch (t.category) {
+            case TokenCategory::Junk:
+            case TokenCategory::Unclassified:
+               out //<< f[t.start.line].sub_string(t.start.column, t.end.column)
+                   << " in file " << path
+                   << " at line " << t.start.line
+                   << ", column " << t.start.column;
+               break;
+            default:
+               break;
+            }
+            out << '\n';
+         }
+      }
+      catch(const OpenAxiom::EndOfStringUnseen& e) {
+         std::cerr << path << ": syntax error: "
+                   << "premature end of line before matching quote "
+                   << "of string literal on line " << e.line
+                   << " at column " << e.column
+                   << std::endl;
+      }
+      catch (const OpenAxiom::MissingExponent& e) {
+         std::cerr << path << ": syntax error: "
+                   << "missing exponent of floating point constant "
+                   << "on line " << e.line
+                   << ", column " << e.column
+                   << std::endl;
+      }
+      out << "================================================\n";
+   }
+   out << std::flush;
+}
+
+static void
+process_file(const char* path) {
+   std::ifstream in { path };
+   if (!in) {
+      std::cerr << "error: could not open file `" << path << "'"
+                << std::endl;
+      return;
+   }
+   SourceInput src { in };
+   translate_source_file(src, std::cout, path);
+}
+
+int main(int argc, char* argv[]) {
+   for (int i = 1; i < argc; ++i) {
+      process_file(argv[i]);
+   }
+}
diff --git a/src/include/dialect.H b/src/include/dialect.H
index f63eac04..bcfddd04 100644
--- a/src/include/dialect.H
+++ b/src/include/dialect.H
@@ -1,4 +1,4 @@
-// Copyright (C) 2013, Gabriel Dos Reis.
+// Copyright (C) 2013-2014, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
 //
@@ -36,7 +36,11 @@
 namespace OpenAxiom {
    // Languages for which we have parsers.
    enum class Language {
-      Spad, Boot, Lisp
+      Spad = 0x1,
+      Boot = 0x2,
+      Lisp = 0x4,
+      BootSpad = Spad | Boot,
+      All = Spad | Boot | Lisp,
    };
 }
 
diff --git a/src/include/sexpr.H b/src/include/sexpr.H
index d425b6d8..84513a8b 100644
--- a/src/include/sexpr.H
+++ b/src/include/sexpr.H
@@ -1,4 +1,4 @@
-// Copyright (C) 2010-2013, Gabriel Dos Reis.
+// Copyright (C) 2010-2014, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
 //
@@ -55,28 +55,27 @@ namespace OpenAxiom {
       struct Lexeme {
          enum Type {
             unknown,                 // unidentified token
-            semicolon        = token::value(";"), // comment
-            dot              = token::value("."),
-            comma            = token::value(","),
-            open_paren       = token::value("("),
-            close_paren      = token::value(")"),
-            apostrophe       = token::value("'"),
-            backquote        = token::value("`"),
-            backslash        = token::value("\\"),
-            sharp_open_paren = token::value("#("),
-            sharp_apostrophe = token::value("#'"),
-            sharp_colon      = token::value("#:"),
-            sharp_plus       = token::value("#+"),
-            sharp_minus      = token::value("#-"),
-            sharp_dot        = token::value("#."),
-            comma_at         = token::value(",@"),
-            digraph_end      = token::value(0xff,0xff),
-            integer,                // integer literal
-            character,              // character literal
-            string,                 // string literal
-            identifier,             // plain identifier
-            sharp_integer_equal,    // anchor definition, #n=<form>
-            sharp_integer_sharp     // back reference, #n#
+            semicolon,               // ";" for comment
+            dot,                     // "."
+            comma,                   // ","
+            open_paren,              // "("
+            close_paren,             // ")"
+            apostrophe,              // "'"
+            backquote,               // "`"
+            backslash,               // "\\"
+            sharp_open_paren ,       // "#("
+            sharp_apostrophe,        // "#'"
+            sharp_colon,             // "#:"
+            sharp_plus,              // "#+"
+            sharp_minus,             // "#-"
+            sharp_dot,               // "#."
+            comma_at,                // ",@"
+            integer,                 // integer literal
+            character,               // character literal
+            string,                  // string literal
+            identifier,              // plain identifier
+            sharp_integer_equal,     // anchor definition, #n=<form>
+            sharp_integer_sharp      // back reference, #n#
          };
 
          std::pair<const Byte*, const Byte*> boundary;
diff --git a/src/include/token-value.def b/src/include/token-value.def
new file mode 100644
index 00000000..ea79c9a5
--- /dev/null
+++ b/src/include/token-value.def
@@ -0,0 +1,138 @@
+// Copyright (C) 2014, Gabriel Dos Reis.
+// All rights reserved.
+// Written by Gabriel Dos Reis.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     - Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//
+//     - Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in
+//       the documentation and/or other materials provided with the
+//       distribution.
+//
+//     - Neither the name of OpenAxiom. nor the names of its contributors
+//       may be used to endorse or promote products derived from this
+//       software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+// OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+
+OPENAXIOM_DEFINE_TOKEN(Unknown, "<unknown>", Unclassified, Language::All)
+OPENAXIOM_DEFINE_TOKEN(Apostrophe, "'", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(Backquote, "`", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(Bar, "|", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Dot, ".", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(DotDot, "..", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Colon, ":", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(ColonColon, "::", Operator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(ColonDash, ":-", Operator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(ColonEq, ":=", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(At, "@", Operator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(Exclamation, "!", Punctuator, Language::Boot)
+OPENAXIOM_DEFINE_TOKEN(Comma, ",", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(Semicolon, ";", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(Star, "*", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(StarStar, "**", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Plus, "+", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Minus, "-", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Slash, "/", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(SlashSlash, "//", Operator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(SlashBackslash, "/\\", Operator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Backslash, "\\", Operator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(BackslashSlash, "\\/", Operator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(BackslashBackslash, "\\\\", Operator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Less, "<", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(LessEq, "<=", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Greater, ">", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(GreaterEq, ">=", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Eq, "=", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(EqEq, "==", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Tilde, "~", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(TildeEq, "~=", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Caret, "^", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Pound, "#", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Dollar, "$", Operator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Ampersand, "&", Operator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(RightArrow, "->", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(LeftArrow, "<-", Operator, Language::Boot)
+OPENAXIOM_DEFINE_TOKEN(Implies, "=>", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Equiv, "<=>", Keyword, Language::Boot)
+OPENAXIOM_DEFINE_TOKEN(MapsTo, "+->", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(FatArrow, "==>", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(OpenParen, "(", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(CloseParen, ")", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(OpenMetaParen, "(|", Punctuator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(CloseMetaParen, "|)", Punctuator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(OpenBracket, "[", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(CloseBracket, "]", Punctuator, Language::All)
+OPENAXIOM_DEFINE_TOKEN(OpenMetaBracket, "[|", Punctuator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(CloseMetaBracket, "|]", Punctuator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(OpenBrace, "{", Punctuator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(CloseBrace, "}", Punctuator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(OpenMetaBrace, "{|", Punctuator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(CloseMetaBrace, "|}", Punctuator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(OpenChevron, "<<", Operator, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(CloseChevron, ">>", Operator, Language::Spad)
+
+OPENAXIOM_DEFINE_TOKEN(Wisecrack, "--", Comment, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Commentary, "++", Comment, Language::BootSpad)
+
+OPENAXIOM_DEFINE_TOKEN(Add, "add", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(And, "and", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Assume, "assume", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Break, "break", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(By, "by", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Case, "case", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Catch, "catch", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Cross, "cross", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Do, "do", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Else, "else", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Exists, "exists", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Finally, "finally", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(For, "for", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Forall, "forall", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(From, "from", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Function, "function", Keyword, Language::Boot)
+OPENAXIOM_DEFINE_TOKEN(Has, "has", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(If, "if", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Import, "import", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(In, "in", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Inline, "inline", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Is, "is", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Isnt, "isnt", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Iterate, "iterate", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Leave, "leave", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Macro, "macro", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Mod, "mod", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Namespace, "namespace", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Of, "of", Keyword, Language::Boot)
+OPENAXIOM_DEFINE_TOKEN(Or, "or", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Pretend, "pretend", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Quo, "quo", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Rem, "rem", Operator, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Repeat, "repeat", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Return, "return", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Rule, "rule", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Structure, "structure", Keyword, Language::Boot)
+OPENAXIOM_DEFINE_TOKEN(Then, "then", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(Throw, "throw", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Try, "try", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Until, "until", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(With, "with", Keyword, Language::Spad)
+OPENAXIOM_DEFINE_TOKEN(Where, "where", Keyword, Language::BootSpad)
+OPENAXIOM_DEFINE_TOKEN(While, "while", Keyword, Language::BootSpad)
diff --git a/src/include/token.H b/src/include/token.H
index ef203b12..3b3b2950 100644
--- a/src/include/token.H
+++ b/src/include/token.H
@@ -1,4 +1,4 @@
-// Copyright (C) 2013, Gabriel Dos Reis.
+// Copyright (C) 2013-2014, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
 //
@@ -34,151 +34,556 @@
 #define OPENAXIOM_TOKEN_included
 
 #include <stdint.h>
+#include <stack>
+#include <iosfwd>
 #include <open-axiom/Input>
+#include <open-axiom/dialect>
 
 namespace OpenAxiom {
-   namespace token {
-      // -- Underlying representation of a token class.
-      using base_type = uint32_t;
-
-      // -- 8-bit byte data type
-      using u8 = uint8_t;
-
-      constexpr base_type value(u8 c) { return c; }
-      constexpr base_type value(u8 hi, u8 lo) { return (hi << 8) | lo; }
-      constexpr base_type value(u8 hi, u8 mi, u8 lo) {
-         return (value(hi, mi) << 8) | lo;
-      }
-
-      // -- Type of literal strings of given number of characters.
-      template<int N>
-      using text_chunk = const char(&)[N+1];
-
-      // -- Return the token value of certain literal strings.
-      constexpr base_type value(text_chunk<0>) { return u8(); }
-      constexpr base_type value(text_chunk<1> s) {
-         return value(s[0]);
-      }
-      constexpr base_type value(text_chunk<2> s) {
-         return value(s[0], s[1]);
-      }
-      constexpr base_type value(text_chunk<3> s) {
-         return value(s[0], s[1], s[2]);
-      }
-
-      // -- Abstract values of tokens.
-      enum Value : base_type {
-         Unknown                = value(""),
-         Bar                    = value("|"),
-         Dot                    = value("."),
-         DotDot                 = value(".."),
-         Colon                  = value(":"),
-         ColonColon             = value("::"),
-         ColonDash              = value(":-"),
-         ColonEq                = value(":="),
-         At                     = value("@"),
-         Comma                  = value(","),
-         Semicolon              = value(";"),
-         Star                   = value("*"),
-         Plus                   = value("+"),
-         Minus                  = value("-"),
-         Slash                  = value("/"),
-         Backslash              = value("\\"),
-         SlashSlash             = value("//"),
-         BackslashBackslash     = value("\\\\"),
-         BackslashSlash         = value("\\/"),
-         SlashBackslash         = value("/\\"),
-         Less                   = value("<"),
-         LessEq                 = value("<="),
-         Greater                = value(">"),
-         GreaterEq              = value(">="),
-         Eq                     = value("="),
-         EqEq                   = value("=="),
-         Tilde                  = value("~"),
-         TildeEq                = value("~="),
-         Caret                  = value("^"),
-         Pound                  = value("#"),
-         Dollar                 = value("$"),
-         Ampersand              = value("&"),
-         OpenParen              = value("("),
-         CloseParen             = value(")"),
-         OpenBracket            = value("["),
-         CloseBracket           = value("]"),
-         OpenBrace              = value("{"),
-         CloseBrace             = value("}"),
-         OpenMetParen           = value("(|"),
-         CloseMetaParen         = value("|)"),
-         OpenMetaBracket        = value("[|"),
-         CloseMetaBracket       = value("|]"),
-         OpenMetaBrace          = value("{|"),
-         CloseMetaBrace         = value("|}"),
-         Apostrophe             = value("'"),
-         Backquote              = value("`"),
-         StarStar               = value("**"),
-         Implies                = value("=>"),
-         RightArrow             = value("->"),
-         LeftArrow              = value("<-"),
-         OpenChevron            = value("<<"),
-         CloseChevron           = value(">>"),
-         FatArrow               = value("==>"),
-         Equiv                  = value("<=>"),
-         MapsTo                 = value("+->"),
-
-         Add                    = value("add"),
-         And                    = value("and"),
-         By                     = value("by"),
-         Do                     = value("do"),
-         For                    = value("for"),
-         Has                    = value("has"),
-         If                     = value("if"),
-         In                     = value("in"),
-         Is                     = value("is"),
-         Mod                    = value("mod"),
-         Of                     = value("of"),    // -- Boot only
-         Or                     = value("or"),
-         Quo                    = value("quo"),
-         Rem                    = value("rem"),
-         Try                    = value("try"),
-         LastTrigraph           = 0xffffff,
-
-         Assume,             // "assume"
-         Break,              // "break"
-         Case,               // "case"
-         Catch,              // "catch"
-         Cross,              // "cross"
-         Else,               // "else"
-         Exists,             // "exists"
-         Finally,            // "finally"
-         From,               // "from"
-         Forall,             // "forall"
-         Function,           // "function" -- Boot only
-         Import,             // "import"
-         Inline,             // "inline"
-         Isnt,               // "isnt"
-         Iterate,            // "iterate"
-         Leave,              // "leave"
-         Macro,              // "macro"
-         Module,             // "module"   -- Boot only
-         Namespace,          // "namespace" -- Boot only
-         Pretend,            // "pretend"
-         Repeat,             // "repeat"
-         Return,             // "return"
-         Rule,               // "rule"
-         Structure,          // "structure" -- Boot only
-         Then,               // "then"
-         Throw,              // "throw"
-         Until,              // "until"
-         With,               // "with"
-         Where,              // "where"
-         While,              // "while"
-
-         IntegerLiteral,     // integer literal
-         StringLiteral,      // string literal
-         FPLiteral,          // floating point literal
-         Indent,             // new line indentation, greater than previous
-         Unindent,           // new line indentation, less than previous
-         Justify,            // align indentation with preceding line.
-      };
+   // Categorization of Boot and Spad tokens.
+   enum class TokenCategory : uint8_t {
+      Unclassified,             // token of unknown class
+      Whitespace,               // sequence of white-space characters
+      Comment,                  // a description of an ignorable comment
+      Punctuator,               // a punctuator character
+      Operator,                 // an operator both symbolic and alphabetic
+      Integer,                  // an integer literal
+      FloatingPoint,            // a floating-point literal
+      String,                   // a string literal
+      Keyword,                  // a reserved word both symbolic and alphabetic
+      Identifier,               // an identifier
+      Formatting,               // a layout formatting token
+      Junk,                     // invalid/malformed token
+      EOS                       // end-of-token-stream indicator
+   };
+
+   std::ostream& operator<<(std::ostream&, TokenCategory);
+
+   // The abstract value associated with a token.
+   enum class TokenValue : uint8_t {
+#undef OPENAXIOM_DEFINE_TOKEN
+#define OPENAXIOM_DEFINE_TOKEN(T, ...)  T,
+#include <open-axiom/token-value>
+#undef OPENAXIOM_DEFINE_TOKEN
+      Artificial,               // Tokens after this are artificial
+      Indent,                   // new line indentation, greater than previous
+      Unindent,                 // new line indentation, less than previous
+      Justify,                  // align indentation with preceding line.
+
+      EndOfStream               // end of token stream
+   };
+
+   std::ostream& operator<<(std::ostream&, TokenValue);
+
+   // Given a symbolic or alphabetic token, retrieve its category
+   // and associated abstract value.
+   struct TokenClassification {
+      TokenCategory category;
+      TokenValue value;
+
+      explicit operator bool() const {
+         return category != TokenCategory::Unclassified;
+      }
+   };
+
+   TokenClassification classify(const std::string&);
+
+   // Datatypes for locating lines and columns.
+   using LineNumber = std::size_t;
+   using ColumnIndex = std::size_t;
+
+   // -- Exception types
+   struct EndOfStringUnseen {
+      LineNumber line;
+      ColumnIndex column;
+   };
+
+   struct MissingExponent {
+      LineNumber line;
+      ColumnIndex column;
+   };
+
+   // Object of this datatype decompose a program fragment into a
+   // token stream.  The tokens are of type indicated by Tok.
+   template<typename Frag, typename Tok>
+   struct TokenStream {
+      TokenStream(Frag& f)
+            : frag(f),
+              line(),
+              idx(frag.front().indent)
+      {
+         indents.push(idx);
+      }
+
+      bool eos() const {
+         return line >= frag.size()
+            or (line + 1 == frag.size() and idx >= frag.back().size());
+      }
+
+      Tok get(Language = Language::Spad);
+   private:
+      Frag& frag;
+      std::size_t line;
+      std::size_t idx;
+      std::stack<ColumnIndex> indents;
+
+      std::size_t line_length() const { return frag[line].size(); }
+      LineNumber next_line_number() const {
+         return line + 1 < frag.size()
+            ? frag[line + 1].number
+            : frag.back().number + 1;
+      }
+      ColumnIndex next_indentation() const {
+         return line + 1 < frag.size() ? frag[line + 1].indent : 0;
+      }
+
+      LineNumber line_number() const {
+         return line < frag.size()
+            ? frag[line].number
+            : frag.back().number + 1;
+      }
+
+      ColumnIndex column_number() const {
+         return line < frag.size() ? idx : 0;
+      }
+
+      using Locus = typename Tok::Location;
+      Locus current_locus() {
+         return { line_number(), column_number() };
+      }
+   };
+
+   bool separator_or_punctuator(uint8_t);
+
+   template<typename L, typename T>
+   static void junk(L& line, ColumnIndex& idx, T& t) {
+      while (idx < line.size() and not separator_or_punctuator(line[idx]))
+         ++idx;
+      t.category = TokenCategory::Junk;
+   }
+
+   template<typename L>
+   inline void
+   skip_whitespace(L& line, ColumnIndex& idx) {
+      while (idx < line.size() and isspace(line[idx]))
+         ++idx;
+   }
+
+   template<typename L, typename T>
+   void string(L& line, ColumnIndex& idx, T& t) {
+      bool done = false;
+      bool escape = false;
+      while (idx < line.size() && not done) {
+         switch (line[idx++]) {
+         case '_': escape = !escape; break;
+         case '"': done = !escape;
+            // fallthrough
+         default: escape = false; break;
+         }
+      }
+      if (not done)
+         throw EndOfStringUnseen{ line.number, idx };
+      t.category = TokenCategory::String;
+   }
+
+   template<typename L>
+   void skip_to_end_of_integer(L& line, ColumnIndex& idx) {
+      while (idx < line.size() and isdigit(line[idx]))
+         ++idx;
+   }
+   
+   template<typename L, typename T>
+   void integer(L& line, ColumnIndex& idx, T& t) {
+      skip_to_end_of_integer(line, idx);
+      t.category = TokenCategory::Integer;
+   }
+
+   template<typename L, typename T>
+   T& number(L& line, ColumnIndex& idx, T& t) {
+      integer(line, idx, t);
+      if (idx >= line.size() or line[idx] != '.')
+         return t;
+      if (++idx >= line.size() or not isdigit(line[idx])) {
+         --idx;
+         return t;
+      }
+
+      t.category = TokenCategory::FloatingPoint;
+      skip_to_end_of_integer(line, idx);
+      if (idx >= line.size() or (line[idx] != 'e' and line[idx] != 'E'))
+         return t;
+      if (++idx < line.size() and (line[idx] == '+' or line[idx] == '-'))
+         ++idx;
+      if (idx >= line.size() or not isdigit(line[idx]))
+         throw MissingExponent{ line.number, idx };
+      skip_to_end_of_integer(line, idx);
+      return t;
+   }
+
+   inline bool
+   identifier_head(uint8_t c) {
+      return isalpha(c) or c == '%' or c == '_';
+   }
+
+   inline bool
+   identifier_part(uint8_t c) {
+      return identifier_head(c) or isdigit(c);
+   }
+
+   inline bool
+   identifier_suffix(uint8_t c) {
+      return c == '!' or c == '?';
+   }
+
+   inline bool internal_prefix(uint8_t c) {
+      return c == '%' or c == '$';
+   }
+
+   template<typename L>
+   inline void
+   skip_prefix(L& line, ColumnIndex& idx, uint8_t c) {
+      while (idx < line.size() and line[idx] == c)
+         ++idx;
+   }
+
+   template<typename L, typename T>
+   T& identifier(L& line, ColumnIndex& idx, T& t, Language dialect) {
+      t.category = TokenCategory::Identifier;
+
+      ColumnIndex start = --idx; // idx was ahead by 1.
+      if (dialect == Language::Boot and internal_prefix(line[idx]))
+         skip_prefix(line, idx, line[idx]);
+      bool saw_escape = false;
+      while (idx < line.size()) {
+         if (not identifier_part(line[idx]) and line[idx - 1] != '_')
+            break;
+         else if (line[idx] == '_')
+            saw_escape = true;
+         ++idx;
+      }
+      while (idx < line.size() and identifier_suffix(line[idx]))
+         ++idx;
+
+      if (saw_escape)
+         t.category = TokenCategory::Identifier;
+      else if (auto info = classify(line.sub_string(start, idx))) {
+         t.category = info.category;
+         t.value = info.value;
+      }
+      return t;
+   }
+
+   template<typename Frag, typename Tok>
+   Tok TokenStream<Frag, Tok>::get(Language dialect) {
+      Tok t { };
+      t.start = current_locus();
+      
+      if (eos()) {
+         t.category = TokenCategory::EOS;
+         t.end = current_locus();
+         return t;
+      }
+      else if (isspace(frag[line][idx])) {
+         skip_whitespace(frag[line], idx);
+         t.category = TokenCategory::Whitespace;
+         t.end = current_locus();
+         return t;
+      }
+      else if (idx == line_length() - 1 and frag[line].back() == '_') {
+         ++line;
+         idx = frag[line].indent;
+      }
+      else if (idx == line_length()) {
+         auto indent = indents.top();
+         auto next_indent = next_indentation();
+         t.start = t.end = { next_line_number(), next_indent };
+         if (indent < next_indent) {
+            indents.push(next_indent);
+            ++line;
+            idx = next_indent;
+            t.category = TokenCategory::Formatting;
+            t.value = TokenValue::Indent;
+         }
+         else if (indent > next_indent) {
+            indents.pop();
+            t.category = TokenCategory::Formatting;
+            t.value = TokenValue::Unindent;
+         }
+         else {
+            ++line;
+            idx = next_indent;
+            t.category = TokenCategory::Formatting;
+            t.value = TokenValue::Justify;
+         }
+         return t;
+      }
+
+      switch (auto c = frag[line][idx++]) {
+      case '#':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Pound;
+         break;
+
+      case '@':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::At;
+         break;
+
+      case '^':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Caret;
+         break;
+
+      case '&':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::Ampersand;
+         break;
+         
+      case '!':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::Exclamation;
+         break;
+         
+      case '\'':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::Apostrophe;
+         break;
+      case ',':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::Comma;
+         break;
+         
+      case ';':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::Semicolon;
+         break;
+         
+      case '`':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::Backquote;
+         break;
+         
+      case '(':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::OpenParen;
+         if (idx < line_length() and frag[line][idx] == '|') {
+            ++idx;
+            t.value = TokenValue::OpenMetaParen;
+         }
+         break;
+         
+      case ')':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::CloseParen;
+         break;
+         
+      case '{':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::OpenBrace;
+         if (idx < line_length() and frag[line][idx] == '|') {
+            ++idx;
+            t.value = TokenValue::OpenMetaBrace;
+         }
+         break;
+
+      case '}':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::CloseBrace;
+         break;
+
+      case '[':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::OpenBracket;
+         if (idx < line_length() and frag[line][idx] == '|') {
+            ++idx;
+            t.value = TokenValue::OpenMetaBracket;
+         }
+         break;
+         
+      case ']':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::CloseBracket;
+         break;
+         
+      case ':':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Colon; 
+         if (idx < line_length())
+            switch (frag[line][idx]) {
+            case ':': t.value = TokenValue::ColonColon; ++idx; break;
+            case '=': t.value = TokenValue::ColonEq; ++idx; break;
+            case '-': t.value = TokenValue::ColonDash; ++idx; break;
+            default: break;
+            }
+         break;
+         
+      case '*':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Star;
+         if (idx < line_length() and frag[line][idx] == '*') {
+            t.value = TokenValue::StarStar;
+            ++idx;
+         }
+         break;
+         
+      case '/':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Slash;
+         if (idx < line_length())
+            switch (frag[line][idx]) {
+            case '/': t.value = TokenValue::SlashSlash; ++idx; break;
+            case '\\': t.value = TokenValue::SlashBackslash; ++idx; break;
+            default: break;
+            }
+         break;
+
+      case '\\':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Backslash;
+         if (idx < line_length())
+            switch (frag[line][idx]) {
+            case '\\': t.value = TokenValue::BackslashBackslash; ++idx; break;
+            case '/': t.value = TokenValue::BackslashSlash; ++idx; break;
+            default: break;
+            }
+         break;
+         
+      case '<':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Less;
+         if (idx < line_length())
+            switch (frag[line][idx]) {
+            case '-': t.value = TokenValue::LeftArrow; ++idx; break;
+            case '<': t.value = TokenValue::OpenChevron; ++idx; break;
+            case '=':
+               t.value = TokenValue::LessEq;
+               if (++idx < line_length() and frag[line][idx] == '>') {
+                  t.value = TokenValue::Equiv;
+                  ++idx;
+               }
+               break;
+            default: break;
+            }
+         break;
+         
+      case '=':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Eq;
+         if (idx < line_length())
+            switch (frag[line][idx]) {
+            case '>': t.value = TokenValue::Implies; ++idx; break;
+            case '=':
+               t.value = TokenValue::EqEq;
+               if (++idx < line_length() and frag[line][idx] == '>') {
+                  t.value = TokenValue::FatArrow;
+                  ++idx;
+               }
+               break;
+            default: break;
+            }
+         break;
+         
+      case '~':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Tilde;
+         if (idx < line_length() and frag[line][idx] == '=') {
+            t.value = TokenValue::TildeEq;
+            ++idx;
+         }
+         break;
+
+      case '>':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Greater;
+         if (idx < line_length())
+            switch (frag[line][idx]) {
+            case '=': t.value = TokenValue::GreaterEq; ++idx; break;
+            case '>': t.value = TokenValue::CloseChevron; ++idx; break;
+            }
+         break;
+
+      case '|':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Bar;
+         if (idx < line_length())
+            switch (frag[line][idx]) {
+            case ']': t.value = TokenValue::CloseMetaBracket; ++idx; break;
+            case '}': t.value = TokenValue::CloseMetaBrace; ++idx; break;
+            case ')': t.value = TokenValue::CloseMetaParen; ++idx; break;
+            default: break;
+            }
+         break;
+
+      case '-':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Minus; 
+         if (idx < line_length())
+            switch (frag[line][idx]) {
+            case '>': t.value = TokenValue::RightArrow; ++idx; break;
+            case '-':
+               t.category = TokenCategory::Comment;
+               t.value = TokenValue::Wisecrack;
+               idx = frag[line].size();
+               break;
+            }
+         break;
+
+      case '+':
+         t.category = TokenCategory::Operator;
+         t.value = TokenValue::Plus;
+         if (idx < line_length())
+            switch (frag[line][idx]) {
+            case '+':
+               t.category = TokenCategory::Comment;
+               t.value = TokenValue::Commentary;
+               idx = frag[line].size();
+               break;
+            case '-':
+               if (idx + 1 < line_length() and frag[line][idx+1] == '>') {
+                  t.value = TokenValue::MapsTo;
+                  idx += 2;
+               }
+               break;
+            default: break;
+            }
+         break;
+
+      case '.':
+         t.category = TokenCategory::Punctuator;
+         t.value = TokenValue::Dot;
+         if (idx < line_length() and frag[line][idx] == '.') {
+            t.category = TokenCategory::Operator;
+            t.value = TokenValue::DotDot;
+            ++idx;
+         }
+         break;
+
+      case '"':
+         string(frag[line], idx, t);
+         break;
+
+      case '$':
+         if (dialect != Language::Boot or idx >= line_length()
+             or separator_or_punctuator(frag[line][idx])) {
+            t.category = TokenCategory::Operator;
+            t.value = TokenValue::Dollar;
+         }
+         else
+            identifier(frag[line], idx, t, dialect);
+         break;
+
+      default:
+         if (isdigit(c))
+            number(frag[line], idx, t);
+         else if (identifier_head(c))
+            identifier(frag[line], idx, t, dialect);
+         else
+            junk(frag[line], idx, t);
+         break;
+      }
+
+      t.end = { frag[line].number, idx };
+      return t;
    }
 }
 
diff --git a/src/interp/simpbool.boot b/src/interp/simpbool.boot
index 3432317d..e904869a 100644
--- a/src/interp/simpbool.boot
+++ b/src/interp/simpbool.boot
@@ -175,7 +175,7 @@ list3 l ==
 orDel(a,l) ==
   l is [h,:t] =>
     a = h => t
-    ?ORDER(a,h) => [a,:l]
+    _?ORDER(a,h) => [a,:l]
     [h,:orDel(a,t)]
   [a]
 ordList l ==
@@ -185,7 +185,7 @@ ordUnion(a,b) ==
   a isnt [c,:r] => b
   b isnt [d,:s] => a
   c=d => [c,:ordUnion(r,s)]
-  ?ORDER(a,b) => [c,:ordUnion(r,b)]
+  _?ORDER(a,b) => [c,:ordUnion(r,b)]
   [d,:ordUnion(s,a)]
 ordIntersection(a,b) ==
   a isnt [h,:t] => nil
diff --git a/src/syntax/token.cc b/src/syntax/token.cc
index b1db7ec1..fe3cd7a7 100644
--- a/src/syntax/token.cc
+++ b/src/syntax/token.cc
@@ -31,58 +31,82 @@
 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <open-axiom/token>
+#include <ostream>
+#include <iostream>
 
 namespace OpenAxiom {
-   namespace token {
-      struct Keyword {
-         const char* const key;
-         const Value value;
-      };
+   std::ostream&
+   operator<<(std::ostream& os, TokenCategory tc) {
+      switch (tc) {
+      case TokenCategory::Unclassified: os << "UNCLASSIFIED"; break;
+      case TokenCategory::Whitespace: os << "WHITESPACE"; break;
+      case TokenCategory::Comment: os << "COMMENT"; break;
+      case TokenCategory::Punctuator: os << "PUNCTUATOR"; break;
+      case TokenCategory::Operator: os << "OPERATOR"; break;
+      case TokenCategory::Integer: os << "INTEGER"; break;
+      case TokenCategory::FloatingPoint: os << "FLOATINGPOINT"; break;
+      case TokenCategory::String: os << "STRING"; break;
+      case TokenCategory::Keyword: os << "KEYWORD"; break;
+      case TokenCategory::Identifier: os << "IDENTIFIER"; break;
+      case TokenCategory::Formatting: os << "FORMATTING"; break;
+      case TokenCategory::Junk: os << "JUNK"; break;
+      default: os << "????"; break;
+      }
+      return os;
+   }
+
 
-      const Keyword keyword_map[] = {
-         { "add", Value::Add },
-         { "and", Value::And },
-         { "assume", Value::Assume },
-         { "break", Value::Break },
-         { "by", Value::By },
-         { "case", Value::Case },
-         { "catch", Value::Catch },
-         { "do", Value::Do },
-         { "else", Value::Else },
-         { "exist", Value::Exists },
-         { "finally", Value::Finally },
-         { "for", Value::For },
-         { "from", Value::From },
-         { "function", Value::Function },
-         { "has", Value::Has },
-         { "if", Value::If },
-         { "import", Value::Import },
-         { "in", Value::In },
-         { "inline", Value::Inline },
-         { "is", Value::Is },
-         { "isnt", Value::Isnt },
-         { "iterate", Value::Iterate },
-         { "leave", Value::Leave },
-         { "macro", Value::Macro },
-         { "mod", Value::Mod },
-         { "module", Value::Module },
-         { "namespace", Value::Namespace },
-         { "of", Value::Of },
-         { "or", Value::Or },
-         { "pretend", Value::Pretend },
-         { "quo", Value::Quo },
-         { "rem", Value::Rem },
-         { "repeat", Value::Repeat },
-         { "return", Value::Return },
-         { "rule", Value::Rule },
-         { "structure", Value::Structure },
-         { "then", Value::Then },
-         { "throw", Value::Throw },
-         { "try", Value::Try },
-         { "until", Value::Until },
-         { "with", Value::With },
-         { "where", Value::Where },
-         { "while", Value::While }
+   bool separator_or_punctuator(uint8_t c) {
+      switch (c) {
+      case '.': case '`': case '^': case '&': case '~': case '*':
+      case '-': case '+': case ';': case ',': case '@': case '|':
+      case '\'': case ':': case '=': case '\\': case '"': case '/':
+      case '(': case ')': case '{': case '}': case '[': case ']':
+      case '<': case '>': case '#': case ' ':
+         return true;
+      default:
+         return false;
+      }
+   }
+
+   namespace {
+      struct TokenMapEntry {
+         const char* const text;
+         const TokenCategory category;
+         const TokenValue value;
+         const Language dialect; //  = Language::Spad
       };
    }
+
+   const TokenMapEntry token_map[] {
+#undef OPENAXIOM_DEFINE_TOKEN
+#define OPENAXIOM_DEFINE_TOKEN(T, N, C, ...)        \
+      { N, TokenCategory::C, TokenValue::T, __VA_ARGS__ },
+#include <open-axiom/token-value>
+#undef OPENAXIOM_DEFINE_TOKEN      
+   };
+
+   TokenClassification
+   classify(const std::string& s) {
+      for (auto& t : token_map) {
+         if (t.text == s)
+            return { t.category, t.value };
+      }
+      return { TokenCategory::Unclassified, TokenValue::Unknown };
+   }
+
+   std::ostream&
+   operator<<(std::ostream& os, TokenValue tv) {
+      if (tv < TokenValue::Artificial)
+         os << token_map[uint8_t(tv)].text;
+      else switch (tv) {
+         case TokenValue::Indent: os << "%INDENT"; break;
+         case TokenValue::Unindent: os << "%UNIDENT"; break;
+         case TokenValue::Justify: os << "%JUSTIFY"; break;
+         default: os << "%ALIEN"; break;
+         }
+         
+      return os;
+   }
+
 }
diff --git a/src/utils/Makefile.in b/src/utils/Makefile.in
index 653b3cb5..3f70fda8 100644
--- a/src/utils/Makefile.in
+++ b/src/utils/Makefile.in
@@ -36,7 +36,6 @@ hammer_SOURCES = hammer.cc
 hammer_OBJECTS = $(hammer_SOURCES:.cc=.lo)
 hammer_LDADD = -L. -lOpenAxiom
 
-libOpenAxiom_HEADERS = hash-table.H string-pool.H
 libOpenAxiom_SOURCES = \
 	storage.cc string-pool.cc command.cc \
 	filesystem.cc
@@ -75,37 +74,18 @@ $(oa_target_oalib): libOpenAxiom.$(LIBEXT)
 	fi
 	cp $< $@
 
-stamp-headers: $(libOpenAxiom_HEADERS) Makefile
-	rm -f stamp-headers
-	if [ ! -d $(oa_target_headerdir) ]; then \
-	   mkdir -p -- $(oa_target_headerdir) || exit 1; \
-	fi ; \
-	c1=$(top_builddir)/config/openaxiom-c-macros.h; \
-	c2=$(oa_target_headerdir)/config; \
-	if [ ! -r $$c2 ] || ! cmp -s $$c1 $$c2 ; then \
-	   cp -p $$c1 $$c2; \
-	fi ; \
-	for h in $(oa_public_headers); do \
-           f1=$(srcdir)/$$h.H; f2=$(oa_target_headerdir)/$$h; \
-	   if [ ! -r $$f2 ] || ! cmp -s $$f1 $$f2 ; then \
-	      cp -p $$f1 $$f2; \
-	   fi; \
-	done ; \
-	$(STAMP) stamp-headers
-
 hammer$(EXEEXT): $(hammer_OBJECTS) libOpenAxiom.$(LIBEXT)
 	$(LINK) -o $@ $(hammer_OBJECTS)  $(hammer_LDADD) $(LDFLAGS)
 
 libOpenAxiom.$(LIBEXT): $(libOpenAxiom_OBJECTS)
 	$(LINK) -o $@ $(libOpenAxiom_OBJECTS)
 
-%.lo: %.cc stamp-headers
+%.lo: %.cc 
 	$(COMPILE) ${CXXFLAGS} $(oa_include_flags) -o $@ $<
 
 
 mostlyclean-local:
 	@rm -rf .libs
-	@rm -rf $(oa_include_headerdir) stamp-headers
 	@rm -f $(oa_target_oalib)
 	@rm -f $(libOpenAxiom_OBJECTS)
 	@rm -f *~ core