Add generic Boot and Spad tokenizer.

3 files changed, 372 insertions, 8 deletions

diff --git a/src/boot/Makefile.am b/src/boot/Makefile.am
index b24e923e..23af6c5d 100644
--- a/src/boot/Makefile.am
+++ b/src/boot/Makefile.am
@@ -58,7 +58,7 @@ LISP_LINK = \
 
 # We use a noinst_ primary because we take care of installation
 # procedure ourselves.
-noinst_PROGRAMS = bootsys
+noinst_PROGRAMS = bootsys bemol
 
 bootsys_SOURCES = \
 	utility.boot \
@@ -70,6 +70,14 @@ bootsys_SOURCES = \
 	parser.boot \
 	translator.boot
 
+bemol_SOURCES = \
+	bemol.cc
+
+bemol_LDADD = \
+	-L$(oa_target_libdir) -lOpenAxiom \
+	-L$(top_builddir)/src/syntax -lsyntax \
+	-L$(top_builddir)/src/io -lio
+
 oa_target_bootdir = $(oa_targetdir)/boot
 if OA_ECL_RT
 oa_bootsys_linkset = $(oa_target_bootdir)/linkset
@@ -89,11 +97,19 @@ bootsys_fn = $(bootsys_SOURCES:.boot=.fn)
 
 STAMP = touch
 
+AM_CXXFLAGS = \
+	-I$(top_srcdir)/src/include \
+	-I$(oa_target_includedir) \
+	-I$(top_builddir)/config \
+	-DOPENAXIOM_ROOT_DIRECTORY="\"$(open_axiom_installdir)\""
+
+
 # Make rule toplevel entry points.
 .PHONY: all-boot
 
 all-am: all-boot
 all-boot: $(oa_target_bindir)/bootsys$(EXEEXT) $(oa_bootsys_linkset)
+all-boot: bemol$(EXEEXT)
 
 # The final `bootsys' image.
 $(oa_target_bindir)/bootsys$(EXEEXT): stage2/bootsys$(EXEEXT)
diff --git a/src/boot/Makefile.in b/src/boot/Makefile.in
index d0c35a72..c55e08d3 100644
--- a/src/boot/Makefile.in
+++ b/src/boot/Makefile.in
@@ -128,10 +128,11 @@ POST_UNINSTALL = :
 build_triplet = @build@
 host_triplet = @host@
 target_triplet = @target@
-noinst_PROGRAMS = bootsys$(EXEEXT)
+noinst_PROGRAMS = bootsys$(EXEEXT) bemol$(EXEEXT)
 subdir = src/boot
 DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
-	$(top_srcdir)/config/mkinstalldirs
+	$(top_srcdir)/config/mkinstalldirs \
+	$(top_srcdir)/config/depcomp
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
 am__aclocal_m4_deps = $(top_srcdir)/config/libtool.m4 \
 	$(top_srcdir)/config/ltoptions.m4 \
@@ -146,13 +147,16 @@ CONFIG_HEADER = $(top_builddir)/config/openaxiom-c-macros.h
 CONFIG_CLEAN_FILES =
 CONFIG_CLEAN_VPATH_FILES =
 PROGRAMS = $(noinst_PROGRAMS)
-am_bootsys_OBJECTS =
-bootsys_OBJECTS = $(am_bootsys_OBJECTS)
-bootsys_LDADD = $(LDADD)
+am_bemol_OBJECTS = bemol.$(OBJEXT)
+bemol_OBJECTS = $(am_bemol_OBJECTS)
+bemol_DEPENDENCIES =
 AM_V_lt = $(am__v_lt_@AM_V@)
 am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
 am__v_lt_0 = --silent
 am__v_lt_1 = 
+am_bootsys_OBJECTS =
+bootsys_OBJECTS = $(am_bootsys_OBJECTS)
+bootsys_LDADD = $(LDADD)
 AM_V_P = $(am__v_P_@AM_V@)
 am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
 am__v_P_0 = false
@@ -166,6 +170,27 @@ am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
 am__v_at_0 = @
 am__v_at_1 = 
 DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)/config
+depcomp = $(SHELL) $(top_srcdir)/config/depcomp
+am__depfiles_maybe = depfiles
+am__mv = mv -f
+CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
+LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CXXFLAGS) $(CXXFLAGS)
+AM_V_CXX = $(am__v_CXX_@AM_V@)
+am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@)
+am__v_CXX_0 = @echo "  CXX     " $@;
+am__v_CXX_1 = 
+CXXLD = $(CXX)
+CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \
+	$(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CXXLD = $(am__v_CXXLD_@AM_V@)
+am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@)
+am__v_CXXLD_0 = @echo "  CXXLD   " $@;
+am__v_CXXLD_1 = 
 COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
 	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
 LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
@@ -184,8 +209,8 @@ AM_V_CCLD = $(am__v_CCLD_@AM_V@)
 am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
 am__v_CCLD_0 = @echo "  CCLD    " $@;
 am__v_CCLD_1 = 
-SOURCES = $(bootsys_SOURCES)
-DIST_SOURCES = $(bootsys_SOURCES)
+SOURCES = $(bemol_SOURCES) $(bootsys_SOURCES)
+DIST_SOURCES = $(bemol_SOURCES) $(bootsys_SOURCES)
 am__can_run_installinfo = \
   case $$AM_UPDATE_INFO_DIR in \
     n|no|NO) false;; \
@@ -419,6 +444,14 @@ bootsys_SOURCES = \
 	parser.boot \
 	translator.boot
 
+bemol_SOURCES = \
+	bemol.cc
+
+bemol_LDADD = \
+	-L$(oa_target_libdir) -lOpenAxiom \
+	-L$(top_builddir)/src/syntax -lsyntax \
+	-L$(top_builddir)/src/io -lio
+
 oa_target_bootdir = $(oa_targetdir)/boot
 @OA_ECL_RT_FALSE@oa_bootsys_linkset = 
 @OA_ECL_RT_TRUE@oa_bootsys_linkset = $(oa_target_bootdir)/linkset
@@ -432,6 +465,12 @@ bootsys_objects = $(bootsys_SOURCES:.boot=.$(LNKEXT))
 bootsys_data = $(bootsys_SOURCES:.boot=.data)
 bootsys_fn = $(bootsys_SOURCES:.boot=.fn)
 STAMP = touch
+AM_CXXFLAGS = \
+	-I$(top_srcdir)/src/include \
+	-I$(oa_target_includedir) \
+	-I$(top_builddir)/config \
+	-DOPENAXIOM_ROOT_DIRECTORY="\"$(open_axiom_installdir)\""
+
 
 #
 # The bootstrapping `bootsys' image.
@@ -461,6 +500,7 @@ LISP_COMPILE = \
 all: all-am
 
 .SUFFIXES:
+.SUFFIXES: .cc .lo .o .obj
 $(srcdir)/Makefile.in:  $(srcdir)/Makefile.am  $(am__configure_deps)
 	@for dep in $?; do \
 	  case '$(am__configure_deps)' in \
@@ -500,6 +540,10 @@ clean-noinstPROGRAMS:
 	echo " rm -f" $$list; \
 	rm -f $$list
 
+bemol$(EXEEXT): $(bemol_OBJECTS) $(bemol_DEPENDENCIES) $(EXTRA_bemol_DEPENDENCIES) 
+	@rm -f bemol$(EXEEXT)
+	$(AM_V_CXXLD)$(CXXLINK) $(bemol_OBJECTS) $(bemol_LDADD) $(LIBS)
+
 bootsys$(EXEEXT): $(bootsys_OBJECTS) $(bootsys_DEPENDENCIES) $(EXTRA_bootsys_DEPENDENCIES) 
 	@rm -f bootsys$(EXEEXT)
 	$(AM_V_CCLD)$(LINK) $(bootsys_OBJECTS) $(bootsys_LDADD) $(LIBS)
@@ -510,6 +554,29 @@ mostlyclean-compile:
 distclean-compile:
 	-rm -f *.tab.c
 
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bemol.Po@am__quote@
+
+.cc.o:
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $<
+
+.cc.obj:
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.cc.lo:
+@am__fastdepCXX_TRUE@	$(AM_V_CXX)$(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $<
+
 mostlyclean-libtool:
 	-rm -f *.lo
 
@@ -638,6 +705,7 @@ clean-am: clean-generic clean-libtool clean-local clean-noinstPROGRAMS \
 	mostlyclean-am
 
 distclean: distclean-am
+	-rm -rf ./$(DEPDIR)
 	-rm -f Makefile
 distclean-am: clean-am distclean-compile distclean-generic \
 	distclean-local distclean-tags
@@ -683,6 +751,7 @@ install-ps-am:
 installcheck-am:
 
 maintainer-clean: maintainer-clean-am
+	-rm -rf ./$(DEPDIR)
 	-rm -f Makefile
 maintainer-clean-am: distclean-am maintainer-clean-generic
 
@@ -723,6 +792,7 @@ uninstall-am:
 
 all-am: all-boot
 all-boot: $(oa_target_bindir)/bootsys$(EXEEXT) $(oa_bootsys_linkset)
+all-boot: bemol$(EXEEXT)
 
 # The final `bootsys' image.
 $(oa_target_bindir)/bootsys$(EXEEXT): stage2/bootsys$(EXEEXT)
diff --git a/src/boot/bemol.cc b/src/boot/bemol.cc
new file mode 100644
index 00000000..0399bb44
--- /dev/null
+++ b/src/boot/bemol.cc
@@ -0,0 +1,278 @@
+// -*- C++ -*-
+// Copyright (C) 2014, Gabriel Dos Reis.
+// All rights reserved.
+// Written by Gabriel Dos Reis.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     - Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//
+//     - Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in
+//       the documentation and/or other materials provided with the
+//       distribution.
+//
+//     - Neither the name of OpenAxiom. nor the names of its contributors
+//       may be used to endorse or promote products derived from this
+//       software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+// OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// --% Author: Gabriel Dos Reis
+// --% Description:
+
+#include <open-axiom/diagnostics>
+#include <open-axiom/token>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <string>
+#include <stack>
+#include <iterator>
+#include <ctype.h>
+
+using namespace OpenAxiom;
+
+// 
+// -- Reading input source files --
+// 
+
+// A physical line is just raw text, with coupled with location
+// information such as line and indentation column.
+struct Line : std::string {
+   LineNumber number;
+   ColumnIndex indent;
+   Line() : number(), indent() { }
+
+   std::string sub_string(ColumnIndex s, ColumnIndex e) const {
+      return substr(s, e - s);
+   }
+};
+
+// A program fragment is a logical line, composed of possibly
+// several physical lines subject to the off-side rule.  As a
+// special case, a line ending with the underbar character
+// continues to the next next with disregard to the off-side rule.
+struct Fragment : std::vector<Line> {
+   explicit operator bool() const { return not empty(); }
+   bool line_continuation() const {
+      return not empty() and back().back() == '_';
+   }
+   ColumnIndex last_indent() const {
+      return empty() ? 0 : back().indent;
+   }
+};
+
+// Formatting program fragments.
+static std::ostream&
+operator<<(std::ostream& os, const Fragment& f) {
+   std::copy(f.begin(), f.end(),
+             std::ostream_iterator<std::string>(std::cout, "\n"));
+   return os;
+}
+
+
+// A source input transform a character stream into a program fragment
+// stream, delivering a fragment one at a time.
+struct SourceInput {
+   SourceInput(std::istream& is) : input(is) { }
+   Fragment get();
+
+private:
+   std::istream& input;
+   Line line;
+};
+
+// Return the indentation level of a line.
+// FIXME: reject or expand tabs as appropriate.
+static ColumnIndex
+indentation(const Line& line) {
+   ColumnIndex idx { };
+   for (auto c : line) {
+      if (not isspace(c))
+         break;
+      ++idx;
+   }
+   return idx;
+}
+
+// Remove trailing white-space characters from the line.
+static Line&
+trim_right(Line& line) {
+   auto n = line.length();
+   while (n > 0 and isspace(line[n-1]))
+      --n;
+   line.resize(n);
+   return line;
+}
+
+// Clean up and dress up the line with indentation information.
+static Line&
+prop_up(Line& line) {
+   line.indent = indentation(trim_right(line));
+   return line;
+}
+
+// Return true if line is entirely a negative comment.
+static bool
+negative_comment(const Line& line) {
+   if (line.indent + 1 >= line.length())
+      return false;
+   return line[line.indent] == '-' and line[line.indent + 1] == '-';
+}
+
+// Return true if line is either empty or a negative comment.
+static bool
+blank(const Line& line) {
+   return line.empty() or negative_comment(line);
+}
+
+// Return true if line is entirely a positive comment, i.e. a description.
+static bool
+positive_comment(const Line& line) {
+   if (line.indent + 1 >= line.length())
+      return false;
+   return line[line.indent] == '+' and line[line.indent + 1] == '+';
+}
+
+// Decompose the input souce file into fragments, and return one
+// fragment at a time.
+Fragment
+SourceInput::get() {
+   Fragment fragment;
+   std::stack<ColumnIndex> indents;
+
+   if (not line.empty()) {
+      indents.push(line.indent);
+      fragment.push_back(line);
+   }
+
+   while (std::getline(input, line)) {
+      ++line.number;
+      if (blank(prop_up(line)))
+         continue;              // Don't bother with ignorable comments.
+      else if (fragment.line_continuation())
+         ;
+      else if (indents.empty()) {
+         if (fragment.empty() and line.indent != 0)
+            std::cout << "warning: white space at begining of fragment"
+                      << " on line " << line.number << '\n';
+         indents.push(line.indent);
+      }
+      else if (line.indent == 0 and not positive_comment(fragment.back()))
+         break;                 // A completely new line; save for later.
+      else if (line.indent > indents.top())
+         indents.push(line.indent);
+      else {
+         while (line.indent < indents.top())
+            indents.pop();
+      }
+      fragment.push_back(line);
+   }
+   return fragment;
+}
+
+// 
+// -- Decomposing source files into lexical units of information --
+// 
+
+struct Locus {
+   OpenAxiom::LineNumber line;
+   OpenAxiom::ColumnIndex column;
+};
+
+static std::ostream&
+operator<<(std::ostream& os, const Locus& l)
+{
+   os << '{' << l.line << ", " << l.column << '}';
+   return os;
+}
+
+struct BemolToken {
+   using Location = ::Locus;
+   OpenAxiom::TokenCategory category;
+   OpenAxiom::TokenValue value;
+   Locus start;
+   Locus end;
+
+   explicit operator bool() const { return category != TokenCategory::EOS; }
+};
+
+static std::ostream&
+operator<<(std::ostream& os, const BemolToken& t) {
+   os << t.category << '{' << t.start << '-' << t.end << '}';
+   return os;
+}
+
+static void
+translate_source_file(SourceInput& src, std::ostream& out, const char* path) {
+   while (auto f = src.get()) {
+      out << "================================================\n";
+      out << f;
+      OpenAxiom::TokenStream<Fragment, BemolToken> ts { f };
+      try {
+         while (auto t = ts.get(OpenAxiom::Language::Boot)) {
+            out << '\t' << t;
+            switch (t.category) {
+            case TokenCategory::Junk:
+            case TokenCategory::Unclassified:
+               out //<< f[t.start.line].sub_string(t.start.column, t.end.column)
+                   << " in file " << path
+                   << " at line " << t.start.line
+                   << ", column " << t.start.column;
+               break;
+            default:
+               break;
+            }
+            out << '\n';
+         }
+      }
+      catch(const OpenAxiom::EndOfStringUnseen& e) {
+         std::cerr << path << ": syntax error: "
+                   << "premature end of line before matching quote "
+                   << "of string literal on line " << e.line
+                   << " at column " << e.column
+                   << std::endl;
+      }
+      catch (const OpenAxiom::MissingExponent& e) {
+         std::cerr << path << ": syntax error: "
+                   << "missing exponent of floating point constant "
+                   << "on line " << e.line
+                   << ", column " << e.column
+                   << std::endl;
+      }
+      out << "================================================\n";
+   }
+   out << std::flush;
+}
+
+static void
+process_file(const char* path) {
+   std::ifstream in { path };
+   if (!in) {
+      std::cerr << "error: could not open file `" << path << "'"
+                << std::endl;
+      return;
+   }
+   SourceInput src { in };
+   translate_source_file(src, std::cout, path);
+}
+
+int main(int argc, char* argv[]) {
+   for (int i = 1; i < argc; ++i) {
+      process_file(argv[i]);
+   }
+}