Tidy C++ code tokenizing Boot and Spad inputs.

14 files changed, 23 insertions, 9 deletions

diff --git a/src/include/open-axiom/Constructor b/src/include/open-axiom/Constructor
index bfab3940..d7c9b378 100644
--- a/src/include/open-axiom/Constructor
+++ b/src/include/open-axiom/Constructor
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2013, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
diff --git a/src/include/open-axiom/Database b/src/include/open-axiom/Database
index cea082c8..76189632 100644
--- a/src/include/open-axiom/Database
+++ b/src/include/open-axiom/Database
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2013, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
diff --git a/src/include/open-axiom/Lisp b/src/include/open-axiom/Lisp
index a189dd38..dff5c769 100644
--- a/src/include/open-axiom/Lisp
+++ b/src/include/open-axiom/Lisp
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2013-2014, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
diff --git a/src/include/open-axiom/diagnostics b/src/include/open-axiom/diagnostics
index 9cb0fce1..01edffd4 100644
--- a/src/include/open-axiom/diagnostics
+++ b/src/include/open-axiom/diagnostics
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2013, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
diff --git a/src/include/open-axiom/dialect b/src/include/open-axiom/dialect
index bcfddd04..5d5218c6 100644
--- a/src/include/open-axiom/dialect
+++ b/src/include/open-axiom/dialect
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2013-2014, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
diff --git a/src/include/open-axiom/hash-table b/src/include/open-axiom/hash-table
index d74c7760..906144f7 100644
--- a/src/include/open-axiom/hash-table
+++ b/src/include/open-axiom/hash-table
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2010, Gabriel Dos Reis.
 // All rights reserved.
 //
diff --git a/src/include/open-axiom/sexpr b/src/include/open-axiom/sexpr
index 6920e436..8d8eea0b 100644
--- a/src/include/open-axiom/sexpr
+++ b/src/include/open-axiom/sexpr
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2010-2014, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
diff --git a/src/include/open-axiom/storage b/src/include/open-axiom/storage
index e414dec2..fe12d1eb 100644
--- a/src/include/open-axiom/storage
+++ b/src/include/open-axiom/storage
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2010-2013, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
diff --git a/src/include/open-axiom/string-pool b/src/include/open-axiom/string-pool
index 1f746e6d..d35c67b1 100644
--- a/src/include/open-axiom/string-pool
+++ b/src/include/open-axiom/string-pool
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2010-2015, Gabriel Dos Reis.
 // All rights reserved.
 //
diff --git a/src/include/open-axiom/structure b/src/include/open-axiom/structure
index 33c084f2..24ce4144 100644
--- a/src/include/open-axiom/structure
+++ b/src/include/open-axiom/structure
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2013, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
diff --git a/src/include/open-axiom/token b/src/include/open-axiom/token
index b57b69d6..1b4f78f0 100644
--- a/src/include/open-axiom/token
+++ b/src/include/open-axiom/token
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2013-2014, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
@@ -99,7 +100,7 @@ namespace OpenAxiom {
       TokenValue value;
 
       explicit operator bool() const {
-         return category != TokenCategory::Unclassified;
+         return category != TokenCategory::EOS;
       }
    };
 
@@ -227,7 +228,7 @@ namespace OpenAxiom {
    template<typename L>
    inline void
    skip_whitespace(L& line, ColumnIndex& idx) {
-      while (idx < line.size() and isspace(line[idx]))
+      while (idx < line.size() and isblank(line[idx]))
          ++idx;
    }
 
@@ -334,11 +335,12 @@ namespace OpenAxiom {
       while (idx < line.size() and identifier_suffix(line[idx]))
          ++idx;
 
-      if (saw_escape)
-         t.category = TokenCategory::Identifier;
-      else if (auto info = classify(line.sub_string(start, idx))) {
-         t.category = info.category;
-         t.value = info.value;
+      if (!saw_escape) {
+         auto info = classify(line.sub_string(start, idx));
+         if (info.category != TokenCategory::Identifier) {
+            t.category = info.category;
+            t.value = info.value;
+         }
       }
       return t;
    }
@@ -575,7 +577,7 @@ namespace OpenAxiom {
          t.end = current_locus();
          return t;
       }
-      else if (isspace(frag[pos])) {
+      else if (isblank(frag[pos])) {
          skip_whitespace(frag(pos), pos.column);
          t.category = TokenCategory::Whitespace;
          t.end = current_locus();
diff --git a/src/include/open-axiom/token-value b/src/include/open-axiom/token-value
index 9a949b86..aa8347d3 100644
--- a/src/include/open-axiom/token-value
+++ b/src/include/open-axiom/token-value
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2014, Gabriel Dos Reis.
 // All rights reserved.
 // Written by Gabriel Dos Reis.
diff --git a/src/include/open-axiom/vm b/src/include/open-axiom/vm
index 3bb62250..400fc3a2 100644
--- a/src/include/open-axiom/vm
+++ b/src/include/open-axiom/vm
@@ -1,3 +1,4 @@
+// -*- C++ -*-
 // Copyright (C) 2011-2014, Gabriel Dos Reis.
 // All rights reserved.
 //
diff --git a/src/syntax/token.cc b/src/syntax/token.cc
index d66a849e..74b58fa2 100644
--- a/src/syntax/token.cc
+++ b/src/syntax/token.cc
@@ -96,7 +96,7 @@ namespace OpenAxiom {
          if (t.text == s)
             return { t.category, t.value };
       }
-      return { TokenCategory::Unclassified, TokenValue::Unknown };
+      return { TokenCategory::Identifier, TokenValue::Unknown };
    }
 
    std::ostream&