diff options
| -rw-r--r-- | src/include/open-axiom/InputFragment | 11 | ||||
| -rw-r--r-- | src/include/open-axiom/token | 35 | ||||
| -rw-r--r-- | src/io/InputFragment.cxx | 5 | ||||
| -rw-r--r-- | src/syntax/token.cxx | 7 |
4 files changed, 24 insertions, 34 deletions
diff --git a/src/include/open-axiom/InputFragment b/src/include/open-axiom/InputFragment index 63139db4..67863e44 100644 --- a/src/include/open-axiom/InputFragment +++ b/src/include/open-axiom/InputFragment @@ -36,6 +36,7 @@ #ifndef OPENAXIOM_INPUTFRAGMENT_included #define OPENAXIOM_INPUTFRAGMENT_included +#include <stdint.h> #include <vector> #include <string> #include <stack> @@ -43,7 +44,7 @@ namespace OpenAxiom { // Datatypes for locating lines and columns. using LineNumber = std::size_t; - using ColumnIndex = std::size_t; + using ColumnIndex = uint16_t; enum class LineKind : uint8_t { Ordinary, // Ordinary input line @@ -65,9 +66,11 @@ namespace OpenAxiom { }; // Cursor into a fragment. + // Note: We don't expect people to write large fragements + // either in length or in width. struct FragmentCursor { - std::size_t line; // index of a line in a fragment - std::size_t column; // column number at line. + uint16_t line; // index of a line in a fragment + uint16_t column; // column number at line. inline FragmentCursor& operator++() { ++column; @@ -92,6 +95,8 @@ namespace OpenAxiom { } }; + std::ostream& operator<<(std::ostream&, FragmentCursor); + // A program fragment is a logical line, composed of possibly // several physical lines subject to the off-side rule. As a // special case, a line ending with the underbar character diff --git a/src/include/open-axiom/token b/src/include/open-axiom/token index f487cb3b..943d01cc 100644 --- a/src/include/open-axiom/token +++ b/src/include/open-axiom/token @@ -72,17 +72,10 @@ namespace OpenAxiom { std::ostream& operator<<(std::ostream&, TokenValue); - struct Locus { - LineNumber line; - ColumnIndex column; - }; - - std::ostream& operator<<(std::ostream&, const Locus&); - - // Program text region + // Program text region, with a fragment. struct Region { - Locus start; - Locus end; + FragmentCursor start; + FragmentCursor end; }; // Given a symbolic or alphabetic token, retrieve its category @@ -99,9 +92,7 @@ namespace OpenAxiom { TokenClassification classify(const std::string&); // Token data structure: a region of text with a classification. - struct Token : Region, TokenClassification { - using Location = Locus; - }; + struct Token : Region, TokenClassification { }; // -- Exception types struct EndOfStringUnseen { @@ -180,9 +171,9 @@ namespace OpenAxiom { } template<typename T> - inline T& ws_token(T& t, const Locus& loc) { + inline T& ws_token(T& t, FragmentCursor pos) { t.category = TokenCategory::Whitespace; - t.end = loc; + t.end = pos; return t; } @@ -571,26 +562,20 @@ namespace OpenAxiom { break; } - t.end = { frag(pos).number, pos.column }; + t.end = pos; return t; } - inline Locus location(const Fragment& frag, const FragmentCursor& pos) { - if (pos.line < frag.size()) - return { frag[pos.line].number, pos.column }; - return { frag.back().number + 1, { } }; - } - template<typename Tok> Tok Tokenizer<Tok>::get(Language dialect) { Tok t { }; - t.start = location(frag, pos); + t.start = pos; if (eos()) return eos_token(t); else if (isblank(frag[pos])) { skip_whitespace(frag(pos), pos.column); - return ws_token(t, location(frag, pos)); + return ws_token(t, pos); } else if (line_continuation()) { if (next_line(frag, pos)) @@ -600,7 +585,7 @@ namespace OpenAxiom { else if (pos.column >= line_length()) { if (not next_line(frag, pos)) return eos_token(t); - t.start = t.end = location(frag, pos); + t.start = t.end = pos; auto indent = indents.top(); if (indent.column < pos.column) { indents.push(pos); diff --git a/src/io/InputFragment.cxx b/src/io/InputFragment.cxx index 01970f84..595b88b1 100644 --- a/src/io/InputFragment.cxx +++ b/src/io/InputFragment.cxx @@ -41,6 +41,11 @@ #include <open-axiom/SourceInput> namespace OpenAxiom { + + std::ostream& operator<<(std::ostream& os, FragmentCursor cur) { + return os << '{' << cur.line << '-' << cur.column << '}'; + } + // Formatting program fragments. std::ostream& operator<<(std::ostream& os, const Fragment& f) { std::copy(f.begin(), f.end(), diff --git a/src/syntax/token.cxx b/src/syntax/token.cxx index 011520a8..e1c02e3f 100644 --- a/src/syntax/token.cxx +++ b/src/syntax/token.cxx @@ -35,12 +35,7 @@ #include <iostream> namespace OpenAxiom { - std::ostream& operator<<(std::ostream& os, const Locus& l) { - return os << '{' << l.line << ", " << l.column << '}'; - } - - std::ostream& - operator<<(std::ostream& os, TokenCategory tc) { + std::ostream& operator<<(std::ostream& os, TokenCategory tc) { switch (tc) { case TokenCategory::Unclassified: os << "UNCLASSIFIED"; break; case TokenCategory::Whitespace: os << "WHITESPACE"; break; |