diff options
| -rw-r--r-- | src/include/open-axiom/InputFragment | 51 | ||||
| -rw-r--r-- | src/include/open-axiom/sexpr | 4 | ||||
| -rw-r--r-- | src/include/open-axiom/token | 188 | ||||
| -rw-r--r-- | src/lib/Makefile.in | 7 | ||||
| -rw-r--r-- | src/syntax/Parser.cxx | 1 |
5 files changed, 120 insertions, 131 deletions
diff --git a/src/include/open-axiom/InputFragment b/src/include/open-axiom/InputFragment index e55cd0a6..63139db4 100644 --- a/src/include/open-axiom/InputFragment +++ b/src/include/open-axiom/InputFragment @@ -36,9 +36,22 @@ #ifndef OPENAXIOM_INPUTFRAGMENT_included #define OPENAXIOM_INPUTFRAGMENT_included -#include <open-axiom/token> +#include <vector> +#include <string> +#include <stack> namespace OpenAxiom { + // Datatypes for locating lines and columns. + using LineNumber = std::size_t; + using ColumnIndex = std::size_t; + + enum class LineKind : uint8_t { + Ordinary, // Ordinary input line + Description, // Documentation commentary lines. + Meta, // Input to the attention of the reader + Ignorable, // Ignorable commentary line + }; + // A physical line is just raw text, coupled with location // information such as line and indentation column. struct Line : std::string { @@ -51,6 +64,34 @@ namespace OpenAxiom { } }; + // Cursor into a fragment. + struct FragmentCursor { + std::size_t line; // index of a line in a fragment + std::size_t column; // column number at line. + + inline FragmentCursor& operator++() { + ++column; + return *this; + } + + inline FragmentCursor operator++(int) { + auto tmp = *this; + ++*this; + return tmp; + } + + inline FragmentCursor& operator--() { + --column; + return *this; + } + + inline FragmentCursor operator--(int) { + auto tmp = *this; + --*this; + return tmp; + } + }; + // A program fragment is a logical line, composed of possibly // several physical lines subject to the off-side rule. As a // special case, a line ending with the underbar character @@ -63,19 +104,19 @@ namespace OpenAxiom { } using std::vector<Line>::operator[]; // Reference a line given by a position into this fragment. - const Line& operator()(const OpenAxiom::FragmentCursor& pos) const { + const Line& operator()(const FragmentCursor& pos) const { return (*this)[pos.line]; } // Reference a character code unit at the position into this fragment. - uint8_t operator[](const OpenAxiom::FragmentCursor& pos) const { + uint8_t operator[](const FragmentCursor& pos) const { return (*this)[pos.line][pos.column]; } // Advance the cursor position to the next character code unit. - uint8_t advance(OpenAxiom::FragmentCursor& pos) const { + uint8_t advance(FragmentCursor& pos) const { return (*this)[pos.line][pos.column++]; } // This predicate holds if this fragment covers the cursor position. - bool covering(const OpenAxiom::FragmentCursor& pos) const { + bool covering(const FragmentCursor& pos) const { return pos.column < (*this)[pos.line].size(); } }; diff --git a/src/include/open-axiom/sexpr b/src/include/open-axiom/sexpr index 8d8eea0b..c7dff7c5 100644 --- a/src/include/open-axiom/sexpr +++ b/src/include/open-axiom/sexpr @@ -1,5 +1,5 @@ // -*- C++ -*- -// Copyright (C) 2010-2014, Gabriel Dos Reis. +// Copyright (C) 2010-2017, Gabriel Dos Reis. // All rights reserved. // Written by Gabriel Dos Reis. // @@ -46,7 +46,7 @@ #include <iosfwd> #include <vector> #include <open-axiom/storage> -#include <open-axiom/token> +#include <open-axiom/Input> namespace OpenAxiom { namespace Sexpr { diff --git a/src/include/open-axiom/token b/src/include/open-axiom/token index 56cea373..f487cb3b 100644 --- a/src/include/open-axiom/token +++ b/src/include/open-axiom/token @@ -37,17 +37,10 @@ #include <stdint.h> #include <stack> #include <iosfwd> -#include <open-axiom/Input> #include <open-axiom/dialect> +#include <open-axiom/InputFragment> namespace OpenAxiom { - enum class LineKind : uint8_t { - Ordinary, // Ordinary input line - Description, // Documentation commentary lines. - Meta, // Input to the attention of the reader - Ignorable, // Ignorable commentary line - }; - // Categorization of Boot and Spad tokens. enum class TokenCategory : uint8_t { Unclassified, // token of unknown class @@ -79,10 +72,6 @@ namespace OpenAxiom { std::ostream& operator<<(std::ostream&, TokenValue); - // Datatypes for locating lines and columns. - using LineNumber = std::size_t; - using ColumnIndex = std::size_t; - struct Locus { LineNumber line; ColumnIndex column; @@ -114,34 +103,6 @@ namespace OpenAxiom { using Location = Locus; }; - // Cursor into a fragment. - struct FragmentCursor { - std::size_t line; // index of a line in a fragment - std::size_t column; // column number at line. - - inline FragmentCursor& operator++() { - ++column; - return *this; - } - - inline FragmentCursor operator++(int) { - auto tmp = *this; - ++*this; - return tmp; - } - - inline FragmentCursor& operator--() { - --column; - return *this; - } - - inline FragmentCursor operator--(int) { - auto tmp = *this; - --*this; - return tmp; - } - }; - // -- Exception types struct EndOfStringUnseen { LineNumber line; @@ -155,9 +116,9 @@ namespace OpenAxiom { // Object of this datatype decompose a program fragment into a // token stream. The tokens are of type indicated by Tok. - template<typename Frag, typename Tok> + template<typename Tok> struct Tokenizer { - Tokenizer(Frag& f) + Tokenizer(Fragment& f) : frag(f), pos{ 0, frag.front().indent } { @@ -170,9 +131,9 @@ namespace OpenAxiom { Tok get(Language = Language::Spad); private: - Frag& frag; + Fragment& frag; FragmentCursor pos; - std::stack<Locus> indents; + std::stack<FragmentCursor> indents; std::size_t line_length() const { return frag(pos).size(); } @@ -211,22 +172,22 @@ namespace OpenAxiom { } template<typename T> - inline T& eos_token(T& t, const FragmentCursor& pos) { + inline T& eos_token(T& t) { t.category = TokenCategory::EOS; t.value = TokenValue::EndOfStream; - t.end = pos; + t.end = t.start; return t; } template<typename T> - inline T& ws_token(T& t, const FragmentCursor& pos) { + inline T& ws_token(T& t, const Locus& loc) { t.category = TokenCategory::Whitespace; - t.end = pos; + t.end = loc; return t; } template<typename L, typename T> - static void junk(L& line, ColumnIndex& idx, T& t) { + void junk(L& line, ColumnIndex& idx, T& t) { while (idx < line.size() and not separator_or_punctuator(line[idx])) ++idx; t.category = TokenCategory::Junk; @@ -239,8 +200,8 @@ namespace OpenAxiom { ++idx; } - template<typename Frag, typename Tok> - void string_literal(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void string_literal(Fragment& frag, FragmentCursor& pos, Tok& t) { bool done = false; bool escape = false; while (frag.covering(pos) && not done) { @@ -270,8 +231,7 @@ namespace OpenAxiom { ++idx; } - template<typename Frag> - bool next_line(Frag& frag, FragmentCursor& pos) { + static bool next_line(Fragment& frag, FragmentCursor& pos) { if (++pos.line < frag.size()) { pos.column = frag(pos).indent; return true; @@ -362,9 +322,8 @@ namespace OpenAxiom { return t; } - template<typename Frag, typename Tok> - static void - left_paren_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void left_paren_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { punctuator_token(t, TokenValue::OpenParen); if (frag.covering(pos) and frag[pos] == '|') { ++pos; @@ -372,9 +331,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - left_brace_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void left_brace_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { punctuator_token(t, TokenValue::OpenBrace); if (frag.covering(pos) and frag[pos] == '|') { ++pos; @@ -382,9 +340,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - left_bracket_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void left_bracket_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { punctuator_token(t, TokenValue::OpenBracket); if (frag.covering(pos) and frag[pos] == '|') { ++pos; @@ -392,9 +349,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - colon_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void colon_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Colon); if (frag.covering(pos)) switch (frag[pos]) { @@ -405,9 +361,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - star_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void star_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Star); if (frag.covering(pos) and frag[pos] == '*') { t.value = TokenValue::StarStar; @@ -415,9 +370,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - slash_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void slash_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Slash); if (frag.covering(pos)) switch (frag[pos]) { @@ -427,9 +381,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - backslash_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void backslash_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Backslash); if (frag.covering(pos)) switch (frag[pos]) { @@ -439,9 +392,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - less_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void less_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Less); if (frag.covering(pos)) switch (frag[pos]) { @@ -458,9 +410,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - equal_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void equal_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Eq); if (frag.covering(pos)) switch (frag[pos]) { @@ -476,9 +427,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - tilde_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void tilde_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Tilde); if (frag.covering(pos) and frag[pos] == '=') { t.value = TokenValue::TildeEq; @@ -486,9 +436,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - greater_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void greater_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Greater); if (frag.covering(pos)) switch (frag[pos]) { @@ -497,9 +446,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - bar_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void bar_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { punctuator_token(t, TokenValue::Bar); if (frag.covering(pos)) switch (frag[pos]) { @@ -510,30 +458,28 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - minus_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void minus_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Minus); if (frag.covering(pos)) switch (frag[pos]) { case '>': t.value = TokenValue::RightArrow; ++pos; break; case '-': comment_token(t, TokenValue::Wisecrack); - next_line(frag, pos); + pos.column = frag(pos).length(); break; } } - template<typename Frag, typename Tok> - static void - plus_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void plus_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Plus); if (frag.covering(pos)) switch (frag[pos]) { case '+': comment_token(t, TokenValue::Commentary); - next_line(frag, pos); + pos.column = frag(pos).length(); break; case '-': if (pos.column + 1 < frag(pos).size() @@ -546,9 +492,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - dot_et_al(Frag& frag, FragmentCursor& pos, Tok& t) { + template<typename Tok> + void dot_et_al(Fragment& frag, FragmentCursor& pos, Tok& t) { operator_token(t, TokenValue::Dot); if (frag.covering(pos) and frag[pos] == '.') { t.value = TokenValue::DotDot; @@ -556,9 +501,9 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - static void - dollar_et_al(Frag& frag, FragmentCursor& pos, Tok& t, Language dialect) { + template<typename Tok> + void + dollar_et_al(Fragment& frag, FragmentCursor& pos, Tok& t, Language dialect) { if (dialect != Language::Boot or not frag.covering(pos) or separator_or_punctuator(frag[pos])) operator_token(t, TokenValue::Dollar); @@ -566,9 +511,9 @@ namespace OpenAxiom { identifier(frag(pos), pos.column, t, dialect); } - template<typename Frag, typename Tok> - static void - sharp_et_al(Frag& frag, FragmentCursor& pos, Tok& t, Language dialect) { + template<typename Tok> + void + sharp_et_al(Fragment& frag, FragmentCursor& pos, Tok& t, Language dialect) { if (dialect != Language::Lisp) operator_token(t, TokenValue::Sharp); else if (frag.covering(pos)) @@ -583,8 +528,8 @@ namespace OpenAxiom { } } - template<typename Frag, typename Tok> - Tok Tokenizer<Frag, Tok>::finish(Tok& t, Language dialect) { + template<typename Tok> + Tok Tokenizer<Tok>::finish(Tok& t, Language dialect) { switch (auto c = frag.advance(pos)) { case '#': sharp_et_al(frag, pos, t, dialect); break; case '@': operator_token(t, TokenValue::At); break; @@ -630,26 +575,32 @@ namespace OpenAxiom { return t; } - template<typename Frag, typename Tok> - Tok Tokenizer<Frag, Tok>::get(Language dialect) { + inline Locus location(const Fragment& frag, const FragmentCursor& pos) { + if (pos.line < frag.size()) + return { frag[pos.line].number, pos.column }; + return { frag.back().number + 1, { } }; + } + + template<typename Tok> + Tok Tokenizer<Tok>::get(Language dialect) { Tok t { }; - t.start = pos; + t.start = location(frag, pos); if (eos()) - return eos_token(t, pos); + return eos_token(t); else if (isblank(frag[pos])) { skip_whitespace(frag(pos), pos.column); - return ws_token(t, pos); + return ws_token(t, location(frag, pos)); } else if (line_continuation()) { if (next_line(frag, pos)) return finish(t, dialect); - return eos_token(t, pos); + return eos_token(t); } else if (pos.column >= line_length()) { if (not next_line(frag, pos)) - return eos_token(t, pos); - t.start = t.end = pos; + return eos_token(t); + t.start = t.end = location(frag, pos); auto indent = indents.top(); if (indent.column < pos.column) { indents.push(pos); @@ -669,9 +620,8 @@ namespace OpenAxiom { // -- Token streams. template<typename T> struct TokenStream : std::vector<T> { - template<typename Frag> - explicit TokenStream(Frag& f, Language dialect = Language::Spad) { - Tokenizer<Frag, T> lex { f }; + explicit TokenStream(Fragment& f, Language dialect = Language::Spad) { + Tokenizer<T> lex { f }; while (auto t = lex.get(dialect)) this->push_back(t); } diff --git a/src/lib/Makefile.in b/src/lib/Makefile.in index fecb62bd..725c10eb 100644 --- a/src/lib/Makefile.in +++ b/src/lib/Makefile.in @@ -114,17 +114,14 @@ libspad.$(LIBEXT): $(libspad_objects) $(COMPILE) $(oa_shrobj_flags) -no-suppress -o $@ $(CFLAGS) \ $(oa_includes) $(AXIOM_X11_CFLAGS) $< -# This is a support library, so it does not change often and -# we don't need to remove the produced objects in mostlyclean. -# The remoal is done by clean. mostlyclean-local: @rm -f *.$(OBJEXT) *.lo - -clean-local: mostlyclean-local @rm -f $(oa_target_libdir)/$(oa_shrlib_prefix)open-axiom-core$(SHREXT) @rm -f libopen-axiom-core.$(LIBEXT) @rm -f libspad.$(LIBEXT) @rm -fr .libs _libs @rm -f stamp +clean-local: mostlyclean-local + distclean-local: clean-local diff --git a/src/syntax/Parser.cxx b/src/syntax/Parser.cxx index df856d89..8245b70b 100644 --- a/src/syntax/Parser.cxx +++ b/src/syntax/Parser.cxx @@ -42,6 +42,7 @@ #include <open-axiom/diagnostics> #include <open-axiom/InputFragment> #include <open-axiom/SourceInput> +#include <open-axiom/token> #include <open-axiom/Parser> namespace { |