diff options
| author | dos-reis <gdr@axiomatics.org> | 2014-08-30 02:55:43 +0000 |
|---|---|---|
| committer | dos-reis <gdr@axiomatics.org> | 2014-08-30 02:55:43 +0000 |
| commit | e88dc9861a1d7ae4ca3fca1f5ef14d46f7190812 (patch) | |
| tree | d2dfec2283085251c1c17c82ff8a8f484b340056 /src | |
| parent | 2989854362732a4b0d61a757766158f643d9504b (diff) | |
| download | open-axiom-e88dc9861a1d7ae4ca3fca1f5ef14d46f7190812.tar.gz | |
Add Lisp punctuators to the generic lexer.
Diffstat (limited to 'src')
| -rw-r--r-- | src/include/token-value.def | 9 | ||||
| -rw-r--r-- | src/include/token.H | 30 |
2 files changed, 37 insertions, 2 deletions
diff --git a/src/include/token-value.def b/src/include/token-value.def index ea79c9a5..9a949b86 100644 --- a/src/include/token-value.def +++ b/src/include/token-value.def @@ -45,6 +45,7 @@ OPENAXIOM_DEFINE_TOKEN(ColonEq, ":=", Keyword, Language::BootSpad) OPENAXIOM_DEFINE_TOKEN(At, "@", Operator, Language::All) OPENAXIOM_DEFINE_TOKEN(Exclamation, "!", Punctuator, Language::Boot) OPENAXIOM_DEFINE_TOKEN(Comma, ",", Punctuator, Language::All) +OPENAXIOM_DEFINE_TOKEN(CommaAt, ",@", Operator, Language::Lisp) OPENAXIOM_DEFINE_TOKEN(Semicolon, ";", Punctuator, Language::All) OPENAXIOM_DEFINE_TOKEN(Star, "*", Operator, Language::BootSpad) OPENAXIOM_DEFINE_TOKEN(StarStar, "**", Operator, Language::BootSpad) @@ -65,7 +66,13 @@ OPENAXIOM_DEFINE_TOKEN(EqEq, "==", Keyword, Language::BootSpad) OPENAXIOM_DEFINE_TOKEN(Tilde, "~", Operator, Language::BootSpad) OPENAXIOM_DEFINE_TOKEN(TildeEq, "~=", Operator, Language::BootSpad) OPENAXIOM_DEFINE_TOKEN(Caret, "^", Operator, Language::BootSpad) -OPENAXIOM_DEFINE_TOKEN(Pound, "#", Operator, Language::BootSpad) +OPENAXIOM_DEFINE_TOKEN(Sharp, "#", Operator, Language::BootSpad) +OPENAXIOM_DEFINE_TOKEN(SharpLeftParen, "#(", Punctuator, Language::Lisp) +OPENAXIOM_DEFINE_TOKEN(SharpApostrophe, "#'", Operator, Language::Lisp) +OPENAXIOM_DEFINE_TOKEN(SharpColon, "#:", Operator, Language::Lisp) +OPENAXIOM_DEFINE_TOKEN(SharpPlus, "#+", Punctuator, Language::Lisp) +OPENAXIOM_DEFINE_TOKEN(SharpMinus, "#-", Punctuator, Language::Lisp) +OPENAXIOM_DEFINE_TOKEN(SharpDot, "#.", Operator, Language::Lisp) OPENAXIOM_DEFINE_TOKEN(Dollar, "$", Operator, Language::Spad) OPENAXIOM_DEFINE_TOKEN(Ampersand, "&", Operator, Language::Spad) OPENAXIOM_DEFINE_TOKEN(RightArrow, "->", Operator, Language::BootSpad) diff --git a/src/include/token.H b/src/include/token.H index 9837bda0..719f6e5f 100644 --- a/src/include/token.H +++ b/src/include/token.H @@ -103,6 +103,17 @@ namespace OpenAxiom { return p; } + inline FragmentCursor operator++(FragmentCursor& p, int) { + auto q = p; + ++p; + return q; + } + + inline FragmentCursor& operator--(FragmentCursor& p) { + --p.column; + return p; + } + // -- Exception types struct EndOfStringUnseen { LineNumber line; @@ -512,6 +523,23 @@ namespace OpenAxiom { identifier(frag(pos), pos.column, t, dialect); } + template<typename Frag, typename Tok> + static void + sharp_et_al(Frag& frag, FragmentCursor& pos, Tok& t, Language dialect) { + if (dialect != Language::Lisp) + operator_token(t, TokenValue::Sharp); + else if (frag.covering(pos)) + switch (frag[pos++]) { + case '(': punctuator_token(t, TokenValue::SharpLeftParen); break; + case '\'': operator_token(t, TokenValue::SharpApostrophe); break; + case ':': operator_token(t, TokenValue::SharpColon); break; + case '+': punctuator_token(t, TokenValue::SharpPlus); break; + case '-': punctuator_token(t, TokenValue::SharpMinus); break; + case '.': operator_token(t, TokenValue::SharpDot); break; + default: --pos; break; + } + } + template<typename Frag, typename Tok> Tok Tokenizer<Frag, Tok>::get(Language dialect) { @@ -559,7 +587,7 @@ namespace OpenAxiom { } switch (auto c = frag.advance(pos)) { - case '#': operator_token(t, TokenValue::Pound); break; + case '#': sharp_et_al(frag, pos, t, dialect); break; case '@': operator_token(t, TokenValue::At); break; case '^': operator_token(t, TokenValue::Caret); break; case '&': punctuator_token(t, TokenValue::Ampersand); break; |