aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/include/open-axiom/token117
1 files changed, 58 insertions, 59 deletions
diff --git a/src/include/open-axiom/token b/src/include/open-axiom/token
index 32bb8fd9..56cea373 100644
--- a/src/include/open-axiom/token
+++ b/src/include/open-axiom/token
@@ -161,43 +161,23 @@ namespace OpenAxiom {
: frag(f),
pos{ 0, frag.front().indent }
{
- indents.push(pos.column);
+ indents.push(pos);
}
bool eos() const {
- return pos.line >= frag.size()
- or (pos.line + 1 == frag.size() and pos.column >= frag.back().size());
+ return pos.line >= frag.size();
}
Tok get(Language = Language::Spad);
private:
Frag& frag;
FragmentCursor pos;
- std::stack<ColumnIndex> indents;
+ std::stack<Locus> indents;
std::size_t line_length() const { return frag(pos).size(); }
- LineNumber next_line_number() const {
- return pos.line + 1 < frag.size()
- ? frag[pos.line + 1].number
- : frag.back().number + 1;
- }
- ColumnIndex next_indentation() const {
- return pos.line + 1 < frag.size() ? frag[pos.line + 1].indent : 0;
- }
-
- LineNumber line_number() const {
- return pos.line < frag.size()
- ? frag(pos).number
- : frag.back().number + 1;
- }
- ColumnIndex column_number() const {
- return pos.line < frag.size() ? pos.column : 0;
- }
-
- using Locus = typename Tok::Location;
- Locus current_locus() {
- return { line_number(), column_number() };
+ bool line_continuation() const {
+ return pos.column == line_length() - 1 and frag(pos).back() == '_';
}
Tok finish(Tok&, Language);
@@ -212,6 +192,13 @@ namespace OpenAxiom {
}
template<typename T>
+ inline T& formatting_token(T& t, TokenValue v) {
+ t.category = TokenCategory::Formatting;
+ t.value = v;
+ return t;
+ }
+
+ template<typename T>
inline void operator_token(T& t, TokenValue v) {
t.category = TokenCategory::Operator;
t.value = v;
@@ -223,6 +210,21 @@ namespace OpenAxiom {
t.value = v;
}
+ template<typename T>
+ inline T& eos_token(T& t, const FragmentCursor& pos) {
+ t.category = TokenCategory::EOS;
+ t.value = TokenValue::EndOfStream;
+ t.end = pos;
+ return t;
+ }
+
+ template<typename T>
+ inline T& ws_token(T& t, const FragmentCursor& pos) {
+ t.category = TokenCategory::Whitespace;
+ t.end = pos;
+ return t;
+ }
+
template<typename L, typename T>
static void junk(L& line, ColumnIndex& idx, T& t) {
while (idx < line.size() and not separator_or_punctuator(line[idx]))
@@ -268,6 +270,16 @@ namespace OpenAxiom {
++idx;
}
+ template<typename Frag>
+ bool next_line(Frag& frag, FragmentCursor& pos) {
+ if (++pos.line < frag.size()) {
+ pos.column = frag(pos).indent;
+ return true;
+ }
+ pos.column = 0;
+ return false;
+ }
+
template<typename L, typename T>
void integer(L& line, ColumnIndex& idx, T& t) {
skip_to_end_of_integer(line, idx);
@@ -507,7 +519,7 @@ namespace OpenAxiom {
case '>': t.value = TokenValue::RightArrow; ++pos; break;
case '-':
comment_token(t, TokenValue::Wisecrack);
- pos.column = frag(pos).size();
+ next_line(frag, pos);
break;
}
}
@@ -521,7 +533,7 @@ namespace OpenAxiom {
switch (frag[pos]) {
case '+':
comment_token(t, TokenValue::Commentary);
- pos.column = frag(pos).size();
+ next_line(frag, pos);
break;
case '-':
if (pos.column + 1 < frag(pos).size()
@@ -621,47 +633,34 @@ namespace OpenAxiom {
template<typename Frag, typename Tok>
Tok Tokenizer<Frag, Tok>::get(Language dialect) {
Tok t { };
- t.start = current_locus();
+ t.start = pos;
- if (eos()) {
- t.category = TokenCategory::EOS;
- t.end = current_locus();
- return t;
- }
+ if (eos())
+ return eos_token(t, pos);
else if (isblank(frag[pos])) {
skip_whitespace(frag(pos), pos.column);
- t.category = TokenCategory::Whitespace;
- t.end = current_locus();
- return t;
+ return ws_token(t, pos);
}
- else if (pos.column == line_length() - 1 and frag(pos).back() == '_') {
- ++pos.line;
- pos.column = frag(pos).indent;
- return finish(t, dialect);
+ else if (line_continuation()) {
+ if (next_line(frag, pos))
+ return finish(t, dialect);
+ return eos_token(t, pos);
}
- else if (pos.column == line_length()) {
+ else if (pos.column >= line_length()) {
+ if (not next_line(frag, pos))
+ return eos_token(t, pos);
+ t.start = t.end = pos;
auto indent = indents.top();
- auto next_indent = next_indentation();
- t.start = t.end = { next_line_number(), next_indent };
- if (indent < next_indent) {
- indents.push(next_indent);
- ++pos.line;
- pos.column = next_indent;
- t.category = TokenCategory::Formatting;
- t.value = TokenValue::Indent;
+ if (indent.column < pos.column) {
+ indents.push(pos);
+ return formatting_token(t, TokenValue::Indent);
}
- else if (indent > next_indent) {
+ else if (indent.column > pos.column) {
indents.pop();
- t.category = TokenCategory::Formatting;
- t.value = TokenValue::Unindent;
+ return formatting_token(t, TokenValue::Unindent);
}
- else {
- ++pos.line;
- pos.column = next_indent;
- t.category = TokenCategory::Formatting;
- t.value = TokenValue::Justify;
- }
- return t;
+
+ return formatting_token(t, TokenValue::Justify);
}
return finish(t, dialect);