aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/include/open-axiom/InputFragment11
-rw-r--r--src/include/open-axiom/token35
-rw-r--r--src/io/InputFragment.cxx5
-rw-r--r--src/syntax/token.cxx7
4 files changed, 24 insertions, 34 deletions
diff --git a/src/include/open-axiom/InputFragment b/src/include/open-axiom/InputFragment
index 63139db4..67863e44 100644
--- a/src/include/open-axiom/InputFragment
+++ b/src/include/open-axiom/InputFragment
@@ -36,6 +36,7 @@
#ifndef OPENAXIOM_INPUTFRAGMENT_included
#define OPENAXIOM_INPUTFRAGMENT_included
+#include <stdint.h>
#include <vector>
#include <string>
#include <stack>
@@ -43,7 +44,7 @@
namespace OpenAxiom {
// Datatypes for locating lines and columns.
using LineNumber = std::size_t;
- using ColumnIndex = std::size_t;
+ using ColumnIndex = uint16_t;
enum class LineKind : uint8_t {
Ordinary, // Ordinary input line
@@ -65,9 +66,11 @@ namespace OpenAxiom {
};
// Cursor into a fragment.
+ // Note: We don't expect people to write large fragements
+ // either in length or in width.
struct FragmentCursor {
- std::size_t line; // index of a line in a fragment
- std::size_t column; // column number at line.
+ uint16_t line; // index of a line in a fragment
+ uint16_t column; // column number at line.
inline FragmentCursor& operator++() {
++column;
@@ -92,6 +95,8 @@ namespace OpenAxiom {
}
};
+ std::ostream& operator<<(std::ostream&, FragmentCursor);
+
// A program fragment is a logical line, composed of possibly
// several physical lines subject to the off-side rule. As a
// special case, a line ending with the underbar character
diff --git a/src/include/open-axiom/token b/src/include/open-axiom/token
index f487cb3b..943d01cc 100644
--- a/src/include/open-axiom/token
+++ b/src/include/open-axiom/token
@@ -72,17 +72,10 @@ namespace OpenAxiom {
std::ostream& operator<<(std::ostream&, TokenValue);
- struct Locus {
- LineNumber line;
- ColumnIndex column;
- };
-
- std::ostream& operator<<(std::ostream&, const Locus&);
-
- // Program text region
+ // Program text region, with a fragment.
struct Region {
- Locus start;
- Locus end;
+ FragmentCursor start;
+ FragmentCursor end;
};
// Given a symbolic or alphabetic token, retrieve its category
@@ -99,9 +92,7 @@ namespace OpenAxiom {
TokenClassification classify(const std::string&);
// Token data structure: a region of text with a classification.
- struct Token : Region, TokenClassification {
- using Location = Locus;
- };
+ struct Token : Region, TokenClassification { };
// -- Exception types
struct EndOfStringUnseen {
@@ -180,9 +171,9 @@ namespace OpenAxiom {
}
template<typename T>
- inline T& ws_token(T& t, const Locus& loc) {
+ inline T& ws_token(T& t, FragmentCursor pos) {
t.category = TokenCategory::Whitespace;
- t.end = loc;
+ t.end = pos;
return t;
}
@@ -571,26 +562,20 @@ namespace OpenAxiom {
break;
}
- t.end = { frag(pos).number, pos.column };
+ t.end = pos;
return t;
}
- inline Locus location(const Fragment& frag, const FragmentCursor& pos) {
- if (pos.line < frag.size())
- return { frag[pos.line].number, pos.column };
- return { frag.back().number + 1, { } };
- }
-
template<typename Tok>
Tok Tokenizer<Tok>::get(Language dialect) {
Tok t { };
- t.start = location(frag, pos);
+ t.start = pos;
if (eos())
return eos_token(t);
else if (isblank(frag[pos])) {
skip_whitespace(frag(pos), pos.column);
- return ws_token(t, location(frag, pos));
+ return ws_token(t, pos);
}
else if (line_continuation()) {
if (next_line(frag, pos))
@@ -600,7 +585,7 @@ namespace OpenAxiom {
else if (pos.column >= line_length()) {
if (not next_line(frag, pos))
return eos_token(t);
- t.start = t.end = location(frag, pos);
+ t.start = t.end = pos;
auto indent = indents.top();
if (indent.column < pos.column) {
indents.push(pos);
diff --git a/src/io/InputFragment.cxx b/src/io/InputFragment.cxx
index 01970f84..595b88b1 100644
--- a/src/io/InputFragment.cxx
+++ b/src/io/InputFragment.cxx
@@ -41,6 +41,11 @@
#include <open-axiom/SourceInput>
namespace OpenAxiom {
+
+ std::ostream& operator<<(std::ostream& os, FragmentCursor cur) {
+ return os << '{' << cur.line << '-' << cur.column << '}';
+ }
+
// Formatting program fragments.
std::ostream& operator<<(std::ostream& os, const Fragment& f) {
std::copy(f.begin(), f.end(),
diff --git a/src/syntax/token.cxx b/src/syntax/token.cxx
index 011520a8..e1c02e3f 100644
--- a/src/syntax/token.cxx
+++ b/src/syntax/token.cxx
@@ -35,12 +35,7 @@
#include <iostream>
namespace OpenAxiom {
- std::ostream& operator<<(std::ostream& os, const Locus& l) {
- return os << '{' << l.line << ", " << l.column << '}';
- }
-
- std::ostream&
- operator<<(std::ostream& os, TokenCategory tc) {
+ std::ostream& operator<<(std::ostream& os, TokenCategory tc) {
switch (tc) {
case TokenCategory::Unclassified: os << "UNCLASSIFIED"; break;
case TokenCategory::Whitespace: os << "WHITESPACE"; break;