mirrors/serenity
1
0
Fork 0
mirror of https://github.com/SerenityOS/serenity.git synced 2025-01-24 02:12:09 -05:00
Code Issues Projects Releases Packages Wiki Activity

HackStudio: Make CppTokens have (line,column) positions

Browse source 
These are infinitely more useful than raw indices into the input text.
This commit is contained in:
Andreas Kling 2019-10-25 21:07:45 +02:00
parent 0d53d74d5f
commit 0604fcf9fd
2 changed files with 27 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
DevTools/HackStudio/CppLexer.cpp
View file

@ -17,7 +17,15 @@ char CppLexer::peek(int offset) const
char CppLexer::consume()
{
ASSERT(m_index < m_input.length());
return m_input[m_index++];
char ch = m_input[m_index++];
m_previous_position = m_position;
if (ch == '\n') {
m_position.line++;
m_position.column = 0;
} else {
m_position.column++;
}
return ch;
}
static bool is_valid_first_character_of_identifier(char ch)
@ -41,22 +49,27 @@ Vector<CppToken> CppLexer::lex()
{
Vector<CppToken> tokens;
int token_start_index = 0;
CppPosition token_start_position;
auto emit_token = [&](auto type) {
CppToken token;
token.m_type = type;
token.m_view = StringView(m_input.characters_without_null_termination() + m_index, 1);
token.m_start = m_position;
token.m_end = m_position;
tokens.append(token);
m_index++;
consume();
};
int token_start_index = 0;
auto begin_token = [&] {
token_start_index = m_index;
token_start_position = m_position;
};
auto commit_token = [&](auto type) {
CppToken token;
token.m_type = type;
token.m_view = StringView(m_input.characters_without_null_termination() + token_start_index, m_index - token_start_index);
token.m_start = token_start_position;
token.m_end = m_previous_position;
tokens.append(token);
};

10
DevTools/HackStudio/CppLexer.h
View file

@ -23,6 +23,11 @@
__TOKEN(Keyword) \
__TOKEN(Identifier)
struct CppPosition {
int line { -1 };
int column { -1 };
};
struct CppToken {
enum class Type {
#define __TOKEN(x) x,
@ -43,7 +48,8 @@ struct CppToken {
}
Type m_type { Type::Invalid };
StringView m_view;
CppPosition m_start;
CppPosition m_end;
};
class CppLexer {
@ -58,4 +64,6 @@ private:
StringView m_input;
int m_index { 0 };
CppPosition m_previous_position { 0, 0 };
CppPosition m_position { 0, 0 };
};