encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pre-parse token list. 

This CL updates the parser and lexer to pre-parse the token list and
then the parser works off the list of tokens. This allows the parser to
work with references to the tokens and not have to worry about them
going stale.

For any splittable token a placeholder token is injected after the
token. If the token ends up getting split the placeholder is overwritten
with the new type.

Change-Id: I5a8ccca15d8c14b5027df7dd2734be6753e46fa9
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/97070
Reviewed-by: Ben Clayton <bclayton@google.com>
Commit-Queue: Dan Sinclair <dsinclair@chromium.org>
Kokoro: Kokoro <noreply+kokoro@google.com>
This commit is contained in:
dan sinclair 2022-07-25 16:43:08 +00:00 committed by Dawn LUCI CQ
parent 1308395055
commit 0cbf5a922f
12 changed files with 493 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/tint/reader/wgsl/lexer.cc
View File

@ -37,6 +37,8 @@ static_assert(sizeof(decltype(tint::Source::FileContent::data[0])) == sizeof(uin
"tint::reader::wgsl requires the size of a std::string element " "tint::reader::wgsl requires the size of a std::string element "
"to be a single byte"); "to be a single byte");
static constexpr size_t kDefaultListSize = 512;
bool read_blankspace(std::string_view str, size_t i, bool* is_blankspace, size_t* blankspace_size) { bool read_blankspace(std::string_view str, size_t i, bool* is_blankspace, size_t* blankspace_size) {
// See https://www.w3.org/TR/WGSL/#blankspace // See https://www.w3.org/TR/WGSL/#blankspace
@ -88,6 +90,27 @@ Lexer::Lexer(const Source::File* file) : file_(file), location_{1, 1} {}
Lexer::~Lexer() = default; Lexer::~Lexer() = default;
std::vector<Token> Lexer::Lex() {
std::vector<Token> tokens;
tokens.reserve(kDefaultListSize);
while (true) {
tokens.emplace_back(next());
// If the token can be split, we insert a placeholder element into
// the stream to hold the split character.
if (tokens.back().IsSplittable()) {
auto src = tokens.back().source();
src.range.begin.column++;
tokens.emplace_back(Token(Token::Type::kPlaceholder, src));
}
if (tokens.back().IsEof() || tokens.back().IsError()) {
break;
}
}
return tokens;
}
const std::string_view Lexer::line() const { const std::string_view Lexer::line() const {
if (file_->content.lines.size() == 0) { if (file_->content.lines.size() == 0) {
static const char* empty_string = ""; static const char* empty_string = "";

6
src/tint/reader/wgsl/lexer.h
View File

@ -16,6 +16,7 @@
#define SRC_TINT_READER_WGSL_LEXER_H_ #define SRC_TINT_READER_WGSL_LEXER_H_
#include <string> #include <string>
#include <vector>
#include "src/tint/reader/wgsl/token.h" #include "src/tint/reader/wgsl/token.h"
@ -29,11 +30,14 @@ class Lexer {
explicit Lexer(const Source::File* file); explicit Lexer(const Source::File* file);
~Lexer(); ~Lexer();
/// @return the token list.
std::vector<Token> Lex();
private:
/// Returns the next token in the input stream. /// Returns the next token in the input stream.
/// @return Token /// @return Token
Token next(); Token next();
private:
/// Advances past blankspace and comments, if present at the current position. /// Advances past blankspace and comments, if present at the current position.
/// @returns error token, EOF, or uninitialized /// @returns error token, EOF, or uninitialized
Token skip_blankspace_and_comments(); Token skip_blankspace_and_comments();

474
src/tint/reader/wgsl/lexer_test.cc
View File

@ -46,24 +46,33 @@ using LexerTest = testing::Test;
TEST_F(LexerTest, Empty) { TEST_F(LexerTest, Empty) {
Source::File file("", ""); Source::File file("", "");
Lexer l(&file); Lexer l(&file);
auto t = l.next();
EXPECT_TRUE(t.IsEof()); auto list = l.Lex();
ASSERT_EQ(1u, list.size());
EXPECT_TRUE(list[0].IsEof());
} }
TEST_F(LexerTest, Skips_Blankspace_Basic) { TEST_F(LexerTest, Skips_Blankspace_Basic) {
Source::File file("", "\t\r\n\t ident\t\n\t \r "); Source::File file("", "\t\r\n\t ident\t\n\t \r ");
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.IsIdentifier()); ASSERT_EQ(2u, list.size());
EXPECT_EQ(t.source().range.begin.line, 2u);
EXPECT_EQ(t.source().range.begin.column, 6u);
EXPECT_EQ(t.source().range.end.line, 2u);
EXPECT_EQ(t.source().range.end.column, 11u);
EXPECT_EQ(t.to_str(), "ident");
t = l.next(); {
EXPECT_TRUE(t.IsEof()); auto& t = list[0];
EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.line, 2u);
EXPECT_EQ(t.source().range.begin.column, 6u);
EXPECT_EQ(t.source().range.end.line, 2u);
EXPECT_EQ(t.source().range.end.column, 11u);
EXPECT_EQ(t.to_str(), "ident");
}
{
auto& t = list[1];
EXPECT_TRUE(t.IsEof());
}
} }
TEST_F(LexerTest, Skips_Blankspace_Exotic) { TEST_F(LexerTest, Skips_Blankspace_Exotic) {
@ -73,16 +82,23 @@ TEST_F(LexerTest, Skips_Blankspace_Exotic) {
kVTab kFF kNL kLS kPS kL2R kR2L); kVTab kFF kNL kLS kPS kL2R kR2L);
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.IsIdentifier()); ASSERT_EQ(2u, list.size());
EXPECT_EQ(t.source().range.begin.line, 6u);
EXPECT_EQ(t.source().range.begin.column, 7u);
EXPECT_EQ(t.source().range.end.line, 6u);
EXPECT_EQ(t.source().range.end.column, 12u);
EXPECT_EQ(t.to_str(), "ident");
t = l.next(); {
EXPECT_TRUE(t.IsEof()); auto& t = list[0];
EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.line, 6u);
EXPECT_EQ(t.source().range.begin.column, 7u);
EXPECT_EQ(t.source().range.end.line, 6u);
EXPECT_EQ(t.source().range.end.column, 12u);
EXPECT_EQ(t.to_str(), "ident");
}
{
auto& t = list[1];
EXPECT_TRUE(t.IsEof());
}
} }
TEST_F(LexerTest, Skips_Comments_Line) { TEST_F(LexerTest, Skips_Comments_Line) {
@ -92,24 +108,33 @@ ident1 //ends with comment
ident2)"); ident2)");
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.IsIdentifier()); ASSERT_EQ(3u, list.size());
EXPECT_EQ(t.source().range.begin.line, 2u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 2u);
EXPECT_EQ(t.source().range.end.column, 7u);
EXPECT_EQ(t.to_str(), "ident1");
t = l.next(); {
EXPECT_TRUE(t.IsIdentifier()); auto& t = list[0];
EXPECT_EQ(t.source().range.begin.line, 4u); EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.column, 2u); EXPECT_EQ(t.source().range.begin.line, 2u);
EXPECT_EQ(t.source().range.end.line, 4u); EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.column, 8u); EXPECT_EQ(t.source().range.end.line, 2u);
EXPECT_EQ(t.to_str(), "ident2"); EXPECT_EQ(t.source().range.end.column, 7u);
EXPECT_EQ(t.to_str(), "ident1");
}
t = l.next(); {
EXPECT_TRUE(t.IsEof()); auto& t = list[1];
EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.line, 4u);
EXPECT_EQ(t.source().range.begin.column, 2u);
EXPECT_EQ(t.source().range.end.line, 4u);
EXPECT_EQ(t.source().range.end.column, 8u);
EXPECT_EQ(t.to_str(), "ident2");
}
{
auto& t = list[2];
EXPECT_TRUE(t.IsEof());
}
} }
TEST_F(LexerTest, Skips_Comments_Unicode) { TEST_F(LexerTest, Skips_Comments_Unicode) {
@ -119,24 +144,33 @@ ident1 //ends with 🙂🙂🙂
ident2)"); ident2)");
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.IsIdentifier()); ASSERT_EQ(3u, list.size());
EXPECT_EQ(t.source().range.begin.line, 2u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 2u);
EXPECT_EQ(t.source().range.end.column, 7u);
EXPECT_EQ(t.to_str(), "ident1");
t = l.next(); {
EXPECT_TRUE(t.IsIdentifier()); auto& t = list[0];
EXPECT_EQ(t.source().range.begin.line, 4u); EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.column, 2u); EXPECT_EQ(t.source().range.begin.line, 2u);
EXPECT_EQ(t.source().range.end.line, 4u); EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.column, 8u); EXPECT_EQ(t.source().range.end.line, 2u);
EXPECT_EQ(t.to_str(), "ident2"); EXPECT_EQ(t.source().range.end.column, 7u);
EXPECT_EQ(t.to_str(), "ident1");
}
t = l.next(); {
EXPECT_TRUE(t.IsEof()); auto& t = list[1];
EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.line, 4u);
EXPECT_EQ(t.source().range.begin.column, 2u);
EXPECT_EQ(t.source().range.end.line, 4u);
EXPECT_EQ(t.source().range.end.column, 8u);
EXPECT_EQ(t.to_str(), "ident2");
}
{
auto& t = list[2];
EXPECT_TRUE(t.IsEof());
}
} }
using LineCommentTerminatorTest = testing::TestWithParam<const char*>; using LineCommentTerminatorTest = testing::TestWithParam<const char*>;
@ -150,21 +184,29 @@ TEST_P(LineCommentTerminatorTest, Terminators) {
Source::File file("", src); Source::File file("", src);
Lexer l(&file); Lexer l(&file);
auto t = l.next();
EXPECT_TRUE(t.Is(Token::Type::kConst));
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 6u);
auto is_same_line = [](std::string_view v) { auto is_same_line = [](std::string_view v) {
return v == kSpace || v == kHTab || v == kL2R || v == kR2L; return v == kSpace || v == kHTab || v == kL2R || v == kR2L;
}; };
auto list = l.Lex();
ASSERT_EQ(is_same_line(c) ? 2u : 3u, list.size());
size_t idx = 0;
{
auto& t = list[idx++];
EXPECT_TRUE(t.Is(Token::Type::kConst));
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 6u);
}
if (!is_same_line(c)) { if (!is_same_line(c)) {
size_t line = is_same_line(c) ? 1u : 2u; size_t line = is_same_line(c) ? 1u : 2u;
size_t col = is_same_line(c) ? 25u : 1u; size_t col = is_same_line(c) ? 25u : 1u;
t = l.next();
auto& t = list[idx++];
EXPECT_TRUE(t.IsIdentifier()); EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.line, line); EXPECT_EQ(t.source().range.begin.line, line);
EXPECT_EQ(t.source().range.begin.column, col); EXPECT_EQ(t.source().range.begin.column, col);
@ -173,8 +215,10 @@ TEST_P(LineCommentTerminatorTest, Terminators) {
EXPECT_EQ(t.to_str(), "ident"); EXPECT_EQ(t.to_str(), "ident");
} }
t = l.next(); {
EXPECT_TRUE(t.IsEof()); auto& t = list[idx];
EXPECT_TRUE(t.IsEof());
}
} }
INSTANTIATE_TEST_SUITE_P(LexerTest, INSTANTIATE_TEST_SUITE_P(LexerTest,
LineCommentTerminatorTest, LineCommentTerminatorTest,
@ -198,16 +242,23 @@ TEST_F(LexerTest, Skips_Comments_Block) {
text */ident)"); text */ident)");
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.IsIdentifier()); ASSERT_EQ(2u, list.size());
EXPECT_EQ(t.source().range.begin.line, 2u);
EXPECT_EQ(t.source().range.begin.column, 8u);
EXPECT_EQ(t.source().range.end.line, 2u);
EXPECT_EQ(t.source().range.end.column, 13u);
EXPECT_EQ(t.to_str(), "ident");
t = l.next(); {
EXPECT_TRUE(t.IsEof()); auto& t = list[0];
EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.line, 2u);
EXPECT_EQ(t.source().range.begin.column, 8u);
EXPECT_EQ(t.source().range.end.line, 2u);
EXPECT_EQ(t.source().range.end.column, 13u);
EXPECT_EQ(t.to_str(), "ident");
}
{
auto& t = list[1];
EXPECT_TRUE(t.IsEof());
}
} }
TEST_F(LexerTest, Skips_Comments_Block_Nested) { TEST_F(LexerTest, Skips_Comments_Block_Nested) {
@ -216,16 +267,23 @@ text // nested line comments are ignored /* more text
/////**/ */*/ident)"); /////**/ */*/ident)");
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.IsIdentifier()); ASSERT_EQ(2u, list.size());
EXPECT_EQ(t.source().range.begin.line, 3u);
EXPECT_EQ(t.source().range.begin.column, 14u);
EXPECT_EQ(t.source().range.end.line, 3u);
EXPECT_EQ(t.source().range.end.column, 19u);
EXPECT_EQ(t.to_str(), "ident");
t = l.next(); {
EXPECT_TRUE(t.IsEof()); auto& t = list[0];
EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.line, 3u);
EXPECT_EQ(t.source().range.begin.column, 14u);
EXPECT_EQ(t.source().range.end.line, 3u);
EXPECT_EQ(t.source().range.end.column, 19u);
EXPECT_EQ(t.to_str(), "ident");
}
{
auto& t = list[1];
EXPECT_TRUE(t.IsEof());
}
} }
TEST_F(LexerTest, Skips_Comments_Block_Unterminated) { TEST_F(LexerTest, Skips_Comments_Block_Unterminated) {
@ -237,7 +295,10 @@ TEST_F(LexerTest, Skips_Comments_Block_Unterminated) {
abcd)"); abcd)");
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
ASSERT_EQ(1u, list.size());
auto& t = list[0];
ASSERT_TRUE(t.Is(Token::Type::kError)); ASSERT_TRUE(t.Is(Token::Type::kError));
EXPECT_EQ(t.to_str(), "unterminated block comment"); EXPECT_EQ(t.to_str(), "unterminated block comment");
EXPECT_EQ(t.source().range.begin.line, 2u); EXPECT_EQ(t.source().range.begin.line, 2u);
@ -250,7 +311,10 @@ TEST_F(LexerTest, Null_InBlankspace_IsError) {
Source::File file("", std::string{' ', 0, ' '}); Source::File file("", std::string{' ', 0, ' '});
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
ASSERT_EQ(1u, list.size());
auto& t = list[0];
EXPECT_TRUE(t.IsError()); EXPECT_TRUE(t.IsError());
EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 2u); EXPECT_EQ(t.source().range.begin.column, 2u);
@ -263,7 +327,10 @@ TEST_F(LexerTest, Null_InLineComment_IsError) {
Source::File file("", std::string{'/', '/', ' ', 0, ' '}); Source::File file("", std::string{'/', '/', ' ', 0, ' '});
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
ASSERT_EQ(1u, list.size());
auto& t = list[0];
EXPECT_TRUE(t.IsError()); EXPECT_TRUE(t.IsError());
EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 4u); EXPECT_EQ(t.source().range.begin.column, 4u);
@ -276,7 +343,10 @@ TEST_F(LexerTest, Null_InBlockComment_IsError) {
Source::File file("", std::string{'/', '*', ' ', 0, '*', '/'}); Source::File file("", std::string{'/', '*', ' ', 0, '*', '/'});
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
ASSERT_EQ(1u, list.size());
auto& t = list[0];
EXPECT_TRUE(t.IsError()); EXPECT_TRUE(t.IsError());
EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 4u); EXPECT_EQ(t.source().range.begin.column, 4u);
@ -292,16 +362,24 @@ TEST_F(LexerTest, Null_InIdentifier_IsError) {
Source::File file("", std::string{'a', 0, 'c'}); Source::File file("", std::string{'a', 0, 'c'});
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.IsIdentifier()); ASSERT_EQ(2u, list.size());
EXPECT_EQ(t.to_str(), "a");
t = l.next(); {
EXPECT_TRUE(t.IsError()); auto& t = list[0];
EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.source().range.begin.column, 2u); EXPECT_EQ(t.to_str(), "a");
EXPECT_EQ(t.source().range.end.line, 1u); }
EXPECT_EQ(t.source().range.end.column, 2u);
EXPECT_EQ(t.to_str(), "null character found"); {
auto& t = list[1];
EXPECT_TRUE(t.IsError());
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 2u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 2u);
EXPECT_EQ(t.to_str(), "null character found");
}
} }
struct FloatData { struct FloatData {
@ -318,22 +396,29 @@ TEST_P(FloatTest, Parse) {
Source::File file("", params.input); Source::File file("", params.input);
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
if (std::string(params.input).back() == 'f') { ASSERT_EQ(2u, list.size());
EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral_F));
} else if (std::string(params.input).back() == 'h') {
EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral_H));
} else {
EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral));
}
EXPECT_EQ(t.to_f64(), params.result);
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
t = l.next(); {
EXPECT_TRUE(t.IsEof()); auto& t = list[0];
if (std::string(params.input).back() == 'f') {
EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral_F));
} else if (std::string(params.input).back() == 'h') {
EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral_H));
} else {
EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral));
}
EXPECT_EQ(t.to_f64(), params.result);
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
}
{
auto& t = list[1];
EXPECT_TRUE(t.IsEof());
}
} }
INSTANTIATE_TEST_SUITE_P(LexerTest, INSTANTIATE_TEST_SUITE_P(LexerTest,
FloatTest, FloatTest,
@ -437,7 +522,10 @@ TEST_P(FloatTest_Invalid, Handles) {
Source::File file("", GetParam()); Source::File file("", GetParam());
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
ASSERT_FALSE(list.empty());
auto& t = list[0];
EXPECT_FALSE(t.Is(Token::Type::kFloatLiteral) || t.Is(Token::Type::kFloatLiteral_F) || EXPECT_FALSE(t.Is(Token::Type::kFloatLiteral) || t.Is(Token::Type::kFloatLiteral_F) ||
t.Is(Token::Type::kFloatLiteral_H)); t.Is(Token::Type::kFloatLiteral_H));
} }
@ -478,13 +566,23 @@ TEST_P(AsciiIdentifierTest, Parse) {
Source::File file("", GetParam()); Source::File file("", GetParam());
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.IsIdentifier()); ASSERT_EQ(2u, list.size());
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u); {
EXPECT_EQ(t.source().range.end.line, 1u); auto& t = list[0];
EXPECT_EQ(t.source().range.end.column, 1u + strlen(GetParam())); EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.to_str(), GetParam()); EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(GetParam()));
EXPECT_EQ(t.to_str(), GetParam());
}
{
auto& t = list[1];
EXPECT_TRUE(t.IsEof());
}
} }
INSTANTIATE_TEST_SUITE_P(LexerTest, INSTANTIATE_TEST_SUITE_P(LexerTest,
AsciiIdentifierTest, AsciiIdentifierTest,
@ -510,13 +608,23 @@ TEST_P(ValidUnicodeIdentifierTest, Parse) {
Source::File file("", GetParam().utf8); Source::File file("", GetParam().utf8);
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.IsIdentifier()); ASSERT_EQ(2u, list.size());
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u); {
EXPECT_EQ(t.source().range.end.line, 1u); auto& t = list[0];
EXPECT_EQ(t.source().range.end.column, 1u + GetParam().count); EXPECT_TRUE(t.IsIdentifier());
EXPECT_EQ(t.to_str(), GetParam().utf8); EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + GetParam().count);
EXPECT_EQ(t.to_str(), GetParam().utf8);
}
{
auto& t = list[1];
EXPECT_TRUE(t.IsEof());
}
} }
INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P(
LexerTest, LexerTest,
@ -554,7 +662,10 @@ TEST_P(InvalidUnicodeIdentifierTest, Parse) {
Source::File file("", GetParam()); Source::File file("", GetParam());
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
ASSERT_FALSE(list.empty());
auto& t = list[0];
EXPECT_TRUE(t.IsError()); EXPECT_TRUE(t.IsError());
EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u); EXPECT_EQ(t.source().range.begin.column, 1u);
@ -602,7 +713,10 @@ TEST_F(LexerTest, IdentifierTest_SingleUnderscoreDoesNotMatch) {
Source::File file("", "_"); Source::File file("", "_");
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
ASSERT_FALSE(list.empty());
auto& t = list[0];
EXPECT_FALSE(t.IsIdentifier()); EXPECT_FALSE(t.IsIdentifier());
} }
@ -610,7 +724,10 @@ TEST_F(LexerTest, IdentifierTest_DoesNotStartWithDoubleUnderscore) {
Source::File file("", "__test"); Source::File file("", "__test");
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
ASSERT_FALSE(list.empty());
auto& t = list[0];
EXPECT_FALSE(t.IsIdentifier()); EXPECT_FALSE(t.IsIdentifier());
} }
@ -618,7 +735,10 @@ TEST_F(LexerTest, IdentifierTest_DoesNotStartWithNumber) {
Source::File file("", "01test"); Source::File file("", "01test");
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_FALSE(list.empty());
auto& t = list[0];
EXPECT_FALSE(t.IsIdentifier()); EXPECT_FALSE(t.IsIdentifier());
} }
@ -641,7 +761,12 @@ TEST_P(ParseIntegerTest, Parse) {
auto params = std::get<1>(GetParam()); auto params = std::get<1>(GetParam());
Source::File file("", params.input); Source::File file("", params.input);
auto t = Lexer(&file).next(); Lexer l(&file);
auto list = l.Lex();
ASSERT_FALSE(list.empty());
auto& t = list[0];
switch (suffix) { switch (suffix) {
case 'i': case 'i':
EXPECT_TRUE(t.Is(Token::Type::kIntLiteral_I)); EXPECT_TRUE(t.Is(Token::Type::kIntLiteral_I));
@ -770,7 +895,12 @@ TEST_P(ParseIntegerTest_CannotBeRepresented, Parse) {
auto type = std::get<0>(GetParam()); auto type = std::get<0>(GetParam());
auto source = std::get<1>(GetParam()); auto source = std::get<1>(GetParam());
Source::File file("", source); Source::File file("", source);
auto t = Lexer(&file).next();
Lexer l(&file);
auto list = l.Lex();
ASSERT_FALSE(list.empty());
auto& t = list[0];
EXPECT_TRUE(t.Is(Token::Type::kError)); EXPECT_TRUE(t.Is(Token::Type::kError));
auto expect = "value cannot be represented as '" + std::string(type) + "'"; auto expect = "value cannot be represented as '" + std::string(type) + "'";
EXPECT_EQ(t.to_str(), expect); EXPECT_EQ(t.to_str(), expect);
@ -800,7 +930,12 @@ INSTANTIATE_TEST_SUITE_P(u32,
using ParseIntegerTest_LeadingZeros = testing::TestWithParam<const char*>; using ParseIntegerTest_LeadingZeros = testing::TestWithParam<const char*>;
TEST_P(ParseIntegerTest_LeadingZeros, Parse) { TEST_P(ParseIntegerTest_LeadingZeros, Parse) {
Source::File file("", GetParam()); Source::File file("", GetParam());
auto t = Lexer(&file).next();
Lexer l(&file);
auto list = l.Lex();
ASSERT_FALSE(list.empty());
auto& t = list[0];
EXPECT_TRUE(t.Is(Token::Type::kError)); EXPECT_TRUE(t.Is(Token::Type::kError));
EXPECT_EQ(t.to_str(), "integer literal cannot have leading 0s"); EXPECT_EQ(t.to_str(), "integer literal cannot have leading 0s");
} }
@ -815,7 +950,12 @@ INSTANTIATE_TEST_SUITE_P(LeadingZero,
using ParseIntegerTest_NoSignificantDigits = testing::TestWithParam<const char*>; using ParseIntegerTest_NoSignificantDigits = testing::TestWithParam<const char*>;
TEST_P(ParseIntegerTest_NoSignificantDigits, Parse) { TEST_P(ParseIntegerTest_NoSignificantDigits, Parse) {
Source::File file("", GetParam()); Source::File file("", GetParam());
auto t = Lexer(&file).next();
Lexer l(&file);
auto list = l.Lex();
ASSERT_FALSE(list.empty());
auto& t = list[0];
EXPECT_TRUE(t.Is(Token::Type::kError)); EXPECT_TRUE(t.Is(Token::Type::kError));
EXPECT_EQ(t.to_str(), "integer or float hex literal has no significant digits"); EXPECT_EQ(t.to_str(), "integer or float hex literal has no significant digits");
} }
@ -849,15 +989,22 @@ TEST_P(PunctuationTest, Parses) {
Source::File file("", params.input); Source::File file("", params.input);
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
EXPECT_TRUE(t.Is(params.type)); ASSERT_GE(list.size(), 2u);
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
t = l.next(); {
EXPECT_EQ(t.source().range.begin.column, 1 + std::string(params.input).size()); auto& t = list[0];
EXPECT_TRUE(t.Is(params.type));
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
}
{
auto& t = list[1];
EXPECT_EQ(t.source().range.begin.column, 1 + std::string(params.input).size());
}
} }
INSTANTIATE_TEST_SUITE_P(LexerTest, INSTANTIATE_TEST_SUITE_P(LexerTest,
PunctuationTest, PunctuationTest,
@ -876,8 +1023,6 @@ INSTANTIATE_TEST_SUITE_P(LexerTest,
TokenData{"=", Token::Type::kEqual}, TokenData{"=", Token::Type::kEqual},
TokenData{"==", Token::Type::kEqualEqual}, TokenData{"==", Token::Type::kEqualEqual},
TokenData{">", Token::Type::kGreaterThan}, TokenData{">", Token::Type::kGreaterThan},
TokenData{">=", Token::Type::kGreaterThanEqual},
TokenData{">>", Token::Type::kShiftRight},
TokenData{"<", Token::Type::kLessThan}, TokenData{"<", Token::Type::kLessThan},
TokenData{"<=", Token::Type::kLessThanEqual}, TokenData{"<=", Token::Type::kLessThanEqual},
TokenData{"<<", Token::Type::kShiftLeft}, TokenData{"<<", Token::Type::kShiftLeft},
@ -906,21 +1051,60 @@ INSTANTIATE_TEST_SUITE_P(LexerTest,
TokenData{"|=", Token::Type::kOrEqual}, TokenData{"|=", Token::Type::kOrEqual},
TokenData{"^=", Token::Type::kXorEqual})); TokenData{"^=", Token::Type::kXorEqual}));
using SplittablePunctuationTest = testing::TestWithParam<TokenData>;
TEST_P(SplittablePunctuationTest, Parses) {
auto params = GetParam();
Source::File file("", params.input);
Lexer l(&file);
auto list = l.Lex();
ASSERT_GE(list.size(), 3u);
{
auto& t = list[0];
EXPECT_TRUE(t.Is(params.type));
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
}
{
auto& t = list[1];
EXPECT_TRUE(t.Is(Token::Type::kPlaceholder));
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 2u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
}
{
auto& t = list[2];
EXPECT_EQ(t.source().range.begin.column, 1 + std::string(params.input).size());
}
}
INSTANTIATE_TEST_SUITE_P(LexerTest,
SplittablePunctuationTest,
testing::Values(TokenData{">=", Token::Type::kGreaterThanEqual},
TokenData{">>", Token::Type::kShiftRight}));
using KeywordTest = testing::TestWithParam<TokenData>; using KeywordTest = testing::TestWithParam<TokenData>;
TEST_P(KeywordTest, Parses) { TEST_P(KeywordTest, Parses) {
auto params = GetParam(); auto params = GetParam();
Source::File file("", params.input); Source::File file("", params.input);
Lexer l(&file); Lexer l(&file);
auto t = l.next(); auto list = l.Lex();
ASSERT_GE(list.size(), 2u);
auto& t = list[0];
EXPECT_TRUE(t.Is(params.type)) << params.input; EXPECT_TRUE(t.Is(params.type)) << params.input;
EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u); EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u); EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
t = l.next(); EXPECT_EQ(list[1].source().range.begin.column, 1 + std::string(params.input).size());
EXPECT_EQ(t.source().range.begin.column, 1 + std::string(params.input).size());
} }
INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P(
LexerTest, LexerTest,

151
src/tint/reader/wgsl/parser_impl.cc
View File

@ -123,7 +123,7 @@ const char kStageAttribute[] = "stage";
const char kWorkgroupSizeAttribute[] = "workgroup_size"; const char kWorkgroupSizeAttribute[] = "workgroup_size";
// https://gpuweb.github.io/gpuweb/wgsl.html#reserved-keywords // https://gpuweb.github.io/gpuweb/wgsl.html#reserved-keywords
bool is_reserved(Token t) { bool is_reserved(const Token& t) {
return t == "asm" || t == "bf16" || t == "do" || t == "enum" || t == "f64" || t == "handle" || return t == "asm" || t == "bf16" || t == "do" || t == "enum" || t == "f64" || t == "handle" ||
t == "i8" || t == "i16" || t == "i64" || t == "mat" || t == "premerge" || t == "i8" || t == "i16" || t == "i64" || t == "mat" || t == "premerge" ||
t == "regardless" || t == "typedef" || t == "u8" || t == "u16" || t == "u64" || t == "regardless" || t == "typedef" || t == "u8" || t == "u16" || t == "u64" ||
@ -181,7 +181,7 @@ class ParserImpl::MultiTokenSource {
/// Implicit conversion to Source that returns the combined source from start /// Implicit conversion to Source that returns the combined source from start
/// to the current last token's source. /// to the current last token's source.
operator Source() const { operator Source() const {
Source end = parser_->last_token().source().End(); Source end = parser_->last_source().End();
if (end < start_) { if (end < start_) {
end = start_; end = start_;
} }
@ -241,7 +241,7 @@ ParserImpl::VarDeclInfo::VarDeclInfo(Source source_in,
ParserImpl::VarDeclInfo::~VarDeclInfo() = default; ParserImpl::VarDeclInfo::~VarDeclInfo() = default;
ParserImpl::ParserImpl(Source::File const* file) : lexer_(std::make_unique<Lexer>(file)) {} ParserImpl::ParserImpl(Source::File const* file) : file_(file) {}
ParserImpl::~ParserImpl() = default; ParserImpl::~ParserImpl() = default;
@ -274,33 +274,67 @@ void ParserImpl::deprecated(const Source& source, const std::string& msg) {
"use of deprecated language feature: " + msg, source); "use of deprecated language feature: " + msg, source);
} }
Token ParserImpl::next() { const Token& ParserImpl::next() {
if (!token_queue_.empty()) { if (!tokens_[next_token_idx_].IsEof() && !tokens_[next_token_idx_].IsError()) {
auto t = token_queue_.front(); // Skip over any placeholder elements
token_queue_.pop_front(); while (true) {
last_token_ = t; if (!tokens_[next_token_idx_].IsPlaceholder()) {
return last_token_; break;
}
next_token_idx_++;
}
} }
last_token_ = lexer_->next(); last_source_ = tokens_[next_token_idx_].source();
return last_token_; return tokens_[next_token_idx_++];
} }
Token ParserImpl::peek(size_t idx) { const Token& ParserImpl::peek(size_t idx) {
while (token_queue_.size() < (idx + 1)) { if (next_token_idx_ + idx >= tokens_.size()) {
token_queue_.push_back(lexer_->next()); return tokens_[tokens_.size() - 1];
} }
return token_queue_[idx];
// Skip over any placeholder elements
while (true) {
if (!tokens_[next_token_idx_ + idx].IsPlaceholder()) {
break;
}
idx++;
}
return tokens_[next_token_idx_ + idx];
} }
bool ParserImpl::peek_is(Token::Type tok, size_t idx) { bool ParserImpl::peek_is(Token::Type tok, size_t idx) {
return peek(idx).Is(tok); return peek(idx).Is(tok);
} }
Token ParserImpl::last_token() const { void ParserImpl::split_token(Token::Type lhs, Token::Type rhs) {
return last_token_; if (next_token_idx_ == 0) {
TINT_ICE(Reader, builder_.Diagnostics())
<< "attempt to update placeholder at beginning of tokens";
}
if (next_token_idx_ >= tokens_.size()) {
TINT_ICE(Reader, builder_.Diagnostics())
<< "attempt to update placeholder past end of tokens";
}
if (!tokens_[next_token_idx_].IsPlaceholder()) {
TINT_ICE(Reader, builder_.Diagnostics()) << "attempt to update non-placeholder token";
}
tokens_[next_token_idx_ - 1].SetType(lhs);
tokens_[next_token_idx_].SetType(rhs);
}
Source ParserImpl::last_source() const {
return last_source_;
}
void ParserImpl::InitializeLex() {
Lexer l{file_};
tokens_ = l.Lex();
} }
bool ParserImpl::Parse() { bool ParserImpl::Parse() {
InitializeLex();
translation_unit(); translation_unit();
return !has_error(); return !has_error();
} }
@ -310,7 +344,7 @@ bool ParserImpl::Parse() {
void ParserImpl::translation_unit() { void ParserImpl::translation_unit() {
bool after_global_decl = false; bool after_global_decl = false;
while (continue_parsing()) { while (continue_parsing()) {
auto p = peek(); auto& p = peek();
if (p.IsEof()) { if (p.IsEof()) {
break; break;
} }
@ -353,7 +387,7 @@ Maybe<bool> ParserImpl::enable_directive() {
// Match the extension name. // Match the extension name.
Expect<std::string> name = {""}; Expect<std::string> name = {""};
auto t = peek(); auto& t = peek();
if (t.IsIdentifier()) { if (t.IsIdentifier()) {
synchronized_ = true; synchronized_ = true;
next(); next();
@ -508,7 +542,7 @@ Maybe<bool> ParserImpl::global_decl() {
} }
// We have a statement outside of a function? // We have a statement outside of a function?
auto t = peek(); auto& t = peek();
auto stat = without_error([&] { return statement(); }); auto stat = without_error([&] { return statement(); });
if (stat.matched) { if (stat.matched) {
// Attempt to jump to the next '}' - the function might have just been // Attempt to jump to the next '}' - the function might have just been
@ -882,7 +916,7 @@ Maybe<const ast::Type*> ParserImpl::depth_texture() {
// | 'rgba32sint' // | 'rgba32sint'
// | 'rgba32float' // | 'rgba32float'
Expect<ast::TexelFormat> ParserImpl::expect_texel_format(std::string_view use) { Expect<ast::TexelFormat> ParserImpl::expect_texel_format(std::string_view use) {
auto t = next(); auto& t = next();
if (t == "rgba8unorm") { if (t == "rgba8unorm") {
return ast::TexelFormat::kRgba8Unorm; return ast::TexelFormat::kRgba8Unorm;
} }
@ -951,7 +985,7 @@ Expect<ParserImpl::TypedIdentifier> ParserImpl::expect_variable_ident_decl(std::
return Failure::kErrored; return Failure::kErrored;
} }
auto t = peek(); auto& t = peek();
auto type = type_decl(); auto type = type_decl();
if (type.errored) { if (type.errored) {
return Failure::kErrored; return Failure::kErrored;
@ -1021,7 +1055,7 @@ Maybe<const ast::Alias*> ParserImpl::type_alias() {
return Failure::kNoMatch; return Failure::kNoMatch;
} }
auto t = next(); auto& t = next();
const char* use = "type alias"; const char* use = "type alias";
auto name = expect_ident(use); auto name = expect_ident(use);
@ -1069,7 +1103,7 @@ Maybe<const ast::Alias*> ParserImpl::type_alias() {
// | MAT4x4 LESS_THAN type_decl GREATER_THAN // | MAT4x4 LESS_THAN type_decl GREATER_THAN
// | texture_samplers // | texture_samplers
Maybe<const ast::Type*> ParserImpl::type_decl() { Maybe<const ast::Type*> ParserImpl::type_decl() {
auto t = peek(); auto& t = peek();
Source source; Source source;
if (match(Token::Type::kIdentifier, &source)) { if (match(Token::Type::kIdentifier, &source)) {
return builder_.create<ast::TypeName>(source, builder_.Symbols().Register(t.to_str())); return builder_.create<ast::TypeName>(source, builder_.Symbols().Register(t.to_str()));
@ -1139,7 +1173,7 @@ Expect<const ast::Type*> ParserImpl::expect_type(std::string_view use) {
return type.value; return type.value;
} }
Expect<const ast::Type*> ParserImpl::expect_type_decl_pointer(Token t) { Expect<const ast::Type*> ParserImpl::expect_type_decl_pointer(const Token& t) {
const char* use = "ptr declaration"; const char* use = "ptr declaration";
auto storage_class = ast::StorageClass::kNone; auto storage_class = ast::StorageClass::kNone;
@ -1180,7 +1214,7 @@ Expect<const ast::Type*> ParserImpl::expect_type_decl_pointer(Token t) {
access); access);
} }
Expect<const ast::Type*> ParserImpl::expect_type_decl_atomic(Token t) { Expect<const ast::Type*> ParserImpl::expect_type_decl_atomic(const Token& t) {
const char* use = "atomic declaration"; const char* use = "atomic declaration";
auto subtype = expect_lt_gt_block(use, [&] { return expect_type(use); }); auto subtype = expect_lt_gt_block(use, [&] { return expect_type(use); });
@ -1191,7 +1225,7 @@ Expect<const ast::Type*> ParserImpl::expect_type_decl_atomic(Token t) {
return builder_.ty.atomic(make_source_range_from(t.source()), subtype.value); return builder_.ty.atomic(make_source_range_from(t.source()), subtype.value);
} }
Expect<const ast::Type*> ParserImpl::expect_type_decl_vector(Token t) { Expect<const ast::Type*> ParserImpl::expect_type_decl_vector(const Token& t) {
uint32_t count = 2; uint32_t count = 2;
if (t.Is(Token::Type::kVec3)) { if (t.Is(Token::Type::kVec3)) {
count = 3; count = 3;
@ -1212,7 +1246,7 @@ Expect<const ast::Type*> ParserImpl::expect_type_decl_vector(Token t) {
return builder_.ty.vec(make_source_range_from(t.source()), subtype, count); return builder_.ty.vec(make_source_range_from(t.source()), subtype, count);
} }
Expect<const ast::Type*> ParserImpl::expect_type_decl_array(Token t) { Expect<const ast::Type*> ParserImpl::expect_type_decl_array(const Token& t) {
const char* use = "array declaration"; const char* use = "array declaration";
const ast::Expression* size = nullptr; const ast::Expression* size = nullptr;
@ -1244,7 +1278,7 @@ Expect<const ast::Type*> ParserImpl::expect_type_decl_array(Token t) {
return builder_.ty.array(make_source_range_from(t.source()), subtype.value, size); return builder_.ty.array(make_source_range_from(t.source()), subtype.value, size);
} }
Expect<const ast::Type*> ParserImpl::expect_type_decl_matrix(Token t) { Expect<const ast::Type*> ParserImpl::expect_type_decl_matrix(const Token& t) {
uint32_t rows = 2; uint32_t rows = 2;
uint32_t columns = 2; uint32_t columns = 2;
if (t.IsMat3xN()) { if (t.IsMat3xN()) {
@ -1305,7 +1339,7 @@ Expect<ast::StorageClass> ParserImpl::expect_storage_class(std::string_view use)
// struct_decl // struct_decl
// : STRUCT IDENT struct_body_decl // : STRUCT IDENT struct_body_decl
Maybe<const ast::Struct*> ParserImpl::struct_decl() { Maybe<const ast::Struct*> ParserImpl::struct_decl() {
auto t = peek(); auto& t = peek();
auto source = t.source(); auto source = t.source();
if (!match(Token::Type::kStruct)) { if (!match(Token::Type::kStruct)) {
@ -1334,7 +1368,7 @@ Expect<ast::StructMemberList> ParserImpl::expect_struct_body_decl() {
bool errored = false; bool errored = false;
while (continue_parsing()) { while (continue_parsing()) {
// Check for the end of the list. // Check for the end of the list.
auto t = peek(); auto& t = peek();
if (!t.IsIdentifier() && !t.Is(Token::Type::kAttr)) { if (!t.IsIdentifier() && !t.Is(Token::Type::kAttr)) {
break; break;
} }
@ -1481,7 +1515,7 @@ Expect<ast::ParameterList> ParserImpl::expect_param_list() {
ast::ParameterList ret; ast::ParameterList ret;
while (continue_parsing()) { while (continue_parsing()) {
// Check for the end of the list. // Check for the end of the list.
auto t = peek(); auto& t = peek();
if (!t.IsIdentifier() && !t.Is(Token::Type::kAttr)) { if (!t.IsIdentifier() && !t.Is(Token::Type::kAttr)) {
break; break;
} }
@ -1522,7 +1556,7 @@ Expect<ast::Parameter*> ParserImpl::expect_param() {
// | FRAGMENT // | FRAGMENT
// | COMPUTE // | COMPUTE
Expect<ast::PipelineStage> ParserImpl::expect_pipeline_stage() { Expect<ast::PipelineStage> ParserImpl::expect_pipeline_stage() {
auto t = peek(); auto& t = peek();
if (t == kVertexStage) { if (t == kVertexStage) {
next(); // Consume the peek next(); // Consume the peek
return {ast::PipelineStage::kVertex, t.source()}; return {ast::PipelineStage::kVertex, t.source()};
@ -2010,7 +2044,7 @@ Maybe<const ast::CaseStatement*> ParserImpl::switch_body() {
return Failure::kNoMatch; return Failure::kNoMatch;
} }
auto t = next(); auto& t = next();
auto source = t.source(); auto source = t.source();
ast::CaseSelectorList selector_list; ast::CaseSelectorList selector_list;
@ -2263,8 +2297,8 @@ Maybe<const ast::WhileStatement*> ParserImpl::while_stmt() {
// func_call_stmt // func_call_stmt
// : IDENT argument_expression_list // : IDENT argument_expression_list
Maybe<const ast::CallStatement*> ParserImpl::func_call_stmt() { Maybe<const ast::CallStatement*> ParserImpl::func_call_stmt() {
auto t = peek(); auto& t = peek();
auto t2 = peek(1); auto& t2 = peek(1);
if (!t.IsIdentifier() || !t2.Is(Token::Type::kParenLeft)) { if (!t.IsIdentifier() || !t2.Is(Token::Type::kParenLeft)) {
return Failure::kNoMatch; return Failure::kNoMatch;
} }
@ -2325,7 +2359,7 @@ Maybe<const ast::BlockStatement*> ParserImpl::continuing_stmt() {
// | paren_expression // | paren_expression
// | BITCAST LESS_THAN type_decl GREATER_THAN paren_expression // | BITCAST LESS_THAN type_decl GREATER_THAN paren_expression
Maybe<const ast::Expression*> ParserImpl::primary_expression() { Maybe<const ast::Expression*> ParserImpl::primary_expression() {
auto t = peek(); auto& t = peek();
auto source = t.source(); auto source = t.source();
auto lit = const_literal(); auto lit = const_literal();
@ -2491,7 +2525,7 @@ Expect<ast::ExpressionList> ParserImpl::expect_argument_expression_list(std::str
// | STAR unary_expression // | STAR unary_expression
// | AND unary_expression // | AND unary_expression
Maybe<const ast::Expression*> ParserImpl::unary_expression() { Maybe<const ast::Expression*> ParserImpl::unary_expression() {
auto t = peek(); auto& t = peek();
if (match(Token::Type::kPlusPlus) || match(Token::Type::kMinusMinus)) { if (match(Token::Type::kPlusPlus) || match(Token::Type::kMinusMinus)) {
add_error(t.source(), add_error(t.source(),
@ -2556,7 +2590,7 @@ Expect<const ast::Expression*> ParserImpl::expect_multiplicative_expr(const ast:
return lhs; return lhs;
} }
auto t = next(); auto& t = next();
auto source = t.source(); auto source = t.source();
auto name = t.to_name(); auto name = t.to_name();
@ -2603,7 +2637,7 @@ Expect<const ast::Expression*> ParserImpl::expect_additive_expr(const ast::Expre
return lhs; return lhs;
} }
auto t = next(); auto& t = next();
auto source = t.source(); auto source = t.source();
auto rhs = multiplicative_expression(); auto rhs = multiplicative_expression();
@ -2651,7 +2685,7 @@ Expect<const ast::Expression*> ParserImpl::expect_shift_expr(const ast::Expressi
return lhs; return lhs;
} }
auto t = next(); auto& t = next();
auto source = t.source(); auto source = t.source();
auto rhs = additive_expression(); auto rhs = additive_expression();
if (rhs.errored) { if (rhs.errored) {
@ -2702,7 +2736,7 @@ Expect<const ast::Expression*> ParserImpl::expect_relational_expr(const ast::Exp
return lhs; return lhs;
} }
auto t = next(); auto& t = next();
auto source = t.source(); auto source = t.source();
auto name = t.to_name(); auto name = t.to_name();
@ -2749,7 +2783,7 @@ Expect<const ast::Expression*> ParserImpl::expect_equality_expr(const ast::Expre
return lhs; return lhs;
} }
auto t = next(); auto& t = next();
auto source = t.source(); auto source = t.source();
auto name = t.to_name(); auto name = t.to_name();
@ -2790,7 +2824,7 @@ Expect<const ast::Expression*> ParserImpl::expect_and_expr(const ast::Expression
return lhs; return lhs;
} }
auto t = next(); auto& t = next();
auto source = t.source(); auto source = t.source();
auto rhs = equality_expression(); auto rhs = equality_expression();
@ -2903,7 +2937,7 @@ Expect<const ast::Expression*> ParserImpl::expect_logical_and_expr(const ast::Ex
return lhs; return lhs;
} }
auto t = next(); auto& t = next();
auto source = t.source(); auto source = t.source();
auto rhs = inclusive_or_expression(); auto rhs = inclusive_or_expression();
@ -3013,7 +3047,7 @@ Maybe<ast::BinaryOp> ParserImpl::compound_assignment_operator() {
// decrement_stmt // decrement_stmt
// | lhs_expression MINUS_MINUS // | lhs_expression MINUS_MINUS
Maybe<const ast::Statement*> ParserImpl::assignment_stmt() { Maybe<const ast::Statement*> ParserImpl::assignment_stmt() {
auto t = peek(); auto& t = peek();
auto source = t.source(); auto source = t.source();
// tint:295 - Test for `ident COLON` - this is invalid grammar, and without // tint:295 - Test for `ident COLON` - this is invalid grammar, and without
@ -3076,7 +3110,7 @@ Maybe<const ast::Statement*> ParserImpl::assignment_stmt() {
// | TRUE // | TRUE
// | FALSE // | FALSE
Maybe<const ast::LiteralExpression*> ParserImpl::const_literal() { Maybe<const ast::LiteralExpression*> ParserImpl::const_literal() {
auto t = peek(); auto& t = peek();
if (match(Token::Type::kIntLiteral)) { if (match(Token::Type::kIntLiteral)) {
return create<ast::IntLiteralExpression>(t.source(), t.to_i64(), return create<ast::IntLiteralExpression>(t.source(), t.to_i64(),
ast::IntLiteralExpression::Suffix::kNone); ast::IntLiteralExpression::Suffix::kNone);
@ -3141,7 +3175,7 @@ Maybe<ast::AttributeList> ParserImpl::attribute_list() {
} }
Expect<const ast::Attribute*> ParserImpl::expect_attribute() { Expect<const ast::Attribute*> ParserImpl::expect_attribute() {
auto t = peek(); auto& t = peek();
auto attr = attribute(); auto attr = attribute();
if (attr.errored) { if (attr.errored) {
return Failure::kErrored; return Failure::kErrored;
@ -3154,7 +3188,7 @@ Expect<const ast::Attribute*> ParserImpl::expect_attribute() {
Maybe<const ast::Attribute*> ParserImpl::attribute() { Maybe<const ast::Attribute*> ParserImpl::attribute() {
using Result = Maybe<const ast::Attribute*>; using Result = Maybe<const ast::Attribute*>;
auto t = next(); auto& t = next();
if (!t.IsIdentifier()) { if (!t.IsIdentifier()) {
return Failure::kNoMatch; return Failure::kNoMatch;
@ -3204,7 +3238,7 @@ Maybe<const ast::Attribute*> ParserImpl::attribute() {
ast::InterpolationType type; ast::InterpolationType type;
ast::InterpolationSampling sampling = ast::InterpolationSampling::kNone; ast::InterpolationSampling sampling = ast::InterpolationSampling::kNone;
auto type_tok = next(); auto& type_tok = next();
if (type_tok == "perspective") { if (type_tok == "perspective") {
type = ast::InterpolationType::kPerspective; type = ast::InterpolationType::kPerspective;
} else if (type_tok == "linear") { } else if (type_tok == "linear") {
@ -3217,7 +3251,7 @@ Maybe<const ast::Attribute*> ParserImpl::attribute() {
if (match(Token::Type::kComma)) { if (match(Token::Type::kComma)) {
if (!peek_is(Token::Type::kParenRight)) { if (!peek_is(Token::Type::kParenRight)) {
auto sampling_tok = next(); auto& sampling_tok = next();
if (sampling_tok == "center") { if (sampling_tok == "center") {
sampling = ast::InterpolationSampling::kCenter; sampling = ast::InterpolationSampling::kCenter;
} else if (sampling_tok == "centroid") { } else if (sampling_tok == "centroid") {
@ -3227,7 +3261,6 @@ Maybe<const ast::Attribute*> ParserImpl::attribute() {
} else { } else {
return add_error(sampling_tok, "invalid interpolation sampling"); return add_error(sampling_tok, "invalid interpolation sampling");
} }
match(Token::Type::kComma); match(Token::Type::kComma);
} }
} }
@ -3384,7 +3417,7 @@ bool ParserImpl::expect_attributes_consumed(ast::AttributeList& in) {
} }
bool ParserImpl::match(Token::Type tok, Source* source /*= nullptr*/) { bool ParserImpl::match(Token::Type tok, Source* source /*= nullptr*/) {
auto t = peek(); auto& t = peek();
if (source != nullptr) { if (source != nullptr) {
*source = t.source(); *source = t.source();
@ -3398,7 +3431,7 @@ bool ParserImpl::match(Token::Type tok, Source* source /*= nullptr*/) {
} }
bool ParserImpl::expect(std::string_view use, Token::Type tok) { bool ParserImpl::expect(std::string_view use, Token::Type tok) {
auto t = peek(); auto& t = peek();
if (t.Is(tok)) { if (t.Is(tok)) {
next(); next();
synchronized_ = true; synchronized_ = true;
@ -3414,9 +3447,9 @@ bool ParserImpl::expect(std::string_view use, Token::Type tok) {
auto source = t.source(); auto source = t.source();
source.range.begin.column++; source.range.begin.column++;
if (t.Is(Token::Type::kShiftRight)) { if (t.Is(Token::Type::kShiftRight)) {
token_queue_.push_front(Token(Token::Type::kGreaterThan, source)); split_token(Token::Type::kGreaterThan, Token::Type::kGreaterThan);
} else if (t.Is(Token::Type::kGreaterThanEqual)) { } else if (t.Is(Token::Type::kGreaterThanEqual)) {
token_queue_.push_front(Token(Token::Type::kEqual, source)); split_token(Token::Type::kGreaterThan, Token::Type::kEqual);
} }
synchronized_ = true; synchronized_ = true;
@ -3439,7 +3472,7 @@ bool ParserImpl::expect(std::string_view use, Token::Type tok) {
} }
Expect<int32_t> ParserImpl::expect_sint(std::string_view use) { Expect<int32_t> ParserImpl::expect_sint(std::string_view use) {
auto t = peek(); auto& t = peek();
if (!t.Is(Token::Type::kIntLiteral) && !t.Is(Token::Type::kIntLiteral_I)) { if (!t.Is(Token::Type::kIntLiteral) && !t.Is(Token::Type::kIntLiteral_I)) {
return add_error(t.source(), "expected signed integer literal", use); return add_error(t.source(), "expected signed integer literal", use);
} }
@ -3482,7 +3515,7 @@ Expect<uint32_t> ParserImpl::expect_nonzero_positive_sint(std::string_view use)
} }
Expect<std::string> ParserImpl::expect_ident(std::string_view use) { Expect<std::string> ParserImpl::expect_ident(std::string_view use) {
auto t = peek(); auto& t = peek();
if (t.IsIdentifier()) { if (t.IsIdentifier()) {
synchronized_ = true; synchronized_ = true;
next(); next();
@ -3578,7 +3611,7 @@ bool ParserImpl::sync_to(Token::Type tok, bool consume) {
BlockCounters counters; BlockCounters counters;
for (size_t i = 0; i < kMaxResynchronizeLookahead; i++) { for (size_t i = 0; i < kMaxResynchronizeLookahead; i++) {
auto t = peek(i); auto& t = peek(i);
if (counters.consume(t) > 0) { if (counters.consume(t) > 0) {
continue; // Nested block continue; // Nested block
} }

32
src/tint/reader/wgsl/parser_impl.h
View File

@ -15,7 +15,6 @@
#ifndef SRC_TINT_READER_WGSL_PARSER_IMPL_H_ #ifndef SRC_TINT_READER_WGSL_PARSER_IMPL_H_
#define SRC_TINT_READER_WGSL_PARSER_IMPL_H_ #define SRC_TINT_READER_WGSL_PARSER_IMPL_H_
#include <deque>
#include <memory> #include <memory>
#include <string> #include <string>
#include <string_view> #include <string_view>
@ -300,6 +299,10 @@ class ParserImpl {
explicit ParserImpl(Source::File const* file); explicit ParserImpl(Source::File const* file);
~ParserImpl(); ~ParserImpl();
/// Reads tokens from the source file. This will be called automatically
/// by |parse|.
void InitializeLex();
/// Run the parser /// Run the parser
/// @returns true if the parse was successful, false otherwise. /// @returns true if the parse was successful, false otherwise.
bool Parse(); bool Parse();
@ -330,19 +333,19 @@ class ParserImpl {
ProgramBuilder& builder() { return builder_; } ProgramBuilder& builder() { return builder_; }
/// @returns the next token /// @returns the next token
Token next(); const Token& next();
/// Peeks ahead and returns the token at `idx` ahead of the current position /// Peeks ahead and returns the token at `idx` ahead of the current position
/// @param idx the index of the token to return /// @param idx the index of the token to return
/// @returns the token `idx` positions ahead without advancing /// @returns the token `idx` positions ahead without advancing
Token peek(size_t idx = 0); const Token& peek(size_t idx = 0);
/// Peeks ahead and returns true if the token at `idx` ahead of the current /// Peeks ahead and returns true if the token at `idx` ahead of the current
/// position is |tok| /// position is |tok|
/// @param idx the index of the token to return /// @param idx the index of the token to return
/// @param tok the token to look for /// @param tok the token to look for
/// @returns true if the token `idx` positions ahead is |tok| /// @returns true if the token `idx` positions ahead is |tok|
bool peek_is(Token::Type tok, size_t idx = 0); bool peek_is(Token::Type tok, size_t idx = 0);
/// @returns the last token that was returned by `next()` /// @returns the last source location that was returned by `next()`
Token last_token() const; Source last_source() const;
/// Appends an error at `t` with the message `msg` /// Appends an error at `t` with the message `msg`
/// @param t the token to associate the error with /// @param t the token to associate the error with
/// @param msg the error message /// @param msg the error message
@ -812,11 +815,11 @@ class ParserImpl {
/// Used to ensure that all attributes are consumed. /// Used to ensure that all attributes are consumed.
bool expect_attributes_consumed(ast::AttributeList& list); bool expect_attributes_consumed(ast::AttributeList& list);
Expect<const ast::Type*> expect_type_decl_pointer(Token t); Expect<const ast::Type*> expect_type_decl_pointer(const Token& t);
Expect<const ast::Type*> expect_type_decl_atomic(Token t); Expect<const ast::Type*> expect_type_decl_atomic(const Token& t);
Expect<const ast::Type*> expect_type_decl_vector(Token t); Expect<const ast::Type*> expect_type_decl_vector(const Token& t);
Expect<const ast::Type*> expect_type_decl_array(Token t); Expect<const ast::Type*> expect_type_decl_array(const Token& t);
Expect<const ast::Type*> expect_type_decl_matrix(Token t); Expect<const ast::Type*> expect_type_decl_matrix(const Token& t);
Expect<const ast::Type*> expect_type(std::string_view use); Expect<const ast::Type*> expect_type(std::string_view use);
@ -824,6 +827,8 @@ class ParserImpl {
Maybe<const ast::Statement*> for_header_initializer(); Maybe<const ast::Statement*> for_header_initializer();
Maybe<const ast::Statement*> for_header_continuing(); Maybe<const ast::Statement*> for_header_continuing();
void split_token(Token::Type lhs, Token::Type rhs);
class MultiTokenSource; class MultiTokenSource;
MultiTokenSource make_source_range(); MultiTokenSource make_source_range();
MultiTokenSource make_source_range_from(const Source& start); MultiTokenSource make_source_range_from(const Source& start);
@ -837,9 +842,10 @@ class ParserImpl {
return builder_.create<T>(std::forward<ARGS>(args)...); return builder_.create<T>(std::forward<ARGS>(args)...);
} }
std::unique_ptr<Lexer> lexer_; Source::File const* const file_;
std::deque<Token> token_queue_; std::vector<Token> tokens_;
Token last_token_; size_t next_token_idx_ = 0;
Source last_source_;
bool synchronized_ = true; bool synchronized_ = true;
uint32_t parse_depth_ = 0; uint32_t parse_depth_ = 0;
std::vector<Token::Type> sync_tokens_; std::vector<Token::Type> sync_tokens_;

2
src/tint/reader/wgsl/parser_impl_pipeline_stage_test.cc
View File

@ -40,7 +40,7 @@ TEST_P(PipelineStageTest, Parses) {
EXPECT_EQ(stage.source.range.end.line, 1u); EXPECT_EQ(stage.source.range.end.line, 1u);
EXPECT_EQ(stage.source.range.end.column, 1u + params.input.size()); EXPECT_EQ(stage.source.range.end.column, 1u + params.input.size());
auto t = p->next(); auto& t = p->next();
EXPECT_TRUE(t.IsEof()); EXPECT_TRUE(t.IsEof());
} }
INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P(

2
src/tint/reader/wgsl/parser_impl_storage_class_test.cc
View File

@ -37,7 +37,7 @@ TEST_P(StorageClassTest, Parses) {
EXPECT_FALSE(p->has_error()); EXPECT_FALSE(p->has_error());
EXPECT_EQ(sc.value, params.result); EXPECT_EQ(sc.value, params.result);
auto t = p->next(); auto& t = p->next();
EXPECT_TRUE(t.IsEof()); EXPECT_TRUE(t.IsEof());
} }
INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P(

2
src/tint/reader/wgsl/parser_impl_test_helper.h
View File

@ -38,6 +38,7 @@ class ParserImplTest : public testing::Test, public ProgramBuilder {
std::unique_ptr<ParserImpl> parser(const std::string& str) { std::unique_ptr<ParserImpl> parser(const std::string& str) {
auto file = std::make_unique<Source::File>("test.wgsl", str); auto file = std::make_unique<Source::File>("test.wgsl", str);
auto impl = std::make_unique<ParserImpl>(file.get()); auto impl = std::make_unique<ParserImpl>(file.get());
impl->InitializeLex();
files_.emplace_back(std::move(file)); files_.emplace_back(std::move(file));
return impl; return impl;
} }
@ -60,6 +61,7 @@ class ParserImplTestWithParam : public testing::TestWithParam<T>, public Program
std::unique_ptr<ParserImpl> parser(const std::string& str) { std::unique_ptr<ParserImpl> parser(const std::string& str) {
auto file = std::make_unique<Source::File>("test.wgsl", str); auto file = std::make_unique<Source::File>("test.wgsl", str);
auto impl = std::make_unique<ParserImpl>(file.get()); auto impl = std::make_unique<ParserImpl>(file.get());
impl->InitializeLex();
files_.emplace_back(std::move(file)); files_.emplace_back(std::move(file));
return impl; return impl;
} }

2
src/tint/reader/wgsl/parser_impl_variable_decl_test.cc
View File

@ -68,7 +68,7 @@ TEST_F(ParserImplTest, VariableDecl_MissingVar) {
EXPECT_FALSE(v.errored); EXPECT_FALSE(v.errored);
EXPECT_FALSE(p->has_error()); EXPECT_FALSE(p->has_error());
auto t = p->next(); auto& t = p->next();
ASSERT_TRUE(t.IsIdentifier()); ASSERT_TRUE(t.IsIdentifier());
} }

6
src/tint/reader/wgsl/parser_impl_variable_qualifier_test.cc
View File

@ -40,7 +40,7 @@ TEST_P(VariableQualifierTest, ParsesStorageClass) {
EXPECT_EQ(sc->storage_class, params.storage_class); EXPECT_EQ(sc->storage_class, params.storage_class);
EXPECT_EQ(sc->access, params.access); EXPECT_EQ(sc->access, params.access);
auto t = p->next(); auto& t = p->next();
EXPECT_TRUE(t.IsEof()); EXPECT_TRUE(t.IsEof());
} }
INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P(
@ -83,7 +83,7 @@ TEST_F(ParserImplTest, VariableQualifier_MissingLessThan) {
EXPECT_FALSE(sc.errored); EXPECT_FALSE(sc.errored);
EXPECT_FALSE(sc.matched); EXPECT_FALSE(sc.matched);
auto t = p->next(); auto& t = p->next();
ASSERT_TRUE(t.Is(Token::Type::kIdentifier)); ASSERT_TRUE(t.Is(Token::Type::kIdentifier));
} }
@ -94,7 +94,7 @@ TEST_F(ParserImplTest, VariableQualifier_MissingLessThan_AfterSC) {
EXPECT_FALSE(sc.errored); EXPECT_FALSE(sc.errored);
EXPECT_FALSE(sc.matched); EXPECT_FALSE(sc.matched);
auto t = p->next(); auto& t = p->next();
ASSERT_TRUE(t.Is(Token::Type::kIdentifier)); ASSERT_TRUE(t.Is(Token::Type::kIdentifier));
} }

8
src/tint/reader/wgsl/token.cc
View File

@ -37,6 +37,8 @@ std::string_view Token::TypeToName(Type type) {
return "'i'-suffixed integer literal"; return "'i'-suffixed integer literal";
case Token::Type::kIntLiteral_U: case Token::Type::kIntLiteral_U:
return "'u'-suffixed integer literal"; return "'u'-suffixed integer literal";
case Token::Type::kPlaceholder:
return "placeholder";
case Token::Type::kUninitialized: case Token::Type::kUninitialized:
return "uninitialized"; return "uninitialized";
@ -285,13 +287,9 @@ Token::Token(Type type, const Source& source) : type_(type), source_(source) {}
Token::Token(Token&&) = default; Token::Token(Token&&) = default;
Token::Token(const Token&) = default;
Token::~Token() = default; Token::~Token() = default;
Token& Token::operator=(const Token& rhs) = default; bool Token::operator==(std::string_view ident) const {
bool Token::operator==(std::string_view ident) {
if (type_ != Type::kIdentifier) { if (type_ != Type::kIdentifier) {
return false; return false;
} }

22
src/tint/reader/wgsl/token.h
View File

@ -32,6 +32,8 @@ class Token {
kError = -2, kError = -2,
/// Uninitialized token /// Uninitialized token
kUninitialized = 0, kUninitialized = 0,
/// Placeholder token which maybe fillled in later
kPlaceholder = 1,
/// End of input string reached /// End of input string reached
kEOF, kEOF,
@ -310,19 +312,16 @@ class Token {
Token(Type type, const Source& source, double val); Token(Type type, const Source& source, double val);
/// Move constructor /// Move constructor
Token(Token&&); Token(Token&&);
/// Copy constructor
Token(const Token&);
~Token(); ~Token();
/// Assignment operator
/// @param b the token to copy
/// @return Token
Token& operator=(const Token& b);
/// Equality operator with an identifier /// Equality operator with an identifier
/// @param ident the identifier string /// @param ident the identifier string
/// @return true if this token is an identifier and is equal to ident. /// @return true if this token is an identifier and is equal to ident.
bool operator==(std::string_view ident); bool operator==(std::string_view ident) const;
/// Sets the token to the given type
/// @param type the type to set
void SetType(Token::Type type) { type_ = type; }
/// Returns true if the token is of the given type /// Returns true if the token is of the given type
/// @param t the type to check against. /// @param t the type to check against.
@ -331,6 +330,8 @@ class Token {
/// @returns true if the token is uninitialized /// @returns true if the token is uninitialized
bool IsUninitialized() const { return type_ == Type::kUninitialized; } bool IsUninitialized() const { return type_ == Type::kUninitialized; }
/// @returns true if the token is a placeholder
bool IsPlaceholder() const { return type_ == Type::kPlaceholder; }
/// @returns true if the token is EOF /// @returns true if the token is EOF
bool IsEof() const { return type_ == Type::kEOF; } bool IsEof() const { return type_ == Type::kEOF; }
/// @returns true if the token is Error /// @returns true if the token is Error
@ -372,6 +373,11 @@ class Token {
return type_ == Type::kVec2 || type_ == Type::kVec3 || type_ == Type::kVec4; return type_ == Type::kVec2 || type_ == Type::kVec3 || type_ == Type::kVec4;
} }
/// @returns true if the token can be split during parse into component tokens
bool IsSplittable() const {
return Is(Token::Type::kShiftRight) || Is(Token::Type::kGreaterThanEqual);
}
/// @returns the source information for this token /// @returns the source information for this token
Source source() const { return source_; } Source source() const { return source_; }