From 0cbf5a922fe7b0b759db48ab07a329645b53e6bd Mon Sep 17 00:00:00 2001 From: dan sinclair Date: Mon, 25 Jul 2022 16:43:08 +0000 Subject: [PATCH] Pre-parse token list. This CL updates the parser and lexer to pre-parse the token list and then the parser works off the list of tokens. This allows the parser to work with references to the tokens and not have to worry about them going stale. For any splittable token a placeholder token is injected after the token. If the token ends up getting split the placeholder is overwritten with the new type. Change-Id: I5a8ccca15d8c14b5027df7dd2734be6753e46fa9 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/97070 Reviewed-by: Ben Clayton Commit-Queue: Dan Sinclair Kokoro: Kokoro --- src/tint/reader/wgsl/lexer.cc | 23 + src/tint/reader/wgsl/lexer.h | 6 +- src/tint/reader/wgsl/lexer_test.cc | 474 ++++++++++++------ src/tint/reader/wgsl/parser_impl.cc | 151 +++--- src/tint/reader/wgsl/parser_impl.h | 32 +- .../wgsl/parser_impl_pipeline_stage_test.cc | 2 +- .../wgsl/parser_impl_storage_class_test.cc | 2 +- .../reader/wgsl/parser_impl_test_helper.h | 2 + .../wgsl/parser_impl_variable_decl_test.cc | 2 +- .../parser_impl_variable_qualifier_test.cc | 6 +- src/tint/reader/wgsl/token.cc | 8 +- src/tint/reader/wgsl/token.h | 22 +- 12 files changed, 493 insertions(+), 237 deletions(-) diff --git a/src/tint/reader/wgsl/lexer.cc b/src/tint/reader/wgsl/lexer.cc index 48c3e5d80b..c9056e0ab9 100644 --- a/src/tint/reader/wgsl/lexer.cc +++ b/src/tint/reader/wgsl/lexer.cc @@ -37,6 +37,8 @@ static_assert(sizeof(decltype(tint::Source::FileContent::data[0])) == sizeof(uin "tint::reader::wgsl requires the size of a std::string element " "to be a single byte"); +static constexpr size_t kDefaultListSize = 512; + bool read_blankspace(std::string_view str, size_t i, bool* is_blankspace, size_t* blankspace_size) { // See https://www.w3.org/TR/WGSL/#blankspace @@ -88,6 +90,27 @@ Lexer::Lexer(const Source::File* file) : file_(file), location_{1, 1} {} Lexer::~Lexer() = default; +std::vector Lexer::Lex() { + std::vector tokens; + tokens.reserve(kDefaultListSize); + while (true) { + tokens.emplace_back(next()); + + // If the token can be split, we insert a placeholder element into + // the stream to hold the split character. + if (tokens.back().IsSplittable()) { + auto src = tokens.back().source(); + src.range.begin.column++; + tokens.emplace_back(Token(Token::Type::kPlaceholder, src)); + } + + if (tokens.back().IsEof() || tokens.back().IsError()) { + break; + } + } + return tokens; +} + const std::string_view Lexer::line() const { if (file_->content.lines.size() == 0) { static const char* empty_string = ""; diff --git a/src/tint/reader/wgsl/lexer.h b/src/tint/reader/wgsl/lexer.h index d51c26c272..8e0306b6a8 100644 --- a/src/tint/reader/wgsl/lexer.h +++ b/src/tint/reader/wgsl/lexer.h @@ -16,6 +16,7 @@ #define SRC_TINT_READER_WGSL_LEXER_H_ #include +#include #include "src/tint/reader/wgsl/token.h" @@ -29,11 +30,14 @@ class Lexer { explicit Lexer(const Source::File* file); ~Lexer(); + /// @return the token list. + std::vector Lex(); + + private: /// Returns the next token in the input stream. /// @return Token Token next(); - private: /// Advances past blankspace and comments, if present at the current position. /// @returns error token, EOF, or uninitialized Token skip_blankspace_and_comments(); diff --git a/src/tint/reader/wgsl/lexer_test.cc b/src/tint/reader/wgsl/lexer_test.cc index 799f9c20b5..f82045ab80 100644 --- a/src/tint/reader/wgsl/lexer_test.cc +++ b/src/tint/reader/wgsl/lexer_test.cc @@ -46,24 +46,33 @@ using LexerTest = testing::Test; TEST_F(LexerTest, Empty) { Source::File file("", ""); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsEof()); + + auto list = l.Lex(); + ASSERT_EQ(1u, list.size()); + EXPECT_TRUE(list[0].IsEof()); } TEST_F(LexerTest, Skips_Blankspace_Basic) { Source::File file("", "\t\r\n\t ident\t\n\t \r "); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 2u); - EXPECT_EQ(t.source().range.begin.column, 6u); - EXPECT_EQ(t.source().range.end.line, 2u); - EXPECT_EQ(t.source().range.end.column, 11u); - EXPECT_EQ(t.to_str(), "ident"); + auto list = l.Lex(); + ASSERT_EQ(2u, list.size()); - t = l.next(); - EXPECT_TRUE(t.IsEof()); + { + auto& t = list[0]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 2u); + EXPECT_EQ(t.source().range.begin.column, 6u); + EXPECT_EQ(t.source().range.end.line, 2u); + EXPECT_EQ(t.source().range.end.column, 11u); + EXPECT_EQ(t.to_str(), "ident"); + } + + { + auto& t = list[1]; + EXPECT_TRUE(t.IsEof()); + } } TEST_F(LexerTest, Skips_Blankspace_Exotic) { @@ -73,16 +82,23 @@ TEST_F(LexerTest, Skips_Blankspace_Exotic) { kVTab kFF kNL kLS kPS kL2R kR2L); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 6u); - EXPECT_EQ(t.source().range.begin.column, 7u); - EXPECT_EQ(t.source().range.end.line, 6u); - EXPECT_EQ(t.source().range.end.column, 12u); - EXPECT_EQ(t.to_str(), "ident"); + auto list = l.Lex(); + ASSERT_EQ(2u, list.size()); - t = l.next(); - EXPECT_TRUE(t.IsEof()); + { + auto& t = list[0]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 6u); + EXPECT_EQ(t.source().range.begin.column, 7u); + EXPECT_EQ(t.source().range.end.line, 6u); + EXPECT_EQ(t.source().range.end.column, 12u); + EXPECT_EQ(t.to_str(), "ident"); + } + + { + auto& t = list[1]; + EXPECT_TRUE(t.IsEof()); + } } TEST_F(LexerTest, Skips_Comments_Line) { @@ -92,24 +108,33 @@ ident1 //ends with comment ident2)"); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 2u); - EXPECT_EQ(t.source().range.begin.column, 1u); - EXPECT_EQ(t.source().range.end.line, 2u); - EXPECT_EQ(t.source().range.end.column, 7u); - EXPECT_EQ(t.to_str(), "ident1"); + auto list = l.Lex(); + ASSERT_EQ(3u, list.size()); - t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 4u); - EXPECT_EQ(t.source().range.begin.column, 2u); - EXPECT_EQ(t.source().range.end.line, 4u); - EXPECT_EQ(t.source().range.end.column, 8u); - EXPECT_EQ(t.to_str(), "ident2"); + { + auto& t = list[0]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 2u); + EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 2u); + EXPECT_EQ(t.source().range.end.column, 7u); + EXPECT_EQ(t.to_str(), "ident1"); + } - t = l.next(); - EXPECT_TRUE(t.IsEof()); + { + auto& t = list[1]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 4u); + EXPECT_EQ(t.source().range.begin.column, 2u); + EXPECT_EQ(t.source().range.end.line, 4u); + EXPECT_EQ(t.source().range.end.column, 8u); + EXPECT_EQ(t.to_str(), "ident2"); + } + + { + auto& t = list[2]; + EXPECT_TRUE(t.IsEof()); + } } TEST_F(LexerTest, Skips_Comments_Unicode) { @@ -119,24 +144,33 @@ ident1 //ends with 🙂🙂🙂 ident2)"); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 2u); - EXPECT_EQ(t.source().range.begin.column, 1u); - EXPECT_EQ(t.source().range.end.line, 2u); - EXPECT_EQ(t.source().range.end.column, 7u); - EXPECT_EQ(t.to_str(), "ident1"); + auto list = l.Lex(); + ASSERT_EQ(3u, list.size()); - t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 4u); - EXPECT_EQ(t.source().range.begin.column, 2u); - EXPECT_EQ(t.source().range.end.line, 4u); - EXPECT_EQ(t.source().range.end.column, 8u); - EXPECT_EQ(t.to_str(), "ident2"); + { + auto& t = list[0]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 2u); + EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 2u); + EXPECT_EQ(t.source().range.end.column, 7u); + EXPECT_EQ(t.to_str(), "ident1"); + } - t = l.next(); - EXPECT_TRUE(t.IsEof()); + { + auto& t = list[1]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 4u); + EXPECT_EQ(t.source().range.begin.column, 2u); + EXPECT_EQ(t.source().range.end.line, 4u); + EXPECT_EQ(t.source().range.end.column, 8u); + EXPECT_EQ(t.to_str(), "ident2"); + } + + { + auto& t = list[2]; + EXPECT_TRUE(t.IsEof()); + } } using LineCommentTerminatorTest = testing::TestWithParam; @@ -150,21 +184,29 @@ TEST_P(LineCommentTerminatorTest, Terminators) { Source::File file("", src); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.Is(Token::Type::kConst)); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 1u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 6u); - auto is_same_line = [](std::string_view v) { return v == kSpace || v == kHTab || v == kL2R || v == kR2L; }; + auto list = l.Lex(); + ASSERT_EQ(is_same_line(c) ? 2u : 3u, list.size()); + + size_t idx = 0; + + { + auto& t = list[idx++]; + EXPECT_TRUE(t.Is(Token::Type::kConst)); + EXPECT_EQ(t.source().range.begin.line, 1u); + EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 6u); + } + if (!is_same_line(c)) { size_t line = is_same_line(c) ? 1u : 2u; size_t col = is_same_line(c) ? 25u : 1u; - t = l.next(); + + auto& t = list[idx++]; EXPECT_TRUE(t.IsIdentifier()); EXPECT_EQ(t.source().range.begin.line, line); EXPECT_EQ(t.source().range.begin.column, col); @@ -173,8 +215,10 @@ TEST_P(LineCommentTerminatorTest, Terminators) { EXPECT_EQ(t.to_str(), "ident"); } - t = l.next(); - EXPECT_TRUE(t.IsEof()); + { + auto& t = list[idx]; + EXPECT_TRUE(t.IsEof()); + } } INSTANTIATE_TEST_SUITE_P(LexerTest, LineCommentTerminatorTest, @@ -198,16 +242,23 @@ TEST_F(LexerTest, Skips_Comments_Block) { text */ident)"); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 2u); - EXPECT_EQ(t.source().range.begin.column, 8u); - EXPECT_EQ(t.source().range.end.line, 2u); - EXPECT_EQ(t.source().range.end.column, 13u); - EXPECT_EQ(t.to_str(), "ident"); + auto list = l.Lex(); + ASSERT_EQ(2u, list.size()); - t = l.next(); - EXPECT_TRUE(t.IsEof()); + { + auto& t = list[0]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 2u); + EXPECT_EQ(t.source().range.begin.column, 8u); + EXPECT_EQ(t.source().range.end.line, 2u); + EXPECT_EQ(t.source().range.end.column, 13u); + EXPECT_EQ(t.to_str(), "ident"); + } + + { + auto& t = list[1]; + EXPECT_TRUE(t.IsEof()); + } } TEST_F(LexerTest, Skips_Comments_Block_Nested) { @@ -216,16 +267,23 @@ text // nested line comments are ignored /* more text /////**/ */*/ident)"); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 3u); - EXPECT_EQ(t.source().range.begin.column, 14u); - EXPECT_EQ(t.source().range.end.line, 3u); - EXPECT_EQ(t.source().range.end.column, 19u); - EXPECT_EQ(t.to_str(), "ident"); + auto list = l.Lex(); + ASSERT_EQ(2u, list.size()); - t = l.next(); - EXPECT_TRUE(t.IsEof()); + { + auto& t = list[0]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 3u); + EXPECT_EQ(t.source().range.begin.column, 14u); + EXPECT_EQ(t.source().range.end.line, 3u); + EXPECT_EQ(t.source().range.end.column, 19u); + EXPECT_EQ(t.to_str(), "ident"); + } + + { + auto& t = list[1]; + EXPECT_TRUE(t.IsEof()); + } } TEST_F(LexerTest, Skips_Comments_Block_Unterminated) { @@ -237,7 +295,10 @@ TEST_F(LexerTest, Skips_Comments_Block_Unterminated) { abcd)"); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + ASSERT_EQ(1u, list.size()); + + auto& t = list[0]; ASSERT_TRUE(t.Is(Token::Type::kError)); EXPECT_EQ(t.to_str(), "unterminated block comment"); EXPECT_EQ(t.source().range.begin.line, 2u); @@ -250,7 +311,10 @@ TEST_F(LexerTest, Null_InBlankspace_IsError) { Source::File file("", std::string{' ', 0, ' '}); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + ASSERT_EQ(1u, list.size()); + + auto& t = list[0]; EXPECT_TRUE(t.IsError()); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 2u); @@ -263,7 +327,10 @@ TEST_F(LexerTest, Null_InLineComment_IsError) { Source::File file("", std::string{'/', '/', ' ', 0, ' '}); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + ASSERT_EQ(1u, list.size()); + + auto& t = list[0]; EXPECT_TRUE(t.IsError()); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 4u); @@ -276,7 +343,10 @@ TEST_F(LexerTest, Null_InBlockComment_IsError) { Source::File file("", std::string{'/', '*', ' ', 0, '*', '/'}); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + ASSERT_EQ(1u, list.size()); + + auto& t = list[0]; EXPECT_TRUE(t.IsError()); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 4u); @@ -292,16 +362,24 @@ TEST_F(LexerTest, Null_InIdentifier_IsError) { Source::File file("", std::string{'a', 0, 'c'}); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.to_str(), "a"); - t = l.next(); - EXPECT_TRUE(t.IsError()); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 2u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 2u); - EXPECT_EQ(t.to_str(), "null character found"); + auto list = l.Lex(); + ASSERT_EQ(2u, list.size()); + + { + auto& t = list[0]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.to_str(), "a"); + } + + { + auto& t = list[1]; + EXPECT_TRUE(t.IsError()); + EXPECT_EQ(t.source().range.begin.line, 1u); + EXPECT_EQ(t.source().range.begin.column, 2u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 2u); + EXPECT_EQ(t.to_str(), "null character found"); + } } struct FloatData { @@ -318,22 +396,29 @@ TEST_P(FloatTest, Parse) { Source::File file("", params.input); Lexer l(&file); - auto t = l.next(); - if (std::string(params.input).back() == 'f') { - EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral_F)); - } else if (std::string(params.input).back() == 'h') { - EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral_H)); - } else { - EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral)); - } - EXPECT_EQ(t.to_f64(), params.result); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 1u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); + auto list = l.Lex(); + ASSERT_EQ(2u, list.size()); - t = l.next(); - EXPECT_TRUE(t.IsEof()); + { + auto& t = list[0]; + if (std::string(params.input).back() == 'f') { + EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral_F)); + } else if (std::string(params.input).back() == 'h') { + EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral_H)); + } else { + EXPECT_TRUE(t.Is(Token::Type::kFloatLiteral)); + } + EXPECT_EQ(t.to_f64(), params.result); + EXPECT_EQ(t.source().range.begin.line, 1u); + EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); + } + + { + auto& t = list[1]; + EXPECT_TRUE(t.IsEof()); + } } INSTANTIATE_TEST_SUITE_P(LexerTest, FloatTest, @@ -437,7 +522,10 @@ TEST_P(FloatTest_Invalid, Handles) { Source::File file("", GetParam()); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + ASSERT_FALSE(list.empty()); + + auto& t = list[0]; EXPECT_FALSE(t.Is(Token::Type::kFloatLiteral) || t.Is(Token::Type::kFloatLiteral_F) || t.Is(Token::Type::kFloatLiteral_H)); } @@ -478,13 +566,23 @@ TEST_P(AsciiIdentifierTest, Parse) { Source::File file("", GetParam()); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 1u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 1u + strlen(GetParam())); - EXPECT_EQ(t.to_str(), GetParam()); + auto list = l.Lex(); + ASSERT_EQ(2u, list.size()); + + { + auto& t = list[0]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 1u); + EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(GetParam())); + EXPECT_EQ(t.to_str(), GetParam()); + } + + { + auto& t = list[1]; + EXPECT_TRUE(t.IsEof()); + } } INSTANTIATE_TEST_SUITE_P(LexerTest, AsciiIdentifierTest, @@ -510,13 +608,23 @@ TEST_P(ValidUnicodeIdentifierTest, Parse) { Source::File file("", GetParam().utf8); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 1u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 1u + GetParam().count); - EXPECT_EQ(t.to_str(), GetParam().utf8); + auto list = l.Lex(); + ASSERT_EQ(2u, list.size()); + + { + auto& t = list[0]; + EXPECT_TRUE(t.IsIdentifier()); + EXPECT_EQ(t.source().range.begin.line, 1u); + EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + GetParam().count); + EXPECT_EQ(t.to_str(), GetParam().utf8); + } + + { + auto& t = list[1]; + EXPECT_TRUE(t.IsEof()); + } } INSTANTIATE_TEST_SUITE_P( LexerTest, @@ -554,7 +662,10 @@ TEST_P(InvalidUnicodeIdentifierTest, Parse) { Source::File file("", GetParam()); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + ASSERT_FALSE(list.empty()); + + auto& t = list[0]; EXPECT_TRUE(t.IsError()); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); @@ -602,7 +713,10 @@ TEST_F(LexerTest, IdentifierTest_SingleUnderscoreDoesNotMatch) { Source::File file("", "_"); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + ASSERT_FALSE(list.empty()); + + auto& t = list[0]; EXPECT_FALSE(t.IsIdentifier()); } @@ -610,7 +724,10 @@ TEST_F(LexerTest, IdentifierTest_DoesNotStartWithDoubleUnderscore) { Source::File file("", "__test"); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + ASSERT_FALSE(list.empty()); + + auto& t = list[0]; EXPECT_FALSE(t.IsIdentifier()); } @@ -618,7 +735,10 @@ TEST_F(LexerTest, IdentifierTest_DoesNotStartWithNumber) { Source::File file("", "01test"); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + EXPECT_FALSE(list.empty()); + + auto& t = list[0]; EXPECT_FALSE(t.IsIdentifier()); } @@ -641,7 +761,12 @@ TEST_P(ParseIntegerTest, Parse) { auto params = std::get<1>(GetParam()); Source::File file("", params.input); - auto t = Lexer(&file).next(); + Lexer l(&file); + + auto list = l.Lex(); + ASSERT_FALSE(list.empty()); + + auto& t = list[0]; switch (suffix) { case 'i': EXPECT_TRUE(t.Is(Token::Type::kIntLiteral_I)); @@ -770,7 +895,12 @@ TEST_P(ParseIntegerTest_CannotBeRepresented, Parse) { auto type = std::get<0>(GetParam()); auto source = std::get<1>(GetParam()); Source::File file("", source); - auto t = Lexer(&file).next(); + + Lexer l(&file); + auto list = l.Lex(); + ASSERT_FALSE(list.empty()); + + auto& t = list[0]; EXPECT_TRUE(t.Is(Token::Type::kError)); auto expect = "value cannot be represented as '" + std::string(type) + "'"; EXPECT_EQ(t.to_str(), expect); @@ -800,7 +930,12 @@ INSTANTIATE_TEST_SUITE_P(u32, using ParseIntegerTest_LeadingZeros = testing::TestWithParam; TEST_P(ParseIntegerTest_LeadingZeros, Parse) { Source::File file("", GetParam()); - auto t = Lexer(&file).next(); + + Lexer l(&file); + auto list = l.Lex(); + ASSERT_FALSE(list.empty()); + + auto& t = list[0]; EXPECT_TRUE(t.Is(Token::Type::kError)); EXPECT_EQ(t.to_str(), "integer literal cannot have leading 0s"); } @@ -815,7 +950,12 @@ INSTANTIATE_TEST_SUITE_P(LeadingZero, using ParseIntegerTest_NoSignificantDigits = testing::TestWithParam; TEST_P(ParseIntegerTest_NoSignificantDigits, Parse) { Source::File file("", GetParam()); - auto t = Lexer(&file).next(); + + Lexer l(&file); + auto list = l.Lex(); + ASSERT_FALSE(list.empty()); + + auto& t = list[0]; EXPECT_TRUE(t.Is(Token::Type::kError)); EXPECT_EQ(t.to_str(), "integer or float hex literal has no significant digits"); } @@ -849,15 +989,22 @@ TEST_P(PunctuationTest, Parses) { Source::File file("", params.input); Lexer l(&file); - auto t = l.next(); - EXPECT_TRUE(t.Is(params.type)); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 1u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); + auto list = l.Lex(); + ASSERT_GE(list.size(), 2u); - t = l.next(); - EXPECT_EQ(t.source().range.begin.column, 1 + std::string(params.input).size()); + { + auto& t = list[0]; + EXPECT_TRUE(t.Is(params.type)); + EXPECT_EQ(t.source().range.begin.line, 1u); + EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); + } + + { + auto& t = list[1]; + EXPECT_EQ(t.source().range.begin.column, 1 + std::string(params.input).size()); + } } INSTANTIATE_TEST_SUITE_P(LexerTest, PunctuationTest, @@ -876,8 +1023,6 @@ INSTANTIATE_TEST_SUITE_P(LexerTest, TokenData{"=", Token::Type::kEqual}, TokenData{"==", Token::Type::kEqualEqual}, TokenData{">", Token::Type::kGreaterThan}, - TokenData{">=", Token::Type::kGreaterThanEqual}, - TokenData{">>", Token::Type::kShiftRight}, TokenData{"<", Token::Type::kLessThan}, TokenData{"<=", Token::Type::kLessThanEqual}, TokenData{"<<", Token::Type::kShiftLeft}, @@ -906,21 +1051,60 @@ INSTANTIATE_TEST_SUITE_P(LexerTest, TokenData{"|=", Token::Type::kOrEqual}, TokenData{"^=", Token::Type::kXorEqual})); +using SplittablePunctuationTest = testing::TestWithParam; +TEST_P(SplittablePunctuationTest, Parses) { + auto params = GetParam(); + Source::File file("", params.input); + Lexer l(&file); + + auto list = l.Lex(); + ASSERT_GE(list.size(), 3u); + + { + auto& t = list[0]; + EXPECT_TRUE(t.Is(params.type)); + EXPECT_EQ(t.source().range.begin.line, 1u); + EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); + } + + { + auto& t = list[1]; + EXPECT_TRUE(t.Is(Token::Type::kPlaceholder)); + EXPECT_EQ(t.source().range.begin.line, 1u); + EXPECT_EQ(t.source().range.begin.column, 2u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); + } + + { + auto& t = list[2]; + EXPECT_EQ(t.source().range.begin.column, 1 + std::string(params.input).size()); + } +} +INSTANTIATE_TEST_SUITE_P(LexerTest, + SplittablePunctuationTest, + testing::Values(TokenData{">=", Token::Type::kGreaterThanEqual}, + TokenData{">>", Token::Type::kShiftRight})); + using KeywordTest = testing::TestWithParam; TEST_P(KeywordTest, Parses) { auto params = GetParam(); Source::File file("", params.input); Lexer l(&file); - auto t = l.next(); + auto list = l.Lex(); + ASSERT_GE(list.size(), 2u); + + auto& t = list[0]; EXPECT_TRUE(t.Is(params.type)) << params.input; EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); EXPECT_EQ(t.source().range.end.line, 1u); EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); - t = l.next(); - EXPECT_EQ(t.source().range.begin.column, 1 + std::string(params.input).size()); + EXPECT_EQ(list[1].source().range.begin.column, 1 + std::string(params.input).size()); } INSTANTIATE_TEST_SUITE_P( LexerTest, diff --git a/src/tint/reader/wgsl/parser_impl.cc b/src/tint/reader/wgsl/parser_impl.cc index 87dc78e604..e08f6cb347 100644 --- a/src/tint/reader/wgsl/parser_impl.cc +++ b/src/tint/reader/wgsl/parser_impl.cc @@ -123,7 +123,7 @@ const char kStageAttribute[] = "stage"; const char kWorkgroupSizeAttribute[] = "workgroup_size"; // https://gpuweb.github.io/gpuweb/wgsl.html#reserved-keywords -bool is_reserved(Token t) { +bool is_reserved(const Token& t) { return t == "asm" || t == "bf16" || t == "do" || t == "enum" || t == "f64" || t == "handle" || t == "i8" || t == "i16" || t == "i64" || t == "mat" || t == "premerge" || t == "regardless" || t == "typedef" || t == "u8" || t == "u16" || t == "u64" || @@ -181,7 +181,7 @@ class ParserImpl::MultiTokenSource { /// Implicit conversion to Source that returns the combined source from start /// to the current last token's source. operator Source() const { - Source end = parser_->last_token().source().End(); + Source end = parser_->last_source().End(); if (end < start_) { end = start_; } @@ -241,7 +241,7 @@ ParserImpl::VarDeclInfo::VarDeclInfo(Source source_in, ParserImpl::VarDeclInfo::~VarDeclInfo() = default; -ParserImpl::ParserImpl(Source::File const* file) : lexer_(std::make_unique(file)) {} +ParserImpl::ParserImpl(Source::File const* file) : file_(file) {} ParserImpl::~ParserImpl() = default; @@ -274,33 +274,67 @@ void ParserImpl::deprecated(const Source& source, const std::string& msg) { "use of deprecated language feature: " + msg, source); } -Token ParserImpl::next() { - if (!token_queue_.empty()) { - auto t = token_queue_.front(); - token_queue_.pop_front(); - last_token_ = t; - return last_token_; +const Token& ParserImpl::next() { + if (!tokens_[next_token_idx_].IsEof() && !tokens_[next_token_idx_].IsError()) { + // Skip over any placeholder elements + while (true) { + if (!tokens_[next_token_idx_].IsPlaceholder()) { + break; + } + next_token_idx_++; + } } - last_token_ = lexer_->next(); - return last_token_; + last_source_ = tokens_[next_token_idx_].source(); + return tokens_[next_token_idx_++]; } -Token ParserImpl::peek(size_t idx) { - while (token_queue_.size() < (idx + 1)) { - token_queue_.push_back(lexer_->next()); +const Token& ParserImpl::peek(size_t idx) { + if (next_token_idx_ + idx >= tokens_.size()) { + return tokens_[tokens_.size() - 1]; } - return token_queue_[idx]; + + // Skip over any placeholder elements + while (true) { + if (!tokens_[next_token_idx_ + idx].IsPlaceholder()) { + break; + } + idx++; + } + + return tokens_[next_token_idx_ + idx]; } bool ParserImpl::peek_is(Token::Type tok, size_t idx) { return peek(idx).Is(tok); } -Token ParserImpl::last_token() const { - return last_token_; +void ParserImpl::split_token(Token::Type lhs, Token::Type rhs) { + if (next_token_idx_ == 0) { + TINT_ICE(Reader, builder_.Diagnostics()) + << "attempt to update placeholder at beginning of tokens"; + } + if (next_token_idx_ >= tokens_.size()) { + TINT_ICE(Reader, builder_.Diagnostics()) + << "attempt to update placeholder past end of tokens"; + } + if (!tokens_[next_token_idx_].IsPlaceholder()) { + TINT_ICE(Reader, builder_.Diagnostics()) << "attempt to update non-placeholder token"; + } + tokens_[next_token_idx_ - 1].SetType(lhs); + tokens_[next_token_idx_].SetType(rhs); +} + +Source ParserImpl::last_source() const { + return last_source_; +} + +void ParserImpl::InitializeLex() { + Lexer l{file_}; + tokens_ = l.Lex(); } bool ParserImpl::Parse() { + InitializeLex(); translation_unit(); return !has_error(); } @@ -310,7 +344,7 @@ bool ParserImpl::Parse() { void ParserImpl::translation_unit() { bool after_global_decl = false; while (continue_parsing()) { - auto p = peek(); + auto& p = peek(); if (p.IsEof()) { break; } @@ -353,7 +387,7 @@ Maybe ParserImpl::enable_directive() { // Match the extension name. Expect name = {""}; - auto t = peek(); + auto& t = peek(); if (t.IsIdentifier()) { synchronized_ = true; next(); @@ -508,7 +542,7 @@ Maybe ParserImpl::global_decl() { } // We have a statement outside of a function? - auto t = peek(); + auto& t = peek(); auto stat = without_error([&] { return statement(); }); if (stat.matched) { // Attempt to jump to the next '}' - the function might have just been @@ -882,7 +916,7 @@ Maybe ParserImpl::depth_texture() { // | 'rgba32sint' // | 'rgba32float' Expect ParserImpl::expect_texel_format(std::string_view use) { - auto t = next(); + auto& t = next(); if (t == "rgba8unorm") { return ast::TexelFormat::kRgba8Unorm; } @@ -951,7 +985,7 @@ Expect ParserImpl::expect_variable_ident_decl(std:: return Failure::kErrored; } - auto t = peek(); + auto& t = peek(); auto type = type_decl(); if (type.errored) { return Failure::kErrored; @@ -1021,7 +1055,7 @@ Maybe ParserImpl::type_alias() { return Failure::kNoMatch; } - auto t = next(); + auto& t = next(); const char* use = "type alias"; auto name = expect_ident(use); @@ -1069,7 +1103,7 @@ Maybe ParserImpl::type_alias() { // | MAT4x4 LESS_THAN type_decl GREATER_THAN // | texture_samplers Maybe ParserImpl::type_decl() { - auto t = peek(); + auto& t = peek(); Source source; if (match(Token::Type::kIdentifier, &source)) { return builder_.create(source, builder_.Symbols().Register(t.to_str())); @@ -1139,7 +1173,7 @@ Expect ParserImpl::expect_type(std::string_view use) { return type.value; } -Expect ParserImpl::expect_type_decl_pointer(Token t) { +Expect ParserImpl::expect_type_decl_pointer(const Token& t) { const char* use = "ptr declaration"; auto storage_class = ast::StorageClass::kNone; @@ -1180,7 +1214,7 @@ Expect ParserImpl::expect_type_decl_pointer(Token t) { access); } -Expect ParserImpl::expect_type_decl_atomic(Token t) { +Expect ParserImpl::expect_type_decl_atomic(const Token& t) { const char* use = "atomic declaration"; auto subtype = expect_lt_gt_block(use, [&] { return expect_type(use); }); @@ -1191,7 +1225,7 @@ Expect ParserImpl::expect_type_decl_atomic(Token t) { return builder_.ty.atomic(make_source_range_from(t.source()), subtype.value); } -Expect ParserImpl::expect_type_decl_vector(Token t) { +Expect ParserImpl::expect_type_decl_vector(const Token& t) { uint32_t count = 2; if (t.Is(Token::Type::kVec3)) { count = 3; @@ -1212,7 +1246,7 @@ Expect ParserImpl::expect_type_decl_vector(Token t) { return builder_.ty.vec(make_source_range_from(t.source()), subtype, count); } -Expect ParserImpl::expect_type_decl_array(Token t) { +Expect ParserImpl::expect_type_decl_array(const Token& t) { const char* use = "array declaration"; const ast::Expression* size = nullptr; @@ -1244,7 +1278,7 @@ Expect ParserImpl::expect_type_decl_array(Token t) { return builder_.ty.array(make_source_range_from(t.source()), subtype.value, size); } -Expect ParserImpl::expect_type_decl_matrix(Token t) { +Expect ParserImpl::expect_type_decl_matrix(const Token& t) { uint32_t rows = 2; uint32_t columns = 2; if (t.IsMat3xN()) { @@ -1305,7 +1339,7 @@ Expect ParserImpl::expect_storage_class(std::string_view use) // struct_decl // : STRUCT IDENT struct_body_decl Maybe ParserImpl::struct_decl() { - auto t = peek(); + auto& t = peek(); auto source = t.source(); if (!match(Token::Type::kStruct)) { @@ -1334,7 +1368,7 @@ Expect ParserImpl::expect_struct_body_decl() { bool errored = false; while (continue_parsing()) { // Check for the end of the list. - auto t = peek(); + auto& t = peek(); if (!t.IsIdentifier() && !t.Is(Token::Type::kAttr)) { break; } @@ -1481,7 +1515,7 @@ Expect ParserImpl::expect_param_list() { ast::ParameterList ret; while (continue_parsing()) { // Check for the end of the list. - auto t = peek(); + auto& t = peek(); if (!t.IsIdentifier() && !t.Is(Token::Type::kAttr)) { break; } @@ -1522,7 +1556,7 @@ Expect ParserImpl::expect_param() { // | FRAGMENT // | COMPUTE Expect ParserImpl::expect_pipeline_stage() { - auto t = peek(); + auto& t = peek(); if (t == kVertexStage) { next(); // Consume the peek return {ast::PipelineStage::kVertex, t.source()}; @@ -2010,7 +2044,7 @@ Maybe ParserImpl::switch_body() { return Failure::kNoMatch; } - auto t = next(); + auto& t = next(); auto source = t.source(); ast::CaseSelectorList selector_list; @@ -2263,8 +2297,8 @@ Maybe ParserImpl::while_stmt() { // func_call_stmt // : IDENT argument_expression_list Maybe ParserImpl::func_call_stmt() { - auto t = peek(); - auto t2 = peek(1); + auto& t = peek(); + auto& t2 = peek(1); if (!t.IsIdentifier() || !t2.Is(Token::Type::kParenLeft)) { return Failure::kNoMatch; } @@ -2325,7 +2359,7 @@ Maybe ParserImpl::continuing_stmt() { // | paren_expression // | BITCAST LESS_THAN type_decl GREATER_THAN paren_expression Maybe ParserImpl::primary_expression() { - auto t = peek(); + auto& t = peek(); auto source = t.source(); auto lit = const_literal(); @@ -2491,7 +2525,7 @@ Expect ParserImpl::expect_argument_expression_list(std::str // | STAR unary_expression // | AND unary_expression Maybe ParserImpl::unary_expression() { - auto t = peek(); + auto& t = peek(); if (match(Token::Type::kPlusPlus) || match(Token::Type::kMinusMinus)) { add_error(t.source(), @@ -2556,7 +2590,7 @@ Expect ParserImpl::expect_multiplicative_expr(const ast: return lhs; } - auto t = next(); + auto& t = next(); auto source = t.source(); auto name = t.to_name(); @@ -2603,7 +2637,7 @@ Expect ParserImpl::expect_additive_expr(const ast::Expre return lhs; } - auto t = next(); + auto& t = next(); auto source = t.source(); auto rhs = multiplicative_expression(); @@ -2651,7 +2685,7 @@ Expect ParserImpl::expect_shift_expr(const ast::Expressi return lhs; } - auto t = next(); + auto& t = next(); auto source = t.source(); auto rhs = additive_expression(); if (rhs.errored) { @@ -2702,7 +2736,7 @@ Expect ParserImpl::expect_relational_expr(const ast::Exp return lhs; } - auto t = next(); + auto& t = next(); auto source = t.source(); auto name = t.to_name(); @@ -2749,7 +2783,7 @@ Expect ParserImpl::expect_equality_expr(const ast::Expre return lhs; } - auto t = next(); + auto& t = next(); auto source = t.source(); auto name = t.to_name(); @@ -2790,7 +2824,7 @@ Expect ParserImpl::expect_and_expr(const ast::Expression return lhs; } - auto t = next(); + auto& t = next(); auto source = t.source(); auto rhs = equality_expression(); @@ -2903,7 +2937,7 @@ Expect ParserImpl::expect_logical_and_expr(const ast::Ex return lhs; } - auto t = next(); + auto& t = next(); auto source = t.source(); auto rhs = inclusive_or_expression(); @@ -3013,7 +3047,7 @@ Maybe ParserImpl::compound_assignment_operator() { // decrement_stmt // | lhs_expression MINUS_MINUS Maybe ParserImpl::assignment_stmt() { - auto t = peek(); + auto& t = peek(); auto source = t.source(); // tint:295 - Test for `ident COLON` - this is invalid grammar, and without @@ -3076,7 +3110,7 @@ Maybe ParserImpl::assignment_stmt() { // | TRUE // | FALSE Maybe ParserImpl::const_literal() { - auto t = peek(); + auto& t = peek(); if (match(Token::Type::kIntLiteral)) { return create(t.source(), t.to_i64(), ast::IntLiteralExpression::Suffix::kNone); @@ -3141,7 +3175,7 @@ Maybe ParserImpl::attribute_list() { } Expect ParserImpl::expect_attribute() { - auto t = peek(); + auto& t = peek(); auto attr = attribute(); if (attr.errored) { return Failure::kErrored; @@ -3154,7 +3188,7 @@ Expect ParserImpl::expect_attribute() { Maybe ParserImpl::attribute() { using Result = Maybe; - auto t = next(); + auto& t = next(); if (!t.IsIdentifier()) { return Failure::kNoMatch; @@ -3204,7 +3238,7 @@ Maybe ParserImpl::attribute() { ast::InterpolationType type; ast::InterpolationSampling sampling = ast::InterpolationSampling::kNone; - auto type_tok = next(); + auto& type_tok = next(); if (type_tok == "perspective") { type = ast::InterpolationType::kPerspective; } else if (type_tok == "linear") { @@ -3217,7 +3251,7 @@ Maybe ParserImpl::attribute() { if (match(Token::Type::kComma)) { if (!peek_is(Token::Type::kParenRight)) { - auto sampling_tok = next(); + auto& sampling_tok = next(); if (sampling_tok == "center") { sampling = ast::InterpolationSampling::kCenter; } else if (sampling_tok == "centroid") { @@ -3227,7 +3261,6 @@ Maybe ParserImpl::attribute() { } else { return add_error(sampling_tok, "invalid interpolation sampling"); } - match(Token::Type::kComma); } } @@ -3384,7 +3417,7 @@ bool ParserImpl::expect_attributes_consumed(ast::AttributeList& in) { } bool ParserImpl::match(Token::Type tok, Source* source /*= nullptr*/) { - auto t = peek(); + auto& t = peek(); if (source != nullptr) { *source = t.source(); @@ -3398,7 +3431,7 @@ bool ParserImpl::match(Token::Type tok, Source* source /*= nullptr*/) { } bool ParserImpl::expect(std::string_view use, Token::Type tok) { - auto t = peek(); + auto& t = peek(); if (t.Is(tok)) { next(); synchronized_ = true; @@ -3414,9 +3447,9 @@ bool ParserImpl::expect(std::string_view use, Token::Type tok) { auto source = t.source(); source.range.begin.column++; if (t.Is(Token::Type::kShiftRight)) { - token_queue_.push_front(Token(Token::Type::kGreaterThan, source)); + split_token(Token::Type::kGreaterThan, Token::Type::kGreaterThan); } else if (t.Is(Token::Type::kGreaterThanEqual)) { - token_queue_.push_front(Token(Token::Type::kEqual, source)); + split_token(Token::Type::kGreaterThan, Token::Type::kEqual); } synchronized_ = true; @@ -3439,7 +3472,7 @@ bool ParserImpl::expect(std::string_view use, Token::Type tok) { } Expect ParserImpl::expect_sint(std::string_view use) { - auto t = peek(); + auto& t = peek(); if (!t.Is(Token::Type::kIntLiteral) && !t.Is(Token::Type::kIntLiteral_I)) { return add_error(t.source(), "expected signed integer literal", use); } @@ -3482,7 +3515,7 @@ Expect ParserImpl::expect_nonzero_positive_sint(std::string_view use) } Expect ParserImpl::expect_ident(std::string_view use) { - auto t = peek(); + auto& t = peek(); if (t.IsIdentifier()) { synchronized_ = true; next(); @@ -3578,7 +3611,7 @@ bool ParserImpl::sync_to(Token::Type tok, bool consume) { BlockCounters counters; for (size_t i = 0; i < kMaxResynchronizeLookahead; i++) { - auto t = peek(i); + auto& t = peek(i); if (counters.consume(t) > 0) { continue; // Nested block } diff --git a/src/tint/reader/wgsl/parser_impl.h b/src/tint/reader/wgsl/parser_impl.h index a9c0423d5a..424fd2eb42 100644 --- a/src/tint/reader/wgsl/parser_impl.h +++ b/src/tint/reader/wgsl/parser_impl.h @@ -15,7 +15,6 @@ #ifndef SRC_TINT_READER_WGSL_PARSER_IMPL_H_ #define SRC_TINT_READER_WGSL_PARSER_IMPL_H_ -#include #include #include #include @@ -300,6 +299,10 @@ class ParserImpl { explicit ParserImpl(Source::File const* file); ~ParserImpl(); + /// Reads tokens from the source file. This will be called automatically + /// by |parse|. + void InitializeLex(); + /// Run the parser /// @returns true if the parse was successful, false otherwise. bool Parse(); @@ -330,19 +333,19 @@ class ParserImpl { ProgramBuilder& builder() { return builder_; } /// @returns the next token - Token next(); + const Token& next(); /// Peeks ahead and returns the token at `idx` ahead of the current position /// @param idx the index of the token to return /// @returns the token `idx` positions ahead without advancing - Token peek(size_t idx = 0); + const Token& peek(size_t idx = 0); /// Peeks ahead and returns true if the token at `idx` ahead of the current /// position is |tok| /// @param idx the index of the token to return /// @param tok the token to look for /// @returns true if the token `idx` positions ahead is |tok| bool peek_is(Token::Type tok, size_t idx = 0); - /// @returns the last token that was returned by `next()` - Token last_token() const; + /// @returns the last source location that was returned by `next()` + Source last_source() const; /// Appends an error at `t` with the message `msg` /// @param t the token to associate the error with /// @param msg the error message @@ -812,11 +815,11 @@ class ParserImpl { /// Used to ensure that all attributes are consumed. bool expect_attributes_consumed(ast::AttributeList& list); - Expect expect_type_decl_pointer(Token t); - Expect expect_type_decl_atomic(Token t); - Expect expect_type_decl_vector(Token t); - Expect expect_type_decl_array(Token t); - Expect expect_type_decl_matrix(Token t); + Expect expect_type_decl_pointer(const Token& t); + Expect expect_type_decl_atomic(const Token& t); + Expect expect_type_decl_vector(const Token& t); + Expect expect_type_decl_array(const Token& t); + Expect expect_type_decl_matrix(const Token& t); Expect expect_type(std::string_view use); @@ -824,6 +827,8 @@ class ParserImpl { Maybe for_header_initializer(); Maybe for_header_continuing(); + void split_token(Token::Type lhs, Token::Type rhs); + class MultiTokenSource; MultiTokenSource make_source_range(); MultiTokenSource make_source_range_from(const Source& start); @@ -837,9 +842,10 @@ class ParserImpl { return builder_.create(std::forward(args)...); } - std::unique_ptr lexer_; - std::deque token_queue_; - Token last_token_; + Source::File const* const file_; + std::vector tokens_; + size_t next_token_idx_ = 0; + Source last_source_; bool synchronized_ = true; uint32_t parse_depth_ = 0; std::vector sync_tokens_; diff --git a/src/tint/reader/wgsl/parser_impl_pipeline_stage_test.cc b/src/tint/reader/wgsl/parser_impl_pipeline_stage_test.cc index 75a1ec200a..3c730e2ab1 100644 --- a/src/tint/reader/wgsl/parser_impl_pipeline_stage_test.cc +++ b/src/tint/reader/wgsl/parser_impl_pipeline_stage_test.cc @@ -40,7 +40,7 @@ TEST_P(PipelineStageTest, Parses) { EXPECT_EQ(stage.source.range.end.line, 1u); EXPECT_EQ(stage.source.range.end.column, 1u + params.input.size()); - auto t = p->next(); + auto& t = p->next(); EXPECT_TRUE(t.IsEof()); } INSTANTIATE_TEST_SUITE_P( diff --git a/src/tint/reader/wgsl/parser_impl_storage_class_test.cc b/src/tint/reader/wgsl/parser_impl_storage_class_test.cc index deb87fc50b..7144f2cf9d 100644 --- a/src/tint/reader/wgsl/parser_impl_storage_class_test.cc +++ b/src/tint/reader/wgsl/parser_impl_storage_class_test.cc @@ -37,7 +37,7 @@ TEST_P(StorageClassTest, Parses) { EXPECT_FALSE(p->has_error()); EXPECT_EQ(sc.value, params.result); - auto t = p->next(); + auto& t = p->next(); EXPECT_TRUE(t.IsEof()); } INSTANTIATE_TEST_SUITE_P( diff --git a/src/tint/reader/wgsl/parser_impl_test_helper.h b/src/tint/reader/wgsl/parser_impl_test_helper.h index f6ee8ea3b7..1b140399fe 100644 --- a/src/tint/reader/wgsl/parser_impl_test_helper.h +++ b/src/tint/reader/wgsl/parser_impl_test_helper.h @@ -38,6 +38,7 @@ class ParserImplTest : public testing::Test, public ProgramBuilder { std::unique_ptr parser(const std::string& str) { auto file = std::make_unique("test.wgsl", str); auto impl = std::make_unique(file.get()); + impl->InitializeLex(); files_.emplace_back(std::move(file)); return impl; } @@ -60,6 +61,7 @@ class ParserImplTestWithParam : public testing::TestWithParam, public Program std::unique_ptr parser(const std::string& str) { auto file = std::make_unique("test.wgsl", str); auto impl = std::make_unique(file.get()); + impl->InitializeLex(); files_.emplace_back(std::move(file)); return impl; } diff --git a/src/tint/reader/wgsl/parser_impl_variable_decl_test.cc b/src/tint/reader/wgsl/parser_impl_variable_decl_test.cc index 39f8b24dfb..3ed0948db3 100644 --- a/src/tint/reader/wgsl/parser_impl_variable_decl_test.cc +++ b/src/tint/reader/wgsl/parser_impl_variable_decl_test.cc @@ -68,7 +68,7 @@ TEST_F(ParserImplTest, VariableDecl_MissingVar) { EXPECT_FALSE(v.errored); EXPECT_FALSE(p->has_error()); - auto t = p->next(); + auto& t = p->next(); ASSERT_TRUE(t.IsIdentifier()); } diff --git a/src/tint/reader/wgsl/parser_impl_variable_qualifier_test.cc b/src/tint/reader/wgsl/parser_impl_variable_qualifier_test.cc index aefd2b5fb6..0dc0a1a9b8 100644 --- a/src/tint/reader/wgsl/parser_impl_variable_qualifier_test.cc +++ b/src/tint/reader/wgsl/parser_impl_variable_qualifier_test.cc @@ -40,7 +40,7 @@ TEST_P(VariableQualifierTest, ParsesStorageClass) { EXPECT_EQ(sc->storage_class, params.storage_class); EXPECT_EQ(sc->access, params.access); - auto t = p->next(); + auto& t = p->next(); EXPECT_TRUE(t.IsEof()); } INSTANTIATE_TEST_SUITE_P( @@ -83,7 +83,7 @@ TEST_F(ParserImplTest, VariableQualifier_MissingLessThan) { EXPECT_FALSE(sc.errored); EXPECT_FALSE(sc.matched); - auto t = p->next(); + auto& t = p->next(); ASSERT_TRUE(t.Is(Token::Type::kIdentifier)); } @@ -94,7 +94,7 @@ TEST_F(ParserImplTest, VariableQualifier_MissingLessThan_AfterSC) { EXPECT_FALSE(sc.errored); EXPECT_FALSE(sc.matched); - auto t = p->next(); + auto& t = p->next(); ASSERT_TRUE(t.Is(Token::Type::kIdentifier)); } diff --git a/src/tint/reader/wgsl/token.cc b/src/tint/reader/wgsl/token.cc index 44255d3239..a7b437fb28 100644 --- a/src/tint/reader/wgsl/token.cc +++ b/src/tint/reader/wgsl/token.cc @@ -37,6 +37,8 @@ std::string_view Token::TypeToName(Type type) { return "'i'-suffixed integer literal"; case Token::Type::kIntLiteral_U: return "'u'-suffixed integer literal"; + case Token::Type::kPlaceholder: + return "placeholder"; case Token::Type::kUninitialized: return "uninitialized"; @@ -285,13 +287,9 @@ Token::Token(Type type, const Source& source) : type_(type), source_(source) {} Token::Token(Token&&) = default; -Token::Token(const Token&) = default; - Token::~Token() = default; -Token& Token::operator=(const Token& rhs) = default; - -bool Token::operator==(std::string_view ident) { +bool Token::operator==(std::string_view ident) const { if (type_ != Type::kIdentifier) { return false; } diff --git a/src/tint/reader/wgsl/token.h b/src/tint/reader/wgsl/token.h index 473588dbce..352c18ff17 100644 --- a/src/tint/reader/wgsl/token.h +++ b/src/tint/reader/wgsl/token.h @@ -32,6 +32,8 @@ class Token { kError = -2, /// Uninitialized token kUninitialized = 0, + /// Placeholder token which maybe fillled in later + kPlaceholder = 1, /// End of input string reached kEOF, @@ -310,19 +312,16 @@ class Token { Token(Type type, const Source& source, double val); /// Move constructor Token(Token&&); - /// Copy constructor - Token(const Token&); ~Token(); - /// Assignment operator - /// @param b the token to copy - /// @return Token - Token& operator=(const Token& b); - /// Equality operator with an identifier /// @param ident the identifier string /// @return true if this token is an identifier and is equal to ident. - bool operator==(std::string_view ident); + bool operator==(std::string_view ident) const; + + /// Sets the token to the given type + /// @param type the type to set + void SetType(Token::Type type) { type_ = type; } /// Returns true if the token is of the given type /// @param t the type to check against. @@ -331,6 +330,8 @@ class Token { /// @returns true if the token is uninitialized bool IsUninitialized() const { return type_ == Type::kUninitialized; } + /// @returns true if the token is a placeholder + bool IsPlaceholder() const { return type_ == Type::kPlaceholder; } /// @returns true if the token is EOF bool IsEof() const { return type_ == Type::kEOF; } /// @returns true if the token is Error @@ -372,6 +373,11 @@ class Token { return type_ == Type::kVec2 || type_ == Type::kVec3 || type_ == Type::kVec4; } + /// @returns true if the token can be split during parse into component tokens + bool IsSplittable() const { + return Is(Token::Type::kShiftRight) || Is(Token::Type::kGreaterThanEqual); + } + /// @returns the source information for this token Source source() const { return source_; }