From 580d6c7f3eedc4fa51a0c4a0438fe2d9779f6a03 Mon Sep 17 00:00:00 2001 From: Ben Clayton Date: Mon, 2 Nov 2020 15:25:18 +0000 Subject: [PATCH] Produce end ranges for tokens and AST nodes This includes a couple of position fixes in `lexer.cc`. The source for identifiers (vars, params, etc) now refer to the identifier, not the first token of the construct. Bug: tint:282 Change-Id: I58cb8422a4af1c7dc5f84431fd7f06b823b514c5 Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/31444 Commit-Queue: Ben Clayton Reviewed-by: dan sinclair --- src/ast/decorated_variable_test.cc | 6 +- src/ast/discard_statement_test.cc | 7 +- src/ast/fallthrough_statement_test.cc | 2 + src/ast/struct_member_test.cc | 6 +- src/ast/struct_test.cc | 11 ++- src/ast/variable_test.cc | 10 ++- src/reader/wgsl/lexer.cc | 37 +++++++--- src/reader/wgsl/lexer.h | 3 +- src/reader/wgsl/lexer_test.cc | 32 +++++++++ src/reader/wgsl/parser_impl.cc | 67 ++++++++----------- src/reader/wgsl/parser_impl.h | 10 ++- .../parser_impl_global_constant_decl_test.cc | 5 ++ .../parser_impl_global_variable_decl_test.cc | 20 ++++++ .../wgsl/parser_impl_param_list_test.cc | 22 +++++- .../wgsl/parser_impl_struct_member_test.cc | 15 +++++ .../wgsl/parser_impl_variable_decl_test.cc | 12 +++- .../parser_impl_variable_ident_decl_test.cc | 31 +++++---- .../wgsl/parser_impl_variable_stmt_test.cc | 15 +++++ src/reader/wgsl/token_test.cc | 11 ++- 19 files changed, 243 insertions(+), 79 deletions(-) diff --git a/src/ast/decorated_variable_test.cc b/src/ast/decorated_variable_test.cc index 3774d1c57d..c5db5afb24 100644 --- a/src/ast/decorated_variable_test.cc +++ b/src/ast/decorated_variable_test.cc @@ -42,10 +42,12 @@ TEST_F(DecoratedVariableTest, Creation) { EXPECT_EQ(dv.type(), &t); EXPECT_EQ(dv.source().range.begin.line, 0u); EXPECT_EQ(dv.source().range.begin.column, 0u); + EXPECT_EQ(dv.source().range.end.line, 0u); + EXPECT_EQ(dv.source().range.end.column, 0u); } TEST_F(DecoratedVariableTest, CreationWithSource) { - Source s{Source::Location{27, 4}}; + Source s{Source::Range{Source::Location{27, 4}, Source::Location{27, 5}}}; type::F32Type t; auto var = std::make_unique(s, "i", StorageClass::kPrivate, &t); DecoratedVariable dv(std::move(var)); @@ -55,6 +57,8 @@ TEST_F(DecoratedVariableTest, CreationWithSource) { EXPECT_EQ(dv.type(), &t); EXPECT_EQ(dv.source().range.begin.line, 27u); EXPECT_EQ(dv.source().range.begin.column, 4u); + EXPECT_EQ(dv.source().range.end.line, 27u); + EXPECT_EQ(dv.source().range.end.column, 5u); } TEST_F(DecoratedVariableTest, NoDecorations) { diff --git a/src/ast/discard_statement_test.cc b/src/ast/discard_statement_test.cc index 4a9e7793c5..9433421c36 100644 --- a/src/ast/discard_statement_test.cc +++ b/src/ast/discard_statement_test.cc @@ -28,12 +28,17 @@ TEST_F(DiscardStatementTest, Creation) { DiscardStatement stmt; EXPECT_EQ(stmt.source().range.begin.line, 0u); EXPECT_EQ(stmt.source().range.begin.column, 0u); + EXPECT_EQ(stmt.source().range.end.line, 0u); + EXPECT_EQ(stmt.source().range.end.column, 0u); } TEST_F(DiscardStatementTest, Creation_WithSource) { - DiscardStatement stmt(Source{Source::Location{20, 2}}); + DiscardStatement stmt( + Source{Source::Range{Source::Location{20, 2}, Source::Location{20, 5}}}); EXPECT_EQ(stmt.source().range.begin.line, 20u); EXPECT_EQ(stmt.source().range.begin.column, 2u); + EXPECT_EQ(stmt.source().range.end.line, 20u); + EXPECT_EQ(stmt.source().range.end.column, 5u); } TEST_F(DiscardStatementTest, IsDiscard) { diff --git a/src/ast/fallthrough_statement_test.cc b/src/ast/fallthrough_statement_test.cc index e1ed7f2ecd..9552627aca 100644 --- a/src/ast/fallthrough_statement_test.cc +++ b/src/ast/fallthrough_statement_test.cc @@ -26,6 +26,8 @@ TEST_F(FallthroughStatementTest, Creation) { FallthroughStatement stmt; EXPECT_EQ(stmt.source().range.begin.line, 0u); EXPECT_EQ(stmt.source().range.begin.column, 0u); + EXPECT_EQ(stmt.source().range.end.line, 0u); + EXPECT_EQ(stmt.source().range.end.column, 0u); } TEST_F(FallthroughStatementTest, Creation_WithSource) { diff --git a/src/ast/struct_member_test.cc b/src/ast/struct_member_test.cc index d20ef7b0e8..f74856a9d3 100644 --- a/src/ast/struct_member_test.cc +++ b/src/ast/struct_member_test.cc @@ -39,11 +39,13 @@ TEST_F(StructMemberTest, Creation) { EXPECT_TRUE(st.decorations()[0]->IsOffset()); EXPECT_EQ(st.source().range.begin.line, 0u); EXPECT_EQ(st.source().range.begin.column, 0u); + EXPECT_EQ(st.source().range.end.line, 0u); + EXPECT_EQ(st.source().range.end.column, 0u); } TEST_F(StructMemberTest, CreationWithSource) { type::I32Type i32; - Source s{Source::Location{27, 4}}; + Source s{Source::Range{Source::Location{27, 4}, Source::Location{27, 8}}}; StructMember st{s, "a", &i32, {}}; EXPECT_EQ(st.name(), "a"); @@ -51,6 +53,8 @@ TEST_F(StructMemberTest, CreationWithSource) { EXPECT_EQ(st.decorations().size(), 0u); EXPECT_EQ(st.source().range.begin.line, 27u); EXPECT_EQ(st.source().range.begin.column, 4u); + EXPECT_EQ(st.source().range.end.line, 27u); + EXPECT_EQ(st.source().range.end.column, 8u); } TEST_F(StructMemberTest, IsValid) { diff --git a/src/ast/struct_test.cc b/src/ast/struct_test.cc index 828f454aee..55abf23d11 100644 --- a/src/ast/struct_test.cc +++ b/src/ast/struct_test.cc @@ -40,6 +40,8 @@ TEST_F(StructTest, Creation) { EXPECT_TRUE(s.decorations().empty()); EXPECT_EQ(s.source().range.begin.line, 0u); EXPECT_EQ(s.source().range.begin.column, 0u); + EXPECT_EQ(s.source().range.end.line, 0u); + EXPECT_EQ(s.source().range.end.column, 0u); } TEST_F(StructTest, Creation_WithDecorations) { @@ -58,6 +60,8 @@ TEST_F(StructTest, Creation_WithDecorations) { EXPECT_EQ(s.decorations()[0], StructDecoration::kBlock); EXPECT_EQ(s.source().range.begin.line, 0u); EXPECT_EQ(s.source().range.begin.column, 0u); + EXPECT_EQ(s.source().range.end.line, 0u); + EXPECT_EQ(s.source().range.end.column, 0u); } TEST_F(StructTest, CreationWithSourceAndDecorations) { @@ -70,13 +74,16 @@ TEST_F(StructTest, CreationWithSourceAndDecorations) { StructDecorationList decos; decos.push_back(StructDecoration::kBlock); - Struct s{Source{Source::Location{27, 4}}, std::move(decos), - std::move(members)}; + Struct s{ + Source{Source::Range{Source::Location{27, 4}, Source::Location{27, 8}}}, + std::move(decos), std::move(members)}; EXPECT_EQ(s.members().size(), 1u); ASSERT_EQ(s.decorations().size(), 1u); EXPECT_EQ(s.decorations()[0], StructDecoration::kBlock); EXPECT_EQ(s.source().range.begin.line, 27u); EXPECT_EQ(s.source().range.begin.column, 4u); + EXPECT_EQ(s.source().range.end.line, 27u); + EXPECT_EQ(s.source().range.end.column, 8u); } TEST_F(StructTest, IsValid) { diff --git a/src/ast/variable_test.cc b/src/ast/variable_test.cc index ad006e305b..3b0f84520c 100644 --- a/src/ast/variable_test.cc +++ b/src/ast/variable_test.cc @@ -34,10 +34,12 @@ TEST_F(VariableTest, Creation) { EXPECT_EQ(v.type(), &t); EXPECT_EQ(v.source().range.begin.line, 0u); EXPECT_EQ(v.source().range.begin.column, 0u); + EXPECT_EQ(v.source().range.end.line, 0u); + EXPECT_EQ(v.source().range.end.column, 0u); } TEST_F(VariableTest, CreationWithSource) { - Source s{Source::Location{27, 4}}; + Source s{Source::Range{Source::Location{27, 4}, Source::Location{27, 5}}}; type::F32Type t; Variable v(s, "i", StorageClass::kPrivate, &t); @@ -46,10 +48,12 @@ TEST_F(VariableTest, CreationWithSource) { EXPECT_EQ(v.type(), &t); EXPECT_EQ(v.source().range.begin.line, 27u); EXPECT_EQ(v.source().range.begin.column, 4u); + EXPECT_EQ(v.source().range.end.line, 27u); + EXPECT_EQ(v.source().range.end.column, 5u); } TEST_F(VariableTest, CreationEmpty) { - Source s{Source::Location{27, 4}}; + Source s{Source::Range{Source::Location{27, 4}, Source::Location{27, 7}}}; Variable v; v.set_source(s); v.set_storage_class(StorageClass::kWorkgroup); @@ -63,6 +67,8 @@ TEST_F(VariableTest, CreationEmpty) { EXPECT_EQ(v.type(), &t); EXPECT_EQ(v.source().range.begin.line, 27u); EXPECT_EQ(v.source().range.begin.column, 4u); + EXPECT_EQ(v.source().range.end.line, 27u); + EXPECT_EQ(v.source().range.end.column, 7u); } TEST_F(VariableTest, IsValid) { diff --git a/src/reader/wgsl/lexer.cc b/src/reader/wgsl/lexer.cc index 2fa4eb852a..eb29668c8c 100644 --- a/src/reader/wgsl/lexer.cc +++ b/src/reader/wgsl/lexer.cc @@ -44,7 +44,7 @@ Token Lexer::next() { skip_comments(); if (is_eof()) { - return {Token::Type::kEOF, make_source()}; + return {Token::Type::kEOF, begin_source()}; } auto t = try_hex_integer(); @@ -77,10 +77,10 @@ Token Lexer::next() { return t; } - return {Token::Type::kError, make_source(), "invalid character found"}; + return {Token::Type::kError, begin_source(), "invalid character found"}; } -Source Lexer::make_source() const { +Source Lexer::begin_source() const { Source src{}; src.file = file_; src.range.begin = location_; @@ -88,6 +88,10 @@ Source Lexer::make_source() const { return src; } +void Lexer::end_source(Source& src) const { + src.range.end = location_; +} + bool Lexer::is_eof() const { return pos_ >= len_; } @@ -153,7 +157,7 @@ Token Lexer::try_float() { auto start = pos_; auto end = pos_; - auto source = make_source(); + auto source = begin_source(); if (matches(end, "-")) { end++; @@ -195,6 +199,8 @@ Token Lexer::try_float() { pos_ = end; location_.column += (end - start); + end_source(source); + auto res = strtod(file_->content.c_str() + start, nullptr); // This handles if the number is a really small in the exponent if (res > 0 && res < static_cast(std::numeric_limits::min())) { @@ -224,6 +230,8 @@ Token Lexer::build_token_from_int_if_possible(Source source, "u32 (" + file_->content.substr(start, end - start) + ") too large"}; } pos_ += 1; + location_.column += 1; + end_source(source); return {source, static_cast(res)}; } @@ -237,6 +245,7 @@ Token Lexer::build_token_from_int_if_possible(Source source, Token::Type::kError, source, "i32 (" + file_->content.substr(start, end - start) + ") too large"}; } + end_source(source); return {source, static_cast(res)}; } @@ -244,7 +253,7 @@ Token Lexer::try_hex_integer() { auto start = pos_; auto end = pos_; - auto source = make_source(); + auto source = begin_source(); if (matches(end, "-")) { end++; @@ -268,7 +277,7 @@ Token Lexer::try_integer() { auto start = pos_; auto end = start; - auto source = make_source(); + auto source = begin_source(); if (matches(end, "-")) { end++; @@ -299,7 +308,7 @@ Token Lexer::try_ident() { return {}; } - auto source = make_source(); + auto source = begin_source(); auto s = pos_; while (!is_eof() && is_alphanum(file_->content[pos_])) { @@ -313,6 +322,8 @@ Token Lexer::try_ident() { return t; } + end_source(source); + t = check_keyword(source, str); if (!t.IsUninitialized()) { return t; @@ -325,7 +336,7 @@ Token Lexer::try_string() { if (!matches(pos_, R"(")")) return {}; - auto source = make_source(); + auto source = begin_source(); pos_++; auto start = pos_; @@ -333,15 +344,19 @@ Token Lexer::try_string() { pos_++; } auto end = pos_; - pos_++; + if (matches(pos_, R"(")")) { + pos_++; + } location_.column += (pos_ - start) + 1; + end_source(source); + return {Token::Type::kStringLiteral, source, file_->content.substr(start, end - start)}; } Token Lexer::try_punctuation() { - auto source = make_source(); + auto source = begin_source(); auto type = Token::Type::kUninitialized; if (matches(pos_, "[[")) { @@ -474,6 +489,8 @@ Token Lexer::try_punctuation() { location_.column += 1; } + end_source(source); + return {type, source}; } diff --git a/src/reader/wgsl/lexer.h b/src/reader/wgsl/lexer.h index 1bfaf0ac3f..ef1a839b86 100644 --- a/src/reader/wgsl/lexer.h +++ b/src/reader/wgsl/lexer.h @@ -53,7 +53,8 @@ class Lexer { Token try_punctuation(); Token try_string(); - Source make_source() const; + Source begin_source() const; + void end_source(Source&) const; bool is_eof() const; bool is_alpha(char ch) const; diff --git a/src/reader/wgsl/lexer_test.cc b/src/reader/wgsl/lexer_test.cc index 810cf51e61..e957cb8672 100644 --- a/src/reader/wgsl/lexer_test.cc +++ b/src/reader/wgsl/lexer_test.cc @@ -40,6 +40,8 @@ TEST_F(LexerTest, Skips_Whitespace) { EXPECT_TRUE(t.IsIdentifier()); EXPECT_EQ(t.source().range.begin.line, 2u); EXPECT_EQ(t.source().range.begin.column, 6u); + EXPECT_EQ(t.source().range.end.line, 2u); + EXPECT_EQ(t.source().range.end.column, 11u); EXPECT_EQ(t.to_str(), "ident"); t = l.next(); @@ -57,12 +59,16 @@ ident1 #ends with comment EXPECT_TRUE(t.IsIdentifier()); EXPECT_EQ(t.source().range.begin.line, 2u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 2u); + EXPECT_EQ(t.source().range.end.column, 7u); EXPECT_EQ(t.to_str(), "ident1"); t = l.next(); EXPECT_TRUE(t.IsIdentifier()); EXPECT_EQ(t.source().range.begin.line, 4u); EXPECT_EQ(t.source().range.begin.column, 2u); + EXPECT_EQ(t.source().range.end.line, 4u); + EXPECT_EQ(t.source().range.end.column, 8u); EXPECT_EQ(t.to_str(), "ident2"); t = l.next(); @@ -78,18 +84,24 @@ TEST_F(LexerTest, StringTest_Parse) { EXPECT_EQ(t.to_str(), "id"); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 3u); t = l.next(); EXPECT_TRUE(t.IsStringLiteral()); EXPECT_EQ(t.to_str(), "this is string content"); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 4u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 28u); t = l.next(); EXPECT_TRUE(t.IsIdentifier()); EXPECT_EQ(t.to_str(), "id2"); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 29u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 32u); } TEST_F(LexerTest, StringTest_Unterminated) { @@ -101,12 +113,16 @@ TEST_F(LexerTest, StringTest_Unterminated) { EXPECT_EQ(t.to_str(), "id"); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 3u); t = l.next(); EXPECT_TRUE(t.IsStringLiteral()); EXPECT_EQ(t.to_str(), "this is string content"); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 4u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 27u); t = l.next(); EXPECT_TRUE(t.IsEof()); @@ -131,6 +147,8 @@ TEST_P(FloatTest, Parse) { EXPECT_EQ(t.to_f32(), params.result); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); t = l.next(); EXPECT_TRUE(t.IsEof()); @@ -182,6 +200,8 @@ TEST_P(IdentifierTest, Parse) { EXPECT_TRUE(t.IsIdentifier()); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(GetParam())); EXPECT_EQ(t.to_str(), GetParam()); } INSTANTIATE_TEST_SUITE_P( @@ -216,6 +236,8 @@ TEST_P(IntegerTest_HexSigned, Matches) { EXPECT_TRUE(t.IsSintLiteral()); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); EXPECT_EQ(t.to_i32(), params.result); } INSTANTIATE_TEST_SUITE_P( @@ -263,6 +285,8 @@ TEST_P(IntegerTest_HexUnsigned, Matches) { EXPECT_TRUE(t.IsUintLiteral()); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); EXPECT_EQ(t.to_u32(), params.result); t = l.next(); @@ -306,6 +330,8 @@ TEST_P(IntegerTest_Unsigned, Matches) { EXPECT_EQ(t.to_u32(), params.result); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); } INSTANTIATE_TEST_SUITE_P(LexerTest, IntegerTest_Unsigned, @@ -333,6 +359,8 @@ TEST_P(IntegerTest_Signed, Matches) { EXPECT_EQ(t.to_i32(), params.result); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); } INSTANTIATE_TEST_SUITE_P( LexerTest, @@ -375,6 +403,8 @@ TEST_P(PunctuationTest, Parses) { EXPECT_TRUE(t.Is(params.type)); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); t = l.next(); EXPECT_EQ(t.source().range.begin.column, @@ -426,6 +456,8 @@ TEST_P(KeywordTest, Parses) { EXPECT_TRUE(t.Is(params.type)) << params.input; EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); t = l.next(); EXPECT_EQ(t.source().range.begin.column, diff --git a/src/reader/wgsl/parser_impl.cc b/src/reader/wgsl/parser_impl.cc index c445f02bdf..7951a53ac9 100644 --- a/src/reader/wgsl/parser_impl.cc +++ b/src/reader/wgsl/parser_impl.cc @@ -354,21 +354,18 @@ std::unique_ptr ParserImpl::global_constant_decl() { if (!t.IsConst()) return nullptr; - auto source = t.source(); next(); // Consume the peek - std::string name; - ast::type::Type* type; - std::tie(name, type) = variable_ident_decl(); + auto decl = variable_ident_decl(); if (has_error()) return nullptr; - if (name.empty() || type == nullptr) { + if (decl.name.empty() || decl.type == nullptr) { set_error(peek(), "error parsing constant variable identifier"); return nullptr; } - auto var = std::make_unique(source, name, - ast::StorageClass::kNone, type); + auto var = std::make_unique( + decl.source, decl.name, ast::StorageClass::kNone, decl.type); var->set_is_const(true); t = next(); @@ -567,7 +564,6 @@ std::unique_ptr ParserImpl::variable_decoration() { // : VAR variable_storage_decoration? variable_ident_decl std::unique_ptr ParserImpl::variable_decl() { auto t = peek(); - auto source = t.source(); if (!t.IsVar()) return nullptr; @@ -577,17 +573,15 @@ std::unique_ptr ParserImpl::variable_decl() { if (has_error()) return {}; - std::string name; - ast::type::Type* type; - std::tie(name, type) = variable_ident_decl(); + auto decl = variable_ident_decl(); if (has_error()) return nullptr; - if (name.empty() || type == nullptr) { + if (decl.name.empty() || decl.type == nullptr) { set_error(peek(), "invalid identifier declaration"); return nullptr; } - return std::make_unique(source, name, sc, type); + return std::make_unique(decl.source, decl.name, sc, decl.type); } // texture_sampler_types @@ -1034,12 +1028,13 @@ ast::type::ImageFormat ParserImpl::image_storage_type() { // variable_ident_decl // : IDENT COLON type_decl -std::pair ParserImpl::variable_ident_decl() { +ParserImpl::TypedIdentifier ParserImpl::variable_ident_decl() { auto t = peek(); if (!t.IsIdentifier()) return {}; auto name = t.to_str(); + auto source = t.source(); next(); // Consume the peek t = next(); @@ -1056,7 +1051,7 @@ std::pair ParserImpl::variable_ident_decl() { return {}; } - return {name, type}; + return {type, name, source}; } // variable_storage_decoration @@ -1620,7 +1615,6 @@ ast::StructMemberList ParserImpl::struct_body_decl() { // : struct_member_decoration_decl+ variable_ident_decl SEMICOLON std::unique_ptr ParserImpl::struct_member() { auto t = peek(); - auto source = t.source(); ast::StructMemberDecorationList decos; for (;;) { @@ -1635,12 +1629,10 @@ std::unique_ptr ParserImpl::struct_member() { if (has_error()) return nullptr; - std::string name; - ast::type::Type* type; - std::tie(name, type) = variable_ident_decl(); + auto decl = variable_ident_decl(); if (has_error()) return nullptr; - if (name.empty() || type == nullptr) { + if (decl.name.empty() || decl.type == nullptr) { set_error(peek(), "invalid identifier declaration"); return nullptr; } @@ -1651,7 +1643,7 @@ std::unique_ptr ParserImpl::struct_member() { return nullptr; } - return std::make_unique(source, name, type, + return std::make_unique(decl.source, decl.name, decl.type, std::move(decos)); } @@ -1981,21 +1973,18 @@ std::unique_ptr ParserImpl::function_header() { // | (variable_ident_decl COMMA)* variable_ident_decl ast::VariableList ParserImpl::param_list() { auto t = peek(); - auto source = t.source(); ast::VariableList ret; - std::string name; - ast::type::Type* type; - std::tie(name, type) = variable_ident_decl(); + auto decl = variable_ident_decl(); if (has_error()) return {}; - if (name.empty() || type == nullptr) + if (decl.name.empty() || decl.type == nullptr) return {}; for (;;) { - auto var = std::make_unique(source, name, - ast::StorageClass::kNone, type); + auto var = std::make_unique( + decl.source, decl.name, ast::StorageClass::kNone, decl.type); // Formal parameters are treated like a const declaration where the // initializer value is provided by the call's argument. The key point is // that it's not updatable after intially set. This is unlike C or GLSL @@ -2007,13 +1996,12 @@ ast::VariableList ParserImpl::param_list() { if (!t.IsComma()) break; - source = t.source(); next(); // Consume the peek - std::tie(name, type) = variable_ident_decl(); + decl = variable_ident_decl(); if (has_error()) return {}; - if (name.empty() || type == nullptr) { + if (decl.name.empty() || decl.type == nullptr) { set_error(t, "found , but no variable declaration"); return {}; } @@ -2279,16 +2267,13 @@ std::unique_ptr ParserImpl::return_stmt() { // | CONST variable_ident_decl EQUAL logical_or_expression std::unique_ptr ParserImpl::variable_stmt() { auto t = peek(); - auto source = t.source(); if (t.IsConst()) { next(); // Consume the peek - std::string name; - ast::type::Type* type; - std::tie(name, type) = variable_ident_decl(); + auto decl = variable_ident_decl(); if (has_error()) return nullptr; - if (name.empty() || type == nullptr) { + if (decl.name.empty() || decl.type == nullptr) { set_error(peek(), "unable to parse variable declaration"); return nullptr; } @@ -2307,12 +2292,13 @@ std::unique_ptr ParserImpl::variable_stmt() { return nullptr; } - auto var = std::make_unique(source, name, - ast::StorageClass::kNone, type); + auto var = std::make_unique( + decl.source, decl.name, ast::StorageClass::kNone, decl.type); var->set_is_const(true); var->set_constructor(std::move(constructor)); - return std::make_unique(source, std::move(var)); + return std::make_unique(decl.source, + std::move(var)); } auto var = variable_decl(); @@ -2334,7 +2320,8 @@ std::unique_ptr ParserImpl::variable_stmt() { var->set_constructor(std::move(constructor)); } - return std::make_unique(source, std::move(var)); + return std::make_unique(var->source(), + std::move(var)); } // if_stmt diff --git a/src/reader/wgsl/parser_impl.h b/src/reader/wgsl/parser_impl.h index 4fb36444ba..82d04abeb3 100644 --- a/src/reader/wgsl/parser_impl.h +++ b/src/reader/wgsl/parser_impl.h @@ -76,6 +76,14 @@ struct ForHeader { /// ParserImpl for WGSL source data class ParserImpl { public: + /// TypedIdentifier holds a parsed identifier and type. Returned by + /// variable_ident_decl(). + struct TypedIdentifier { + ast::type::Type* type = nullptr; /// Parsed type. + std::string name; /// Parsed identifier. + Source source; /// Source to the identifier. + }; + /// Creates a new parser using the given file /// @param ctx the non-null context object /// @param file the input source file to parse @@ -146,7 +154,7 @@ class ParserImpl { std::unique_ptr variable_decl(); /// Parses a `variable_ident_decl` grammar element /// @returns the identifier and type parsed or empty otherwise - std::pair variable_ident_decl(); + TypedIdentifier variable_ident_decl(); /// Parses a `variable_storage_decoration` grammar element /// @returns the storage class or StorageClass::kNone if none matched ast::StorageClass variable_storage_decoration(); diff --git a/src/reader/wgsl/parser_impl_global_constant_decl_test.cc b/src/reader/wgsl/parser_impl_global_constant_decl_test.cc index f08d17ccc4..be0535aa65 100644 --- a/src/reader/wgsl/parser_impl_global_constant_decl_test.cc +++ b/src/reader/wgsl/parser_impl_global_constant_decl_test.cc @@ -34,6 +34,11 @@ TEST_F(ParserImplTest, GlobalConstantDecl) { ASSERT_NE(e->type(), nullptr); EXPECT_TRUE(e->type()->IsF32()); + EXPECT_EQ(e->source().range.begin.line, 1u); + EXPECT_EQ(e->source().range.begin.column, 7u); + EXPECT_EQ(e->source().range.end.line, 1u); + EXPECT_EQ(e->source().range.end.column, 8u); + ASSERT_NE(e->constructor(), nullptr); EXPECT_TRUE(e->constructor()->IsConstructor()); } diff --git a/src/reader/wgsl/parser_impl_global_variable_decl_test.cc b/src/reader/wgsl/parser_impl_global_variable_decl_test.cc index 80238ba539..d6b74c552e 100644 --- a/src/reader/wgsl/parser_impl_global_variable_decl_test.cc +++ b/src/reader/wgsl/parser_impl_global_variable_decl_test.cc @@ -33,6 +33,11 @@ TEST_F(ParserImplTest, GlobalVariableDecl_WithoutConstructor) { EXPECT_TRUE(e->type()->IsF32()); EXPECT_EQ(e->storage_class(), ast::StorageClass::kOutput); + EXPECT_EQ(e->source().range.begin.line, 1u); + EXPECT_EQ(e->source().range.begin.column, 10u); + EXPECT_EQ(e->source().range.end.line, 1u); + EXPECT_EQ(e->source().range.end.column, 11u); + ASSERT_EQ(e->constructor(), nullptr); ASSERT_FALSE(e->IsDecorated()); } @@ -47,6 +52,11 @@ TEST_F(ParserImplTest, GlobalVariableDecl_WithConstructor) { EXPECT_TRUE(e->type()->IsF32()); EXPECT_EQ(e->storage_class(), ast::StorageClass::kOutput); + EXPECT_EQ(e->source().range.begin.line, 1u); + EXPECT_EQ(e->source().range.begin.column, 10u); + EXPECT_EQ(e->source().range.end.line, 1u); + EXPECT_EQ(e->source().range.end.column, 11u); + ASSERT_NE(e->constructor(), nullptr); ASSERT_TRUE(e->constructor()->IsConstructor()); ASSERT_TRUE(e->constructor()->AsConstructor()->IsScalarConstructor()); @@ -66,6 +76,11 @@ TEST_F(ParserImplTest, GlobalVariableDecl_WithDecoration) { EXPECT_TRUE(e->type()->IsF32()); EXPECT_EQ(e->storage_class(), ast::StorageClass::kOutput); + EXPECT_EQ(e->source().range.begin.line, 1u); + EXPECT_EQ(e->source().range.begin.column, 33u); + EXPECT_EQ(e->source().range.end.line, 1u); + EXPECT_EQ(e->source().range.end.column, 34u); + ASSERT_EQ(e->constructor(), nullptr); ASSERT_TRUE(e->IsDecorated()); @@ -89,6 +104,11 @@ TEST_F(ParserImplTest, GlobalVariableDecl_WithDecoration_MulitpleGroups) { EXPECT_TRUE(e->type()->IsF32()); EXPECT_EQ(e->storage_class(), ast::StorageClass::kOutput); + EXPECT_EQ(e->source().range.begin.line, 1u); + EXPECT_EQ(e->source().range.begin.column, 36u); + EXPECT_EQ(e->source().range.end.line, 1u); + EXPECT_EQ(e->source().range.end.column, 37u); + ASSERT_EQ(e->constructor(), nullptr); ASSERT_TRUE(e->IsDecorated()); diff --git a/src/reader/wgsl/parser_impl_param_list_test.cc b/src/reader/wgsl/parser_impl_param_list_test.cc index 1d0b12a635..4ce27eb50b 100644 --- a/src/reader/wgsl/parser_impl_param_list_test.cc +++ b/src/reader/wgsl/parser_impl_param_list_test.cc @@ -39,6 +39,11 @@ TEST_F(ParserImplTest, ParamList_Single) { EXPECT_EQ(e[0]->name(), "a"); EXPECT_EQ(e[0]->type(), i32); EXPECT_TRUE(e[0]->is_const()); + + ASSERT_EQ(e[0]->source().range.begin.line, 1u); + ASSERT_EQ(e[0]->source().range.begin.column, 1u); + ASSERT_EQ(e[0]->source().range.end.line, 1u); + ASSERT_EQ(e[0]->source().range.end.column, 2u); } TEST_F(ParserImplTest, ParamList_Multiple) { @@ -55,13 +60,28 @@ TEST_F(ParserImplTest, ParamList_Multiple) { EXPECT_EQ(e[0]->type(), i32); EXPECT_TRUE(e[0]->is_const()); + ASSERT_EQ(e[0]->source().range.begin.line, 1u); + ASSERT_EQ(e[0]->source().range.begin.column, 1u); + ASSERT_EQ(e[0]->source().range.end.line, 1u); + ASSERT_EQ(e[0]->source().range.end.column, 2u); + EXPECT_EQ(e[1]->name(), "b"); EXPECT_EQ(e[1]->type(), f32); EXPECT_TRUE(e[1]->is_const()); + ASSERT_EQ(e[1]->source().range.begin.line, 1u); + ASSERT_EQ(e[1]->source().range.begin.column, 10u); + ASSERT_EQ(e[1]->source().range.end.line, 1u); + ASSERT_EQ(e[1]->source().range.end.column, 11u); + EXPECT_EQ(e[2]->name(), "c"); EXPECT_EQ(e[2]->type(), vec2); - EXPECT_TRUE(e[1]->is_const()); + EXPECT_TRUE(e[2]->is_const()); + + ASSERT_EQ(e[2]->source().range.begin.line, 1u); + ASSERT_EQ(e[2]->source().range.begin.column, 18u); + ASSERT_EQ(e[2]->source().range.end.line, 1u); + ASSERT_EQ(e[2]->source().range.end.column, 19u); } TEST_F(ParserImplTest, ParamList_Empty) { diff --git a/src/reader/wgsl/parser_impl_struct_member_test.cc b/src/reader/wgsl/parser_impl_struct_member_test.cc index 9ee2d9f38b..d4913c3c10 100644 --- a/src/reader/wgsl/parser_impl_struct_member_test.cc +++ b/src/reader/wgsl/parser_impl_struct_member_test.cc @@ -35,6 +35,11 @@ TEST_F(ParserImplTest, StructMember_Parses) { EXPECT_EQ(m->name(), "a"); EXPECT_EQ(m->type(), i32); EXPECT_EQ(m->decorations().size(), 0u); + + ASSERT_EQ(m->source().range.begin.line, 1u); + ASSERT_EQ(m->source().range.begin.column, 1u); + ASSERT_EQ(m->source().range.end.line, 1u); + ASSERT_EQ(m->source().range.end.column, 2u); } TEST_F(ParserImplTest, StructMember_ParsesWithDecoration) { @@ -50,6 +55,11 @@ TEST_F(ParserImplTest, StructMember_ParsesWithDecoration) { EXPECT_EQ(m->decorations().size(), 1u); EXPECT_TRUE(m->decorations()[0]->IsOffset()); EXPECT_EQ(m->decorations()[0]->AsOffset()->offset(), 2u); + + ASSERT_EQ(m->source().range.begin.line, 1u); + ASSERT_EQ(m->source().range.begin.column, 15u); + ASSERT_EQ(m->source().range.end.line, 1u); + ASSERT_EQ(m->source().range.end.column, 16u); } TEST_F(ParserImplTest, StructMember_ParsesWithMultipleDecorations) { @@ -68,6 +78,11 @@ TEST_F(ParserImplTest, StructMember_ParsesWithMultipleDecorations) { EXPECT_EQ(m->decorations()[0]->AsOffset()->offset(), 2u); EXPECT_TRUE(m->decorations()[1]->IsOffset()); EXPECT_EQ(m->decorations()[1]->AsOffset()->offset(), 4u); + + ASSERT_EQ(m->source().range.begin.line, 2u); + ASSERT_EQ(m->source().range.begin.column, 15u); + ASSERT_EQ(m->source().range.end.line, 2u); + ASSERT_EQ(m->source().range.end.column, 16u); } TEST_F(ParserImplTest, StructMember_InvalidDecoration) { diff --git a/src/reader/wgsl/parser_impl_variable_decl_test.cc b/src/reader/wgsl/parser_impl_variable_decl_test.cc index 2598376de1..97d0115d1d 100644 --- a/src/reader/wgsl/parser_impl_variable_decl_test.cc +++ b/src/reader/wgsl/parser_impl_variable_decl_test.cc @@ -29,9 +29,12 @@ TEST_F(ParserImplTest, VariableDecl_Parses) { ASSERT_NE(var, nullptr); ASSERT_EQ(var->name(), "my_var"); ASSERT_NE(var->type(), nullptr); - ASSERT_EQ(var->source().range.begin.line, 1u); - ASSERT_EQ(var->source().range.begin.column, 1u); ASSERT_TRUE(var->type()->IsF32()); + + ASSERT_EQ(var->source().range.begin.line, 1u); + ASSERT_EQ(var->source().range.begin.column, 5u); + ASSERT_EQ(var->source().range.end.line, 1u); + ASSERT_EQ(var->source().range.end.column, 11u); } TEST_F(ParserImplTest, VariableDecl_MissingVar) { @@ -60,6 +63,11 @@ TEST_F(ParserImplTest, VariableDecl_WithStorageClass) { EXPECT_EQ(v->name(), "my_var"); EXPECT_TRUE(v->type()->IsF32()); EXPECT_EQ(v->storage_class(), ast::StorageClass::kPrivate); + + EXPECT_EQ(v->source().range.begin.line, 1u); + EXPECT_EQ(v->source().range.begin.column, 14u); + EXPECT_EQ(v->source().range.end.line, 1u); + EXPECT_EQ(v->source().range.end.column, 20u); } TEST_F(ParserImplTest, VariableDecl_InvalidStorageClass) { diff --git a/src/reader/wgsl/parser_impl_variable_ident_decl_test.cc b/src/reader/wgsl/parser_impl_variable_ident_decl_test.cc index 2cf0b6b849..dc06863132 100644 --- a/src/reader/wgsl/parser_impl_variable_ident_decl_test.cc +++ b/src/reader/wgsl/parser_impl_variable_ident_decl_test.cc @@ -23,23 +23,24 @@ namespace { TEST_F(ParserImplTest, VariableIdentDecl_Parses) { auto* p = parser("my_var : f32"); - std::string name; - ast::type::Type* type; - std::tie(name, type) = p->variable_ident_decl(); + auto decl = p->variable_ident_decl(); ASSERT_FALSE(p->has_error()); - ASSERT_EQ(name, "my_var"); - ASSERT_NE(type, nullptr); - ASSERT_TRUE(type->IsF32()); + ASSERT_EQ(decl.name, "my_var"); + ASSERT_NE(decl.type, nullptr); + ASSERT_TRUE(decl.type->IsF32()); + + ASSERT_EQ(decl.source.range.begin.line, 1u); + ASSERT_EQ(decl.source.range.begin.column, 1u); + ASSERT_EQ(decl.source.range.end.line, 1u); + ASSERT_EQ(decl.source.range.end.column, 7u); } TEST_F(ParserImplTest, VariableIdentDecl_MissingIdent) { auto* p = parser(": f32"); - std::string name; - ast::type::Type* type; - std::tie(name, type) = p->variable_ident_decl(); + auto decl = p->variable_ident_decl(); ASSERT_FALSE(p->has_error()); - ASSERT_EQ(name, ""); - ASSERT_EQ(type, nullptr); + ASSERT_EQ(decl.name, ""); + ASSERT_EQ(decl.type, nullptr); auto t = p->next(); ASSERT_TRUE(t.IsColon()); @@ -61,12 +62,10 @@ TEST_F(ParserImplTest, VariableIdentDecl_MissingType) { TEST_F(ParserImplTest, VariableIdentDecl_InvalidIdent) { auto* p = parser("123 : f32"); - std::string name; - ast::type::Type* type; - std::tie(name, type) = p->variable_ident_decl(); + auto decl = p->variable_ident_decl(); ASSERT_FALSE(p->has_error()); - ASSERT_EQ(name, ""); - ASSERT_EQ(type, nullptr); + ASSERT_EQ(decl.name, ""); + ASSERT_EQ(decl.type, nullptr); auto t = p->next(); ASSERT_TRUE(t.IsSintLiteral()); diff --git a/src/reader/wgsl/parser_impl_variable_stmt_test.cc b/src/reader/wgsl/parser_impl_variable_stmt_test.cc index 9e3d278828..1330a7a804 100644 --- a/src/reader/wgsl/parser_impl_variable_stmt_test.cc +++ b/src/reader/wgsl/parser_impl_variable_stmt_test.cc @@ -32,6 +32,11 @@ TEST_F(ParserImplTest, VariableStmt_VariableDecl) { ASSERT_NE(e->variable(), nullptr); EXPECT_EQ(e->variable()->name(), "a"); + ASSERT_EQ(e->source().range.begin.line, 1u); + ASSERT_EQ(e->source().range.begin.column, 5u); + ASSERT_EQ(e->source().range.end.line, 1u); + ASSERT_EQ(e->source().range.end.column, 6u); + EXPECT_EQ(e->variable()->constructor(), nullptr); } @@ -44,6 +49,11 @@ TEST_F(ParserImplTest, VariableStmt_VariableDecl_WithInit) { ASSERT_NE(e->variable(), nullptr); EXPECT_EQ(e->variable()->name(), "a"); + ASSERT_EQ(e->source().range.begin.line, 1u); + ASSERT_EQ(e->source().range.begin.column, 5u); + ASSERT_EQ(e->source().range.end.line, 1u); + ASSERT_EQ(e->source().range.end.column, 6u); + ASSERT_NE(e->variable()->constructor(), nullptr); EXPECT_TRUE(e->variable()->constructor()->IsConstructor()); } @@ -70,6 +80,11 @@ TEST_F(ParserImplTest, VariableStmt_Const) { ASSERT_FALSE(p->has_error()) << p->error(); ASSERT_NE(e, nullptr); ASSERT_TRUE(e->IsVariableDecl()); + + ASSERT_EQ(e->source().range.begin.line, 1u); + ASSERT_EQ(e->source().range.begin.column, 7u); + ASSERT_EQ(e->source().range.end.line, 1u); + ASSERT_EQ(e->source().range.end.column, 8u); } TEST_F(ParserImplTest, VariableStmt_Const_InvalidVarIdent) { diff --git a/src/reader/wgsl/token_test.cc b/src/reader/wgsl/token_test.cc index 4911d05d2f..ae56acc4b4 100644 --- a/src/reader/wgsl/token_test.cc +++ b/src/reader/wgsl/token_test.cc @@ -67,9 +67,18 @@ TEST_F(TokenTest, ReturnsMaxU32) { } TEST_F(TokenTest, Source) { - Token t(Token::Type::kUintLiteral, Source{Source::Location{3, 9}}); + Source::File file("", ""); + Source src; + src.file = &file; + src.range.begin = Source::Location{3, 9}; + src.range.end = Source::Location{4, 3}; + + Token t(Token::Type::kUintLiteral, src); EXPECT_EQ(t.source().range.begin.line, 3u); EXPECT_EQ(t.source().range.begin.column, 9u); + EXPECT_EQ(t.source().range.end.line, 4u); + EXPECT_EQ(t.source().range.end.column, 3u); + EXPECT_EQ(t.source().file, &file); } } // namespace