encounter/dawn-cmake
1
0
Fork 0
mirror of https://github.com/encounter/dawn-cmake.git synced 2025-05-16 20:31:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

tint/reader/wgsl/lexer: Make '>>=' splittable

Browse Source 
Could be split when matching template argument lists.

Bug: tint:1810
Change-Id: I34421d2a27a8b2294fa0201793e7340c0f172df7
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/117209
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
This commit is contained in:
Ben Clayton 2023-01-23 18:32:41 +00:00 committed by Dawn LUCI CQ
parent 5f3449f000
commit 6fc130bfcc
3 changed files with 30 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/tint/reader/wgsl/lexer.cc
View File

@ -96,9 +96,10 @@ std::vector<Token> Lexer::Lex() {
while (true) { while (true) {
tokens.emplace_back(next()); tokens.emplace_back(next());
// If the token can be split, we insert a placeholder element into // If the token can be split, we insert a placeholder element(s) into the stream to hold the
// the stream to hold the split character. // split character.
if (tokens.back().IsSplittable()) { size_t num_placeholders = tokens.back().NumPlaceholders();
for (size_t i = 0; i < num_placeholders; i++) {
auto src = tokens.back().source(); auto src = tokens.back().source();
src.range.begin.column++; src.range.begin.column++;
tokens.emplace_back(Token(Token::Type::kPlaceholder, src)); tokens.emplace_back(Token(Token::Type::kPlaceholder, src));

19
src/tint/reader/wgsl/lexer_test.cc
View File

@ -989,7 +989,6 @@ INSTANTIATE_TEST_SUITE_P(LexerTest,
TokenData{"&=", Token::Type::kAndEqual}, TokenData{"&=", Token::Type::kAndEqual},
TokenData{"|=", Token::Type::kOrEqual}, TokenData{"|=", Token::Type::kOrEqual},
TokenData{"^=", Token::Type::kXorEqual}, TokenData{"^=", Token::Type::kXorEqual},
TokenData{">>=", Token::Type::kShiftRightEqual},
TokenData{"<<=", Token::Type::kShiftLeftEqual})); TokenData{"<<=", Token::Type::kShiftLeftEqual}));
using SplittablePunctuationTest = testing::TestWithParam<TokenData>; using SplittablePunctuationTest = testing::TestWithParam<TokenData>;
@ -1010,18 +1009,23 @@ TEST_P(SplittablePunctuationTest, Parses) {
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
} }
{ const size_t num_placeholders = list[0].NumPlaceholders();
auto& t = list[1]; EXPECT_GT(num_placeholders, 0u);
ASSERT_EQ(list.size(), 2u + num_placeholders);
for (size_t i = 0; i < num_placeholders; i++) {
auto& t = list[1 + i];
EXPECT_TRUE(t.Is(Token::Type::kPlaceholder)); EXPECT_TRUE(t.Is(Token::Type::kPlaceholder));
EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 2u); EXPECT_EQ(t.source().range.begin.column, 2u + i);
EXPECT_EQ(t.source().range.end.line, 1u); EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
} }
{ {
auto& t = list[2]; auto& t = list.back();
EXPECT_EQ(t.source().range.begin.column, 1 + std::string(params.input).size()); EXPECT_TRUE(t.Is(Token::Type::kEOF));
EXPECT_EQ(t.source().range.begin.column, 2u + num_placeholders);
} }
} }
INSTANTIATE_TEST_SUITE_P(LexerTest, INSTANTIATE_TEST_SUITE_P(LexerTest,
@ -1029,7 +1033,8 @@ INSTANTIATE_TEST_SUITE_P(LexerTest,
testing::Values(TokenData{"&&", Token::Type::kAndAnd}, testing::Values(TokenData{"&&", Token::Type::kAndAnd},
TokenData{">=", Token::Type::kGreaterThanEqual}, TokenData{">=", Token::Type::kGreaterThanEqual},
TokenData{"--", Token::Type::kMinusMinus}, TokenData{"--", Token::Type::kMinusMinus},
TokenData{">>", Token::Type::kShiftRight})); TokenData{">>", Token::Type::kShiftRight},
TokenData{">>=", Token::Type::kShiftRightEqual}));
using KeywordTest = testing::TestWithParam<TokenData>; using KeywordTest = testing::TestWithParam<TokenData>;
TEST_P(KeywordTest, Parses) { TEST_P(KeywordTest, Parses) {

18
src/tint/reader/wgsl/token.h
View File

@ -379,10 +379,20 @@ class Token {
return type_ == Type::kVec2 || type_ == Type::kVec3 || type_ == Type::kVec4; return type_ == Type::kVec2 || type_ == Type::kVec3 || type_ == Type::kVec4;
} }
/// @returns true if the token can be split during parse into component tokens /// @returns the number of placeholder tokens required to follow the token, in order to provide
bool IsSplittable() const { /// space for token splitting.
return Is(Type::kShiftRight) || Is(Type::kGreaterThanEqual) || Is(Type::kAndAnd) || size_t NumPlaceholders() const {
Is(Type::kMinusMinus); switch (type_) {
case Type::kShiftRightEqual:
return 2;
case Type::kShiftRight:
case Type::kGreaterThanEqual:
case Type::kAndAnd:
case Type::kMinusMinus:
return 1;
default:
return 0;
}
} }
/// @returns true if the token is a binary operator /// @returns true if the token is a binary operator