From 72260e44ae284602e0239d692a1136c044131d6b Mon Sep 17 00:00:00 2001 From: dan sinclair Date: Wed, 24 Feb 2021 18:56:34 +0000 Subject: [PATCH] Remove parsing of StringLiteral. The StringLiteral token was removed from the spec when all usage was removed. This CL removes the remaining parsing bits from Tint. Change-Id: I02f5dbdbad649c62c22c69a55616e0087a0f56d4 Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/42440 Auto-Submit: dan sinclair Commit-Queue: Ryan Harrison Reviewed-by: Ryan Harrison --- src/reader/wgsl/lexer.cc | 28 ------------------ src/reader/wgsl/lexer.h | 1 - src/reader/wgsl/lexer_test.cc | 53 ----------------------------------- src/reader/wgsl/token.cc | 2 -- src/reader/wgsl/token.h | 4 --- src/reader/wgsl/token_test.cc | 5 ---- 6 files changed, 93 deletions(-) diff --git a/src/reader/wgsl/lexer.cc b/src/reader/wgsl/lexer.cc index e919279448..bdee707525 100644 --- a/src/reader/wgsl/lexer.cc +++ b/src/reader/wgsl/lexer.cc @@ -63,11 +63,6 @@ Token Lexer::next() { return t; } - t = try_string(); - if (!t.IsUninitialized()) { - return t; - } - t = try_punctuation(); if (!t.IsUninitialized()) { return t; @@ -334,29 +329,6 @@ Token Lexer::try_ident() { return {Token::Type::kIdentifier, source, str}; } -Token Lexer::try_string() { - if (!matches(pos_, R"(")")) - return {}; - - auto source = begin_source(); - - pos_++; - auto start = pos_; - while (pos_ < len_ && !matches(pos_, R"(")")) { - pos_++; - } - auto end = pos_; - if (matches(pos_, R"(")")) { - pos_++; - } - location_.column += (pos_ - start) + 1; - - end_source(source); - - return {Token::Type::kStringLiteral, source, - content_->data.substr(start, end - start)}; -} - Token Lexer::try_punctuation() { auto source = begin_source(); auto type = Token::Type::kUninitialized; diff --git a/src/reader/wgsl/lexer.h b/src/reader/wgsl/lexer.h index 1fd550a2d2..b7a6d1f34c 100644 --- a/src/reader/wgsl/lexer.h +++ b/src/reader/wgsl/lexer.h @@ -52,7 +52,6 @@ class Lexer { Token try_ident(); Token try_integer(); Token try_punctuation(); - Token try_string(); Source begin_source() const; void end_source(Source&) const; diff --git a/src/reader/wgsl/lexer_test.cc b/src/reader/wgsl/lexer_test.cc index 8652c02448..5d01df1c8a 100644 --- a/src/reader/wgsl/lexer_test.cc +++ b/src/reader/wgsl/lexer_test.cc @@ -75,59 +75,6 @@ ident1 //ends with comment EXPECT_TRUE(t.IsEof()); } -TEST_F(LexerTest, StringTest_Parse) { - Source::FileContent content(R"(id "this is string content" id2)"); - Lexer l("test.wgsl", &content); - - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.to_str(), "id"); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 1u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 3u); - - t = l.next(); - EXPECT_TRUE(t.IsStringLiteral()); - EXPECT_EQ(t.to_str(), "this is string content"); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 4u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 28u); - - t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.to_str(), "id2"); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 29u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 32u); -} - -TEST_F(LexerTest, StringTest_Unterminated) { - Source::FileContent content(R"(id "this is string content)"); - Lexer l("test.wgsl", &content); - - auto t = l.next(); - EXPECT_TRUE(t.IsIdentifier()); - EXPECT_EQ(t.to_str(), "id"); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 1u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 3u); - - t = l.next(); - EXPECT_TRUE(t.IsStringLiteral()); - EXPECT_EQ(t.to_str(), "this is string content"); - EXPECT_EQ(t.source().range.begin.line, 1u); - EXPECT_EQ(t.source().range.begin.column, 4u); - EXPECT_EQ(t.source().range.end.line, 1u); - EXPECT_EQ(t.source().range.end.column, 27u); - - t = l.next(); - EXPECT_TRUE(t.IsEof()); -} - struct FloatData { const char* input; float result; diff --git a/src/reader/wgsl/token.cc b/src/reader/wgsl/token.cc index 69274a03cf..9f8b28d5c6 100644 --- a/src/reader/wgsl/token.cc +++ b/src/reader/wgsl/token.cc @@ -29,8 +29,6 @@ std::string Token::TypeToName(Type type) { return "kEOF"; case Token::Type::kIdentifier: return "kIdentifier"; - case Token::Type::kStringLiteral: - return "kStringLiteral"; case Token::Type::kFloatLiteral: return "kFloatLiteral"; case Token::Type::kSintLiteral: diff --git a/src/reader/wgsl/token.h b/src/reader/wgsl/token.h index 0ce8aab2ef..e27edd9246 100644 --- a/src/reader/wgsl/token.h +++ b/src/reader/wgsl/token.h @@ -42,8 +42,6 @@ class Token { /// An identifier kIdentifier, - /// A string value - kStringLiteral, /// A float value kFloatLiteral, /// An signed int value @@ -381,8 +379,6 @@ class Token { bool IsEof() const { return type_ == Type::kEOF; } /// @returns true if the token is an identifier bool IsIdentifier() const { return type_ == Type::kIdentifier; } - /// @returns true if the token is a string - bool IsStringLiteral() const { return type_ == Type::kStringLiteral; } /// @returns true if the token is a float bool IsFloatLiteral() const { return type_ == Type::kFloatLiteral; } /// @returns true if the token is an signed int diff --git a/src/reader/wgsl/token_test.cc b/src/reader/wgsl/token_test.cc index 24552f3067..d67a547a97 100644 --- a/src/reader/wgsl/token_test.cc +++ b/src/reader/wgsl/token_test.cc @@ -25,11 +25,6 @@ namespace { using TokenTest = testing::Test; -TEST_F(TokenTest, ReturnsStr) { - Token t(Token::Type::kStringLiteral, Source{}, "test string"); - EXPECT_EQ(t.to_str(), "test string"); -} - TEST_F(TokenTest, ReturnsF32) { Token t1(Source{}, -2.345f); EXPECT_EQ(t1.to_f32(), -2.345f);