diff --git a/src/tint/reader/wgsl/lexer.cc b/src/tint/reader/wgsl/lexer.cc index ea5aeb30de..d9a9f34540 100644 --- a/src/tint/reader/wgsl/lexer.cc +++ b/src/tint/reader/wgsl/lexer.cc @@ -20,6 +20,7 @@ #include #include // NOLINT(build/include_order) #include +#include #include #include "src/tint/debug.h" @@ -79,6 +80,29 @@ uint32_t hex_value(char c) { return 0; } +/// LimitCheck is the enumerator result of check_limits(). +enum class LimitCheck { + /// The value was within the limits of the data type. + kWithinLimits, + /// The value was too small to fit within the data type. + kTooSmall, + /// The value was too large to fit within the data type. + kTooLarge, +}; + +/// Checks whether the value fits within the integer type `T` +template +LimitCheck check_limits(int64_t value) { + static_assert(std::is_integral_v, "T must be an integer"); + if (value < static_cast(std::numeric_limits::min())) { + return LimitCheck::kTooSmall; + } + if (value > static_cast(std::numeric_limits::max())) { + return LimitCheck::kTooLarge; + } + return LimitCheck::kWithinLimits; +} + } // namespace Lexer::Lexer(const Source::File* file) : file_(file), location_{1, 1} {} @@ -665,37 +689,49 @@ Token Lexer::build_token_from_int_if_possible(Source source, size_t start, size_t end, int32_t base) { - auto res = strtoll(&at(start), nullptr, base); + int64_t res = strtoll(&at(start), nullptr, base); + + auto str = [&] { return std::string{substr(start, end - start)}; }; if (matches(pos(), "u")) { - if (res < 0) { - return {Token::Type::kError, source, - "u32 (" + std::string{substr(start, end - start)} + ") must not be negative"}; + switch (check_limits(res)) { + case LimitCheck::kTooSmall: + return {Token::Type::kError, source, "unsigned literal cannot be negative"}; + case LimitCheck::kTooLarge: + return {Token::Type::kError, source, str() + " too large for u32"}; + default: + advance(1); + end_source(source); + return {Token::Type::kIntULiteral, source, res}; } - if (static_cast(res) > - static_cast(std::numeric_limits::max())) { - return {Token::Type::kError, source, - "u32 (" + std::string{substr(start, end - start)} + ") too large"}; - } - advance(1); - end_source(source); - return {source, static_cast(res)}; } if (matches(pos(), "i")) { + switch (check_limits(res)) { + case LimitCheck::kTooSmall: + return {Token::Type::kError, source, str() + " too small for i32"}; + case LimitCheck::kTooLarge: + return {Token::Type::kError, source, str() + " too large for i32"}; + default: + break; + } advance(1); + end_source(source); + return {Token::Type::kIntILiteral, source, res}; } - if (res < static_cast(std::numeric_limits::min())) { - return {Token::Type::kError, source, - "i32 (" + std::string{substr(start, end - start)} + ") too small"}; + // TODO(crbug.com/tint/1504): Properly support abstract int: + // Change `AbstractIntType` to `int64_t`, update errors to say 'abstract int'. + using AbstractIntType = int32_t; + switch (check_limits(res)) { + case LimitCheck::kTooSmall: + return {Token::Type::kError, source, str() + " too small for i32"}; + case LimitCheck::kTooLarge: + return {Token::Type::kError, source, str() + " too large for i32"}; + default: + end_source(source); + return {Token::Type::kIntLiteral, source, res}; } - if (res > static_cast(std::numeric_limits::max())) { - return {Token::Type::kError, source, - "i32 (" + std::string{substr(start, end - start)} + ") too large"}; - } - end_source(source); - return {source, static_cast(res)}; } Token Lexer::try_hex_integer() { diff --git a/src/tint/reader/wgsl/lexer_test.cc b/src/tint/reader/wgsl/lexer_test.cc index 0a875e3841..6bcea15fc8 100644 --- a/src/tint/reader/wgsl/lexer_test.cc +++ b/src/tint/reader/wgsl/lexer_test.cc @@ -588,18 +588,31 @@ inline std::ostream& operator<<(std::ostream& out, HexSignedIntData data) { } using IntegerTest_HexSigned = testing::TestWithParam; -TEST_P(IntegerTest_HexSigned, Matches) { +TEST_P(IntegerTest_HexSigned, NoSuffix) { auto params = GetParam(); Source::File file("", params.input); Lexer l(&file); auto t = l.next(); - EXPECT_TRUE(t.Is(Token::Type::kSintLiteral)); + EXPECT_TRUE(t.Is(Token::Type::kIntLiteral)); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); EXPECT_EQ(t.source().range.end.line, 1u); EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); - EXPECT_EQ(t.to_i32(), params.result); + EXPECT_EQ(t.to_i64(), params.result); +} +TEST_P(IntegerTest_HexSigned, ISuffix) { + auto params = GetParam(); + Source::File file("", std::string(params.input) + "i"); + Lexer l(&file); + + auto t = l.next(); + EXPECT_TRUE(t.Is(Token::Type::kIntILiteral)); + EXPECT_EQ(t.source().range.begin.line, 1u); + EXPECT_EQ(t.source().range.begin.column, 1u); + EXPECT_EQ(t.source().range.end.line, 1u); + EXPECT_EQ(t.source().range.end.column, 2u + strlen(params.input)); + EXPECT_EQ(t.to_i64(), params.result); } INSTANTIATE_TEST_SUITE_P( LexerTest, @@ -663,7 +676,7 @@ TEST_F(LexerTest, IntegerTest_HexSignedTooLarge) { auto t = l.next(); ASSERT_TRUE(t.Is(Token::Type::kError)); - EXPECT_EQ(t.to_str(), "i32 (0x80000000) too large"); + EXPECT_EQ(t.to_str(), "0x80000000 too large for i32"); } TEST_F(LexerTest, IntegerTest_HexSignedTooSmall) { @@ -672,7 +685,7 @@ TEST_F(LexerTest, IntegerTest_HexSignedTooSmall) { auto t = l.next(); ASSERT_TRUE(t.Is(Token::Type::kError)); - EXPECT_EQ(t.to_str(), "i32 (-0x8000000F) too small"); + EXPECT_EQ(t.to_str(), "-0x8000000F too small for i32"); } TEST_F(LexerTest, IntegerTest_HexSignedTooManyDigits) { @@ -703,18 +716,19 @@ inline std::ostream& operator<<(std::ostream& out, HexUnsignedIntData data) { return out; } using IntegerTest_HexUnsigned = testing::TestWithParam; +// TODO(crbug.com/tint/1504): Split into NoSuffix and USuffix TEST_P(IntegerTest_HexUnsigned, Matches) { auto params = GetParam(); Source::File file("", params.input); Lexer l(&file); auto t = l.next(); - EXPECT_TRUE(t.Is(Token::Type::kUintLiteral)); + EXPECT_TRUE(t.Is(Token::Type::kIntULiteral)); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); EXPECT_EQ(t.source().range.end.line, 1u); EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input)); - EXPECT_EQ(t.to_u32(), params.result); + EXPECT_EQ(t.to_i64(), params.result); t = l.next(); EXPECT_TRUE(t.IsEof()); @@ -752,8 +766,8 @@ TEST_P(IntegerTest_Unsigned, Matches) { Lexer l(&file); auto t = l.next(); - EXPECT_TRUE(t.Is(Token::Type::kUintLiteral)); - EXPECT_EQ(t.to_u32(), params.result); + EXPECT_TRUE(t.Is(Token::Type::kIntULiteral)); + EXPECT_EQ(t.to_i64(), params.result); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); EXPECT_EQ(t.source().range.end.line, 1u); @@ -789,8 +803,8 @@ TEST_P(IntegerTest_Signed, Matches) { Lexer l(&file); auto t = l.next(); - EXPECT_TRUE(t.Is(Token::Type::kSintLiteral)); - EXPECT_EQ(t.to_i32(), params.result); + EXPECT_TRUE(t.Is(Token::Type::kIntLiteral)); + EXPECT_EQ(t.to_i64(), params.result); EXPECT_EQ(t.source().range.begin.line, 1u); EXPECT_EQ(t.source().range.begin.column, 1u); EXPECT_EQ(t.source().range.end.line, 1u); @@ -820,8 +834,9 @@ TEST_P(IntegerTest_Invalid, Parses) { Lexer l(&file); auto t = l.next(); - EXPECT_FALSE(t.Is(Token::Type::kSintLiteral)); - EXPECT_FALSE(t.Is(Token::Type::kUintLiteral)); + EXPECT_FALSE(t.Is(Token::Type::kIntLiteral)); + EXPECT_FALSE(t.Is(Token::Type::kIntULiteral)); + EXPECT_FALSE(t.Is(Token::Type::kIntILiteral)); } INSTANTIATE_TEST_SUITE_P( LexerTest, diff --git a/src/tint/reader/wgsl/parser_impl.cc b/src/tint/reader/wgsl/parser_impl.cc index a2ca2ec826..9f2539b5dc 100644 --- a/src/tint/reader/wgsl/parser_impl.cc +++ b/src/tint/reader/wgsl/parser_impl.cc @@ -14,6 +14,8 @@ #include "src/tint/reader/wgsl/parser_impl.h" +#include + #include "src/tint/ast/array.h" #include "src/tint/ast/assignment_statement.h" #include "src/tint/ast/bitcast_expression.h" @@ -2776,21 +2778,24 @@ Maybe ParserImpl::assignment_stmt() { // | FALSE Maybe ParserImpl::const_literal() { auto t = peek(); + if (match(Token::Type::kIntLiteral)) { + return create(t.source(), static_cast(t.to_i64())); + } + if (match(Token::Type::kIntILiteral)) { + return create(t.source(), static_cast(t.to_i64())); + } + if (match(Token::Type::kIntULiteral)) { + return create(t.source(), static_cast(t.to_i64())); + } + if (match(Token::Type::kFloatLiteral)) { + return create(t.source(), t.to_f32()); + } if (match(Token::Type::kTrue)) { return create(t.source(), true); } if (match(Token::Type::kFalse)) { return create(t.source(), false); } - if (match(Token::Type::kSintLiteral)) { - return create(t.source(), t.to_i32()); - } - if (match(Token::Type::kUintLiteral)) { - return create(t.source(), t.to_u32()); - } - if (match(Token::Type::kFloatLiteral)) { - return create(t.source(), t.to_f32()); - } if (handle_error(t)) { return Failure::kErrored; } @@ -3119,11 +3124,19 @@ bool ParserImpl::expect(std::string_view use, Token::Type tok) { Expect ParserImpl::expect_sint(std::string_view use) { auto t = peek(); - if (!t.Is(Token::Type::kSintLiteral)) + if (!t.Is(Token::Type::kIntLiteral) && !t.Is(Token::Type::kIntILiteral)) { return add_error(t.source(), "expected signed integer literal", use); + } + + int64_t val = t.to_i64(); + if ((val > std::numeric_limits::max()) || + (val < std::numeric_limits::min())) { + // TODO(crbug.com/tint/1504): Test this when abstract int is implemented + return add_error(t.source(), "value overflows i32", use); + } next(); - return {t.to_i32(), t.source()}; + return {static_cast(t.to_i64()), t.source()}; } Expect ParserImpl::expect_positive_sint(std::string_view use) { diff --git a/src/tint/reader/wgsl/parser_impl_const_literal_test.cc b/src/tint/reader/wgsl/parser_impl_const_literal_test.cc index 45efce2911..03e9919469 100644 --- a/src/tint/reader/wgsl/parser_impl_const_literal_test.cc +++ b/src/tint/reader/wgsl/parser_impl_const_literal_test.cc @@ -105,7 +105,7 @@ TEST_F(ParserImplTest, ConstLiteral_Uint_Negative) { auto c = p->const_literal(); EXPECT_FALSE(c.matched); EXPECT_TRUE(c.errored); - EXPECT_EQ(p->error(), "1:1: u32 (-234) must not be negative"); + EXPECT_EQ(p->error(), "1:1: unsigned literal cannot be negative"); ASSERT_EQ(c.value, nullptr); } diff --git a/src/tint/reader/wgsl/token.cc b/src/tint/reader/wgsl/token.cc index cefa415e57..2f51a85b21 100644 --- a/src/tint/reader/wgsl/token.cc +++ b/src/tint/reader/wgsl/token.cc @@ -20,19 +20,21 @@ namespace tint::reader::wgsl { std::string_view Token::TypeToName(Type type) { switch (type) { case Token::Type::kError: - return "kError"; + return "error"; case Token::Type::kEOF: - return "kEOF"; + return "end of file"; case Token::Type::kIdentifier: - return "kIdentifier"; + return "identifier"; case Token::Type::kFloatLiteral: - return "kFloatLiteral"; - case Token::Type::kSintLiteral: - return "kSintLiteral"; - case Token::Type::kUintLiteral: - return "kUintLiteral"; + return "float literal"; + case Token::Type::kIntLiteral: + return "abstract integer literal"; + case Token::Type::kIntILiteral: + return "'i'-suffixed integer literal"; + case Token::Type::kIntULiteral: + return "'u'-suffixed integer literal"; case Token::Type::kUninitialized: - return "kUninitialized"; + return "uninitialized"; case Token::Type::kAnd: return "&"; @@ -273,11 +275,8 @@ Token::Token(Type type, const Source& source, const std::string& str) Token::Token(Type type, const Source& source, const char* str) : type_(type), source_(source), value_(std::string_view(str)) {} -Token::Token(const Source& source, uint32_t val) - : type_(Type::kUintLiteral), source_(source), value_(val) {} - -Token::Token(const Source& source, int32_t val) - : type_(Type::kSintLiteral), source_(source), value_(val) {} +Token::Token(Type type, const Source& source, int64_t val) + : type_(type), source_(source), value_(val) {} Token::Token(const Source& source, float val) : type_(Type::kFloatLiteral), source_(source), value_(val) {} @@ -306,10 +305,12 @@ std::string Token::to_str() const { switch (type_) { case Type::kFloatLiteral: return std::to_string(std::get(value_)); - case Type::kSintLiteral: - return std::to_string(std::get(value_)); - case Type::kUintLiteral: - return std::to_string(std::get(value_)); + case Type::kIntLiteral: + return std::to_string(std::get(value_)); + case Type::kIntILiteral: + return std::to_string(std::get(value_)) + "i"; + case Type::kIntULiteral: + return std::to_string(std::get(value_)) + "u"; case Type::kIdentifier: case Type::kError: if (auto* view = std::get_if(&value_)) { @@ -325,12 +326,8 @@ float Token::to_f32() const { return std::get(value_); } -uint32_t Token::to_u32() const { - return std::get(value_); -} - -int32_t Token::to_i32() const { - return std::get(value_); +int64_t Token::to_i64() const { + return std::get(value_); } } // namespace tint::reader::wgsl diff --git a/src/tint/reader/wgsl/token.h b/src/tint/reader/wgsl/token.h index 26dc0c1a50..106d1a6ab9 100644 --- a/src/tint/reader/wgsl/token.h +++ b/src/tint/reader/wgsl/token.h @@ -40,10 +40,12 @@ class Token { kIdentifier, /// A float value kFloatLiteral, - /// An signed int value - kSintLiteral, - /// A unsigned int value - kUintLiteral, + /// An integer literal with no suffix + kIntLiteral, + /// An integer literal with an 'i' suffix + kIntILiteral, + /// An integer literal with a 'u' suffix + kIntULiteral, /// A '&' kAnd, @@ -297,14 +299,11 @@ class Token { /// @param source the source of the token /// @param str the source string for the token Token(Type type, const Source& source, const char* str); - /// Create a unsigned integer Token + /// Create a integer Token of the given type + /// @param type the Token::Type of the token /// @param source the source of the token /// @param val the source unsigned for the token - Token(const Source& source, uint32_t val); - /// Create a signed integer Token - /// @param source the source of the token - /// @param val the source integer for the token - Token(const Source& source, int32_t val); + Token(Type type, const Source& source, int64_t val); /// Create a float Token /// @param source the source of the token /// @param val the source float for the token @@ -340,8 +339,9 @@ class Token { bool IsIdentifier() const { return type_ == Type::kIdentifier; } /// @returns true if the token is a literal bool IsLiteral() const { - return type_ == Type::kSintLiteral || type_ == Type::kFalse || - type_ == Type::kUintLiteral || type_ == Type::kTrue || type_ == Type::kFloatLiteral; + return type_ == Type::kIntLiteral || type_ == Type::kIntILiteral || + type_ == Type::kIntULiteral || type_ == Type::kFalse || type_ == Type::kTrue || + type_ == Type::kFloatLiteral; } /// @returns true if token is a 'matNxM' bool IsMatrix() const { @@ -381,14 +381,10 @@ class Token { /// contain a float value. /// @return float float to_f32() const; - /// Returns the uint32 value of the token. 0 is returned if the token does not - /// contain a unsigned integer value. - /// @return uint32_t - uint32_t to_u32() const; - /// Returns the int32 value of the token. 0 is returned if the token does not - /// contain a signed integer value. - /// @return int32_t - int32_t to_i32() const; + /// Returns the int64_t value of the token. 0 is returned if the token does + /// not contain an integer value. + /// @return int64_t + int64_t to_i64() const; /// @returns the token type as string std::string_view to_name() const { return Token::TypeToName(type_); } @@ -399,7 +395,7 @@ class Token { /// The source where the token appeared Source source_; /// The value represented by the token - std::variant value_; + std::variant value_; }; #ifndef NDEBUG diff --git a/src/tint/reader/wgsl/token_test.cc b/src/tint/reader/wgsl/token_test.cc index 48477057e2..dd46616974 100644 --- a/src/tint/reader/wgsl/token_test.cc +++ b/src/tint/reader/wgsl/token_test.cc @@ -32,31 +32,31 @@ TEST_F(TokenTest, ReturnsF32) { } TEST_F(TokenTest, ReturnsI32) { - Token t1(Source{}, -2345); - EXPECT_EQ(t1.to_i32(), -2345); + Token t1(Token::Type::kIntILiteral, Source{}, -2345); + EXPECT_EQ(t1.to_i64(), -2345); - Token t2(Source{}, 2345); - EXPECT_EQ(t2.to_i32(), 2345); + Token t2(Token::Type::kIntILiteral, Source{}, 2345); + EXPECT_EQ(t2.to_i64(), 2345); } TEST_F(TokenTest, HandlesMaxI32) { - Token t1(Source{}, std::numeric_limits::max()); - EXPECT_EQ(t1.to_i32(), std::numeric_limits::max()); + Token t1(Token::Type::kIntILiteral, Source{}, std::numeric_limits::max()); + EXPECT_EQ(t1.to_i64(), std::numeric_limits::max()); } TEST_F(TokenTest, HandlesMinI32) { - Token t1(Source{}, std::numeric_limits::min()); - EXPECT_EQ(t1.to_i32(), std::numeric_limits::min()); + Token t1(Token::Type::kIntILiteral, Source{}, std::numeric_limits::min()); + EXPECT_EQ(t1.to_i64(), std::numeric_limits::min()); } TEST_F(TokenTest, ReturnsU32) { - Token t2(Source{}, 2345u); - EXPECT_EQ(t2.to_u32(), 2345u); + Token t2(Token::Type::kIntULiteral, Source{}, 2345u); + EXPECT_EQ(t2.to_i64(), 2345u); } TEST_F(TokenTest, ReturnsMaxU32) { - Token t1(Source{}, std::numeric_limits::max()); - EXPECT_EQ(t1.to_u32(), std::numeric_limits::max()); + Token t1(Token::Type::kIntULiteral, Source{}, std::numeric_limits::max()); + EXPECT_EQ(t1.to_i64(), std::numeric_limits::max()); } TEST_F(TokenTest, Source) { @@ -64,7 +64,7 @@ TEST_F(TokenTest, Source) { src.range.begin = Source::Location{3, 9}; src.range.end = Source::Location{4, 3}; - Token t(Token::Type::kUintLiteral, src); + Token t(Token::Type::kIntLiteral, src); EXPECT_EQ(t.source().range.begin.line, 3u); EXPECT_EQ(t.source().range.begin.column, 9u); EXPECT_EQ(t.source().range.end.line, 4u);