tint: Lex three types of integer literal

Generate different tokens for:
• 'i' suffixed integer literals
• 'u' suffixed integer literals
• no-suffix integer literals

'i' and no-suffix are currently both treated as i32, but this is the
first step to supporting abstract integers.

Bug: tint:1504
Change-Id: Ib94652e0c829d7879ff594ff7efd279cb05010e6
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/88841
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>

src/tint/reader/wgsl/lexer.cc

@@ -20,6 +20,7 @@
 #include <limits>
 #include <optional>  // NOLINT(build/include_order)
 #include <tuple>
+#include <type_traits>
 #include <utility>
 
 #include "src/tint/debug.h"
@@ -79,6 +80,29 @@ uint32_t hex_value(char c) {
     return 0;
 }
 
+/// LimitCheck is the enumerator result of check_limits().
+enum class LimitCheck {
+    /// The value was within the limits of the data type.
+    kWithinLimits,
+    /// The value was too small to fit within the data type.
+    kTooSmall,
+    /// The value was too large to fit within the data type.
+    kTooLarge,
+};
+
+/// Checks whether the value fits within the integer type `T`
+template <typename T>
+LimitCheck check_limits(int64_t value) {
+    static_assert(std::is_integral_v<T>, "T must be an integer");
+    if (value < static_cast<int64_t>(std::numeric_limits<T>::min())) {
+        return LimitCheck::kTooSmall;
+    }
+    if (value > static_cast<int64_t>(std::numeric_limits<T>::max())) {
+        return LimitCheck::kTooLarge;
+    }
+    return LimitCheck::kWithinLimits;
+}
+
 }  // namespace
 
 Lexer::Lexer(const Source::File* file) : file_(file), location_{1, 1} {}
@@ -665,37 +689,49 @@ Token Lexer::build_token_from_int_if_possible(Source source,
                                               size_t start,
                                               size_t end,
                                               int32_t base) {
-    auto res = strtoll(&at(start), nullptr, base);
+    int64_t res = strtoll(&at(start), nullptr, base);
+
+    auto str = [&] { return std::string{substr(start, end - start)}; };
 
     if (matches(pos(), "u")) {
-        if (res < 0) {
+        switch (check_limits<uint32_t>(res)) {
-            return {Token::Type::kError, source,
+            case LimitCheck::kTooSmall:
-                    "u32 (" + std::string{substr(start, end - start)} + ") must not be negative"};
+                return {Token::Type::kError, source, "unsigned literal cannot be negative"};
+            case LimitCheck::kTooLarge:
+                return {Token::Type::kError, source, str() + " too large for u32"};
+            default:
+                advance(1);
+                end_source(source);
+                return {Token::Type::kIntULiteral, source, res};
         }
-        if (static_cast<uint64_t>(res) >
-            static_cast<uint64_t>(std::numeric_limits<uint32_t>::max())) {
-            return {Token::Type::kError, source,
-                    "u32 (" + std::string{substr(start, end - start)} + ") too large"};
-        }
-        advance(1);
-        end_source(source);
-        return {source, static_cast<uint32_t>(res)};
     }
 
     if (matches(pos(), "i")) {
+        switch (check_limits<int32_t>(res)) {
+            case LimitCheck::kTooSmall:
+                return {Token::Type::kError, source, str() + " too small for i32"};
+            case LimitCheck::kTooLarge:
+                return {Token::Type::kError, source, str() + " too large for i32"};
+            default:
+                break;
+        }
         advance(1);
+        end_source(source);
+        return {Token::Type::kIntILiteral, source, res};
     }
 
-    if (res < static_cast<int64_t>(std::numeric_limits<int32_t>::min())) {
+    // TODO(crbug.com/tint/1504): Properly support abstract int:
-        return {Token::Type::kError, source,
+    // Change `AbstractIntType` to `int64_t`, update errors to say 'abstract int'.
-                "i32 (" + std::string{substr(start, end - start)} + ") too small"};
+    using AbstractIntType = int32_t;
+    switch (check_limits<AbstractIntType>(res)) {
+        case LimitCheck::kTooSmall:
+            return {Token::Type::kError, source, str() + " too small for i32"};
+        case LimitCheck::kTooLarge:
+            return {Token::Type::kError, source, str() + " too large for i32"};
+        default:
+            end_source(source);
+            return {Token::Type::kIntLiteral, source, res};
     }
-    if (res > static_cast<int64_t>(std::numeric_limits<int32_t>::max())) {
-        return {Token::Type::kError, source,
-                "i32 (" + std::string{substr(start, end - start)} + ") too large"};
-    }
-    end_source(source);
-    return {source, static_cast<int32_t>(res)};
 }
 
 Token Lexer::try_hex_integer() {

src/tint/reader/wgsl/lexer_test.cc

@@ -588,18 +588,31 @@ inline std::ostream& operator<<(std::ostream& out, HexSignedIntData data) {
 }
 
 using IntegerTest_HexSigned = testing::TestWithParam<HexSignedIntData>;
-TEST_P(IntegerTest_HexSigned, Matches) {
+TEST_P(IntegerTest_HexSigned, NoSuffix) {
     auto params = GetParam();
     Source::File file("", params.input);
     Lexer l(&file);
 
     auto t = l.next();
-    EXPECT_TRUE(t.Is(Token::Type::kSintLiteral));
+    EXPECT_TRUE(t.Is(Token::Type::kIntLiteral));
     EXPECT_EQ(t.source().range.begin.line, 1u);
     EXPECT_EQ(t.source().range.begin.column, 1u);
     EXPECT_EQ(t.source().range.end.line, 1u);
     EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
-    EXPECT_EQ(t.to_i32(), params.result);
+    EXPECT_EQ(t.to_i64(), params.result);
+}
+TEST_P(IntegerTest_HexSigned, ISuffix) {
+    auto params = GetParam();
+    Source::File file("", std::string(params.input) + "i");
+    Lexer l(&file);
+
+    auto t = l.next();
+    EXPECT_TRUE(t.Is(Token::Type::kIntILiteral));
+    EXPECT_EQ(t.source().range.begin.line, 1u);
+    EXPECT_EQ(t.source().range.begin.column, 1u);
+    EXPECT_EQ(t.source().range.end.line, 1u);
+    EXPECT_EQ(t.source().range.end.column, 2u + strlen(params.input));
+    EXPECT_EQ(t.to_i64(), params.result);
 }
 INSTANTIATE_TEST_SUITE_P(
     LexerTest,
@@ -663,7 +676,7 @@ TEST_F(LexerTest, IntegerTest_HexSignedTooLarge) {
 
     auto t = l.next();
     ASSERT_TRUE(t.Is(Token::Type::kError));
-    EXPECT_EQ(t.to_str(), "i32 (0x80000000) too large");
+    EXPECT_EQ(t.to_str(), "0x80000000 too large for i32");
 }
 
 TEST_F(LexerTest, IntegerTest_HexSignedTooSmall) {
@@ -672,7 +685,7 @@ TEST_F(LexerTest, IntegerTest_HexSignedTooSmall) {
 
     auto t = l.next();
     ASSERT_TRUE(t.Is(Token::Type::kError));
-    EXPECT_EQ(t.to_str(), "i32 (-0x8000000F) too small");
+    EXPECT_EQ(t.to_str(), "-0x8000000F too small for i32");
 }
 
 TEST_F(LexerTest, IntegerTest_HexSignedTooManyDigits) {
@@ -703,18 +716,19 @@ inline std::ostream& operator<<(std::ostream& out, HexUnsignedIntData data) {
     return out;
 }
 using IntegerTest_HexUnsigned = testing::TestWithParam<HexUnsignedIntData>;
+// TODO(crbug.com/tint/1504): Split into NoSuffix and USuffix
 TEST_P(IntegerTest_HexUnsigned, Matches) {
     auto params = GetParam();
     Source::File file("", params.input);
     Lexer l(&file);
 
     auto t = l.next();
-    EXPECT_TRUE(t.Is(Token::Type::kUintLiteral));
+    EXPECT_TRUE(t.Is(Token::Type::kIntULiteral));
     EXPECT_EQ(t.source().range.begin.line, 1u);
     EXPECT_EQ(t.source().range.begin.column, 1u);
     EXPECT_EQ(t.source().range.end.line, 1u);
     EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
-    EXPECT_EQ(t.to_u32(), params.result);
+    EXPECT_EQ(t.to_i64(), params.result);
 
     t = l.next();
     EXPECT_TRUE(t.IsEof());
@@ -752,8 +766,8 @@ TEST_P(IntegerTest_Unsigned, Matches) {
     Lexer l(&file);
 
     auto t = l.next();
-    EXPECT_TRUE(t.Is(Token::Type::kUintLiteral));
+    EXPECT_TRUE(t.Is(Token::Type::kIntULiteral));
-    EXPECT_EQ(t.to_u32(), params.result);
+    EXPECT_EQ(t.to_i64(), params.result);
     EXPECT_EQ(t.source().range.begin.line, 1u);
     EXPECT_EQ(t.source().range.begin.column, 1u);
     EXPECT_EQ(t.source().range.end.line, 1u);
@@ -789,8 +803,8 @@ TEST_P(IntegerTest_Signed, Matches) {
     Lexer l(&file);
 
     auto t = l.next();
-    EXPECT_TRUE(t.Is(Token::Type::kSintLiteral));
+    EXPECT_TRUE(t.Is(Token::Type::kIntLiteral));
-    EXPECT_EQ(t.to_i32(), params.result);
+    EXPECT_EQ(t.to_i64(), params.result);
     EXPECT_EQ(t.source().range.begin.line, 1u);
     EXPECT_EQ(t.source().range.begin.column, 1u);
     EXPECT_EQ(t.source().range.end.line, 1u);
@@ -820,8 +834,9 @@ TEST_P(IntegerTest_Invalid, Parses) {
     Lexer l(&file);
 
     auto t = l.next();
-    EXPECT_FALSE(t.Is(Token::Type::kSintLiteral));
+    EXPECT_FALSE(t.Is(Token::Type::kIntLiteral));
-    EXPECT_FALSE(t.Is(Token::Type::kUintLiteral));
+    EXPECT_FALSE(t.Is(Token::Type::kIntULiteral));
+    EXPECT_FALSE(t.Is(Token::Type::kIntILiteral));
 }
 INSTANTIATE_TEST_SUITE_P(
     LexerTest,

src/tint/reader/wgsl/parser_impl.cc

@@ -14,6 +14,8 @@
 
 #include "src/tint/reader/wgsl/parser_impl.h"
 
+#include <limits>
+
 #include "src/tint/ast/array.h"
 #include "src/tint/ast/assignment_statement.h"
 #include "src/tint/ast/bitcast_expression.h"
@@ -2776,21 +2778,24 @@ Maybe<const ast::Statement*> ParserImpl::assignment_stmt() {
 //   | FALSE
 Maybe<const ast::LiteralExpression*> ParserImpl::const_literal() {
     auto t = peek();
+    if (match(Token::Type::kIntLiteral)) {
+        return create<ast::SintLiteralExpression>(t.source(), static_cast<int32_t>(t.to_i64()));
+    }
+    if (match(Token::Type::kIntILiteral)) {
+        return create<ast::SintLiteralExpression>(t.source(), static_cast<int32_t>(t.to_i64()));
+    }
+    if (match(Token::Type::kIntULiteral)) {
+        return create<ast::UintLiteralExpression>(t.source(), static_cast<uint32_t>(t.to_i64()));
+    }
+    if (match(Token::Type::kFloatLiteral)) {
+        return create<ast::FloatLiteralExpression>(t.source(), t.to_f32());
+    }
     if (match(Token::Type::kTrue)) {
         return create<ast::BoolLiteralExpression>(t.source(), true);
     }
     if (match(Token::Type::kFalse)) {
         return create<ast::BoolLiteralExpression>(t.source(), false);
     }
-    if (match(Token::Type::kSintLiteral)) {
-        return create<ast::SintLiteralExpression>(t.source(), t.to_i32());
-    }
-    if (match(Token::Type::kUintLiteral)) {
-        return create<ast::UintLiteralExpression>(t.source(), t.to_u32());
-    }
-    if (match(Token::Type::kFloatLiteral)) {
-        return create<ast::FloatLiteralExpression>(t.source(), t.to_f32());
-    }
     if (handle_error(t)) {
         return Failure::kErrored;
     }
@@ -3119,11 +3124,19 @@ bool ParserImpl::expect(std::string_view use, Token::Type tok) {
 
 Expect<int32_t> ParserImpl::expect_sint(std::string_view use) {
     auto t = peek();
-    if (!t.Is(Token::Type::kSintLiteral))
+    if (!t.Is(Token::Type::kIntLiteral) && !t.Is(Token::Type::kIntILiteral)) {
         return add_error(t.source(), "expected signed integer literal", use);
+    }
+
+    int64_t val = t.to_i64();
+    if ((val > std::numeric_limits<int32_t>::max()) ||
+        (val < std::numeric_limits<int32_t>::min())) {
+        // TODO(crbug.com/tint/1504): Test this when abstract int is implemented
+        return add_error(t.source(), "value overflows i32", use);
+    }
 
     next();
-    return {t.to_i32(), t.source()};
+    return {static_cast<int32_t>(t.to_i64()), t.source()};
 }
 
 Expect<uint32_t> ParserImpl::expect_positive_sint(std::string_view use) {

src/tint/reader/wgsl/parser_impl_const_literal_test.cc

@@ -105,7 +105,7 @@ TEST_F(ParserImplTest, ConstLiteral_Uint_Negative) {
     auto c = p->const_literal();
     EXPECT_FALSE(c.matched);
     EXPECT_TRUE(c.errored);
-    EXPECT_EQ(p->error(), "1:1: u32 (-234) must not be negative");
+    EXPECT_EQ(p->error(), "1:1: unsigned literal cannot be negative");
     ASSERT_EQ(c.value, nullptr);
 }
 

src/tint/reader/wgsl/token.cc

@@ -20,19 +20,21 @@ namespace tint::reader::wgsl {
 std::string_view Token::TypeToName(Type type) {
     switch (type) {
         case Token::Type::kError:
-            return "kError";
+            return "error";
         case Token::Type::kEOF:
-            return "kEOF";
+            return "end of file";
         case Token::Type::kIdentifier:
-            return "kIdentifier";
+            return "identifier";
         case Token::Type::kFloatLiteral:
-            return "kFloatLiteral";
+            return "float literal";
-        case Token::Type::kSintLiteral:
+        case Token::Type::kIntLiteral:
-            return "kSintLiteral";
+            return "abstract integer literal";
-        case Token::Type::kUintLiteral:
+        case Token::Type::kIntILiteral:
-            return "kUintLiteral";
+            return "'i'-suffixed integer literal";
+        case Token::Type::kIntULiteral:
+            return "'u'-suffixed integer literal";
         case Token::Type::kUninitialized:
-            return "kUninitialized";
+            return "uninitialized";
 
         case Token::Type::kAnd:
             return "&";
@@ -273,11 +275,8 @@ Token::Token(Type type, const Source& source, const std::string& str)
 Token::Token(Type type, const Source& source, const char* str)
     : type_(type), source_(source), value_(std::string_view(str)) {}
 
-Token::Token(const Source& source, uint32_t val)
+Token::Token(Type type, const Source& source, int64_t val)
-    : type_(Type::kUintLiteral), source_(source), value_(val) {}
+    : type_(type), source_(source), value_(val) {}
-
-Token::Token(const Source& source, int32_t val)
-    : type_(Type::kSintLiteral), source_(source), value_(val) {}
 
 Token::Token(const Source& source, float val)
     : type_(Type::kFloatLiteral), source_(source), value_(val) {}
@@ -306,10 +305,12 @@ std::string Token::to_str() const {
     switch (type_) {
         case Type::kFloatLiteral:
             return std::to_string(std::get<float>(value_));
-        case Type::kSintLiteral:
+        case Type::kIntLiteral:
-            return std::to_string(std::get<int32_t>(value_));
+            return std::to_string(std::get<int64_t>(value_));
-        case Type::kUintLiteral:
+        case Type::kIntILiteral:
-            return std::to_string(std::get<uint32_t>(value_));
+            return std::to_string(std::get<int64_t>(value_)) + "i";
+        case Type::kIntULiteral:
+            return std::to_string(std::get<int64_t>(value_)) + "u";
         case Type::kIdentifier:
         case Type::kError:
             if (auto* view = std::get_if<std::string_view>(&value_)) {
@@ -325,12 +326,8 @@ float Token::to_f32() const {
     return std::get<float>(value_);
 }
 
-uint32_t Token::to_u32() const {
+int64_t Token::to_i64() const {
-    return std::get<uint32_t>(value_);
+    return std::get<int64_t>(value_);
-}
-
-int32_t Token::to_i32() const {
-    return std::get<int32_t>(value_);
 }
 
 }  // namespace tint::reader::wgsl

src/tint/reader/wgsl/token.h

@@ -40,10 +40,12 @@ class Token {
         kIdentifier,
         /// A float value
         kFloatLiteral,
-        /// An signed int value
+        /// An integer literal with no suffix
-        kSintLiteral,
+        kIntLiteral,
-        /// A unsigned int value
+        /// An integer literal with an 'i' suffix
-        kUintLiteral,
+        kIntILiteral,
+        /// An integer literal with a 'u' suffix
+        kIntULiteral,
 
         /// A '&'
         kAnd,
@@ -297,14 +299,11 @@ class Token {
     /// @param source the source of the token
     /// @param str the source string for the token
     Token(Type type, const Source& source, const char* str);
-    /// Create a unsigned integer Token
+    /// Create a integer Token of the given type
+    /// @param type the Token::Type of the token
     /// @param source the source of the token
     /// @param val the source unsigned for the token
-    Token(const Source& source, uint32_t val);
+    Token(Type type, const Source& source, int64_t val);
-    /// Create a signed integer Token
-    /// @param source the source of the token
-    /// @param val the source integer for the token
-    Token(const Source& source, int32_t val);
     /// Create a float Token
     /// @param source the source of the token
     /// @param val the source float for the token
@@ -340,8 +339,9 @@ class Token {
     bool IsIdentifier() const { return type_ == Type::kIdentifier; }
     /// @returns true if the token is a literal
     bool IsLiteral() const {
-        return type_ == Type::kSintLiteral || type_ == Type::kFalse ||
+        return type_ == Type::kIntLiteral || type_ == Type::kIntILiteral ||
-               type_ == Type::kUintLiteral || type_ == Type::kTrue || type_ == Type::kFloatLiteral;
+               type_ == Type::kIntULiteral || type_ == Type::kFalse || type_ == Type::kTrue ||
+               type_ == Type::kFloatLiteral;
     }
     /// @returns true if token is a 'matNxM'
     bool IsMatrix() const {
@@ -381,14 +381,10 @@ class Token {
     /// contain a float value.
     /// @return float
     float to_f32() const;
-    /// Returns the uint32 value of the token. 0 is returned if the token does not
+    /// Returns the int64_t value of the token. 0 is returned if the token does
-    /// contain a unsigned integer value.
+    /// not contain an integer value.
-    /// @return uint32_t
+    /// @return int64_t
-    uint32_t to_u32() const;
+    int64_t to_i64() const;
-    /// Returns the int32 value of the token. 0 is returned if the token does not
-    /// contain a signed integer value.
-    /// @return int32_t
-    int32_t to_i32() const;
 
     /// @returns the token type as string
     std::string_view to_name() const { return Token::TypeToName(type_); }
@@ -399,7 +395,7 @@ class Token {
     /// The source where the token appeared
     Source source_;
     /// The value represented by the token
-    std::variant<int32_t, uint32_t, float, std::string, std::string_view> value_;
+    std::variant<int64_t, float, std::string, std::string_view> value_;
 };
 
 #ifndef NDEBUG

src/tint/reader/wgsl/token_test.cc

@@ -32,31 +32,31 @@ TEST_F(TokenTest, ReturnsF32) {
 }
 
 TEST_F(TokenTest, ReturnsI32) {
-    Token t1(Source{}, -2345);
+    Token t1(Token::Type::kIntILiteral, Source{}, -2345);
-    EXPECT_EQ(t1.to_i32(), -2345);
+    EXPECT_EQ(t1.to_i64(), -2345);
 
-    Token t2(Source{}, 2345);
+    Token t2(Token::Type::kIntILiteral, Source{}, 2345);
-    EXPECT_EQ(t2.to_i32(), 2345);
+    EXPECT_EQ(t2.to_i64(), 2345);
 }
 
 TEST_F(TokenTest, HandlesMaxI32) {
-    Token t1(Source{}, std::numeric_limits<int32_t>::max());
+    Token t1(Token::Type::kIntILiteral, Source{}, std::numeric_limits<int32_t>::max());
-    EXPECT_EQ(t1.to_i32(), std::numeric_limits<int32_t>::max());
+    EXPECT_EQ(t1.to_i64(), std::numeric_limits<int32_t>::max());
 }
 
 TEST_F(TokenTest, HandlesMinI32) {
-    Token t1(Source{}, std::numeric_limits<int32_t>::min());
+    Token t1(Token::Type::kIntILiteral, Source{}, std::numeric_limits<int32_t>::min());
-    EXPECT_EQ(t1.to_i32(), std::numeric_limits<int32_t>::min());
+    EXPECT_EQ(t1.to_i64(), std::numeric_limits<int32_t>::min());
 }
 
 TEST_F(TokenTest, ReturnsU32) {
-    Token t2(Source{}, 2345u);
+    Token t2(Token::Type::kIntULiteral, Source{}, 2345u);
-    EXPECT_EQ(t2.to_u32(), 2345u);
+    EXPECT_EQ(t2.to_i64(), 2345u);
 }
 
 TEST_F(TokenTest, ReturnsMaxU32) {
-    Token t1(Source{}, std::numeric_limits<uint32_t>::max());
+    Token t1(Token::Type::kIntULiteral, Source{}, std::numeric_limits<uint32_t>::max());
-    EXPECT_EQ(t1.to_u32(), std::numeric_limits<uint32_t>::max());
+    EXPECT_EQ(t1.to_i64(), std::numeric_limits<uint32_t>::max());
 }
 
 TEST_F(TokenTest, Source) {
@@ -64,7 +64,7 @@ TEST_F(TokenTest, Source) {
     src.range.begin = Source::Location{3, 9};
     src.range.end = Source::Location{4, 3};
 
-    Token t(Token::Type::kUintLiteral, src);
+    Token t(Token::Type::kIntLiteral, src);
     EXPECT_EQ(t.source().range.begin.line, 3u);
     EXPECT_EQ(t.source().range.begin.column, 9u);
     EXPECT_EQ(t.source().range.end.line, 4u);