tint/reader/wgsl: Lex abstract floats

And remove lexer errors about float magnitudes been too small.

Also add tests for non-hex float literal overflow.

Bug: tint:1504
Bug: tint:1564
Change-Id: Ia26817d4f2a99af694e9935692b98ef91f97d2b3
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/91428
Reviewed-by: David Neto <dneto@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Ben Clayton <bclayton@chromium.org>

src/tint/reader/wgsl/lexer.cc

@@ -358,31 +358,22 @@ Token Lexer::try_float() {
     advance(end - start);
     end_source(source);
 
-    double value = strtod(&at(start), nullptr);
+    double value = std::strtod(&at(start), nullptr);
 
     if (has_f_suffix) {
         if (auto f = CheckedConvert<f32>(AFloat(value))) {
-            return {Token::Type::kFloatLiteral_F, source, value};
+            return {Token::Type::kFloatLiteral_F, source, static_cast<double>(f.Get())};
+        } else if (f.Failure() == ConversionFailure::kTooSmall) {
+            return {Token::Type::kFloatLiteral_F, source, 0.0};
         } else {
-            if (f.Failure() == ConversionFailure::kTooSmall) {
-                return {Token::Type::kError, source,
-                        "value magnitude too small to be represented as 'f32'"};
-            }
             return {Token::Type::kError, source, "value cannot be represented as 'f32'"};
         }
     }
 
-    // TODO(crbug.com/tint/1504): Properly support abstract float:
+    if (value == HUGE_VAL || -value == HUGE_VAL) {
-    // Change `AbstractFloatType` to `double`, update errors to say 'abstract int'.
+        return {Token::Type::kError, source, "value cannot be represented as 'abstract-float'"};
-    using AbstractFloatType = f32;
-    if (auto f = CheckedConvert<AbstractFloatType>(AFloat(value))) {
-        return {Token::Type::kFloatLiteral, source, value};
     } else {
-        if (f.Failure() == ConversionFailure::kTooSmall) {
+        return {Token::Type::kFloatLiteral, source, value};
-            return {Token::Type::kError, source,
-                    "value magnitude too small to be represented as 'f32'"};
-        }
-        return {Token::Type::kError, source, "value cannot be represented as 'f32'"};
     }
 }
 

src/tint/reader/wgsl/lexer_test.cc

@@ -362,12 +362,12 @@ INSTANTIATE_TEST_SUITE_P(LexerTest,
                              FloatData{"-5.", -5.},
                              FloatData{"-.7", -.7},
                              // Non-zero with decimal and 'f' suffix
-                             FloatData{"5.7f", 5.7},
+                             FloatData{"5.7f", static_cast<double>(5.7f)},
-                             FloatData{"5.f", 5.},
+                             FloatData{"5.f", static_cast<double>(5.f)},
-                             FloatData{".7f", .7},
+                             FloatData{".7f", static_cast<double>(.7f)},
-                             FloatData{"-5.7f", -5.7},
+                             FloatData{"-5.7f", static_cast<double>(-5.7f)},
-                             FloatData{"-5.f", -5.},
+                             FloatData{"-5.f", static_cast<double>(-5.f)},
-                             FloatData{"-.7f", -.7},
+                             FloatData{"-.7f", static_cast<double>(-.7f)},
 
                              // No decimal, with exponent
                              FloatData{"1e5", 1e5},
@@ -375,10 +375,10 @@ INSTANTIATE_TEST_SUITE_P(LexerTest,
                              FloatData{"1e-5", 1e-5},
                              FloatData{"1E-5", 1e-5},
                              // No decimal, with exponent and 'f' suffix
-                             FloatData{"1e5f", 1e5},
+                             FloatData{"1e5f", static_cast<double>(1e5f)},
-                             FloatData{"1E5f", 1e5},
+                             FloatData{"1E5f", static_cast<double>(1e5f)},
-                             FloatData{"1e-5f", 1e-5},
+                             FloatData{"1e-5f", static_cast<double>(1e-5f)},
-                             FloatData{"1E-5f", 1e-5},
+                             FloatData{"1E-5f", static_cast<double>(1e-5f)},
                              // With decimal and exponents
                              FloatData{"0.2e+12", 0.2e12},
                              FloatData{"1.2e-5", 1.2e-5},
@@ -386,11 +386,15 @@ INSTANTIATE_TEST_SUITE_P(LexerTest,
                              FloatData{"2.5e+0", 2.5},
                              FloatData{"2.5e-0", 2.5},
                              // With decimal and exponents and 'f' suffix
-                             FloatData{"0.2e+12f", 0.2e12},
+                             FloatData{"0.2e+12f", static_cast<double>(0.2e12f)},
-                             FloatData{"1.2e-5f", 1.2e-5},
+                             FloatData{"1.2e-5f", static_cast<double>(1.2e-5f)},
-                             FloatData{"2.57e23f", 2.57e23},
+                             FloatData{"2.57e23f", static_cast<double>(2.57e23f)},
-                             FloatData{"2.5e+0f", 2.5},
+                             FloatData{"2.5e+0f", static_cast<double>(2.5f)},
-                             FloatData{"2.5e-0f", 2.5}));
+                             FloatData{"2.5e-0f", static_cast<double>(2.5f)},
+                             // Quantization
+                             FloatData{"3.141592653589793", 3.141592653589793},   // no quantization
+                             FloatData{"3.141592653589793f", 3.1415927410125732}  // f32 quantized
+                             ));
 
 using FloatTest_Invalid = testing::TestWithParam<const char*>;
 TEST_P(FloatTest_Invalid, Handles) {
@@ -415,11 +419,11 @@ INSTANTIATE_TEST_SUITE_P(LexerTest,
                                          ".e+",
                                          ".e-",
                                          // Overflow
-                                         "2.5e+256",
+                                         "2.5e+256f",
-                                         "-2.5e+127",
+                                         "-2.5e+127f",
                                          // Magnitude smaller than smallest positive f32.
-                                         "2.5e-300",
+                                         "2.5e-300f",
-                                         "-2.5e-300",
+                                         "-2.5e-300f",
                                          // Decimal exponent must immediately
                                          // follow the 'e'.
                                          "2.5e 12",

src/tint/reader/wgsl/parser_impl_const_literal_test.cc

@@ -117,34 +117,6 @@ TEST_F(ParserImplTest, ConstLiteral_Uint_Negative) {
     ASSERT_EQ(c.value, nullptr);
 }
 
-TEST_F(ParserImplTest, ConstLiteral_Float) {
-    auto p = parser("234.e12");
-    auto c = p->const_literal();
-    EXPECT_TRUE(c.matched);
-    EXPECT_FALSE(c.errored);
-    EXPECT_FALSE(p->has_error()) << p->error();
-    ASSERT_NE(c.value, nullptr);
-    ASSERT_TRUE(c->Is<ast::FloatLiteralExpression>());
-    EXPECT_DOUBLE_EQ(c->As<ast::FloatLiteralExpression>()->value, 234e12);
-    EXPECT_EQ(c->As<ast::FloatLiteralExpression>()->suffix,
-              ast::FloatLiteralExpression::Suffix::kNone);
-    EXPECT_EQ(c->source.range, (Source::Range{{1u, 1u}, {1u, 8u}}));
-}
-
-TEST_F(ParserImplTest, ConstLiteral_FloatF) {
-    auto p = parser("234.e12f");
-    auto c = p->const_literal();
-    EXPECT_TRUE(c.matched);
-    EXPECT_FALSE(c.errored);
-    EXPECT_FALSE(p->has_error()) << p->error();
-    ASSERT_NE(c.value, nullptr);
-    ASSERT_TRUE(c->Is<ast::FloatLiteralExpression>());
-    EXPECT_DOUBLE_EQ(c->As<ast::FloatLiteralExpression>()->value, 234e12);
-    EXPECT_EQ(c->As<ast::FloatLiteralExpression>()->suffix,
-              ast::FloatLiteralExpression::Suffix::kF);
-    EXPECT_EQ(c->source.range, (Source::Range{{1u, 1u}, {1u, 9u}}));
-}
-
 TEST_F(ParserImplTest, ConstLiteral_InvalidFloat_IncompleteExponent) {
     auto p = parser("1.0e+");
     auto c = p->const_literal();
@@ -154,33 +126,6 @@ TEST_F(ParserImplTest, ConstLiteral_InvalidFloat_IncompleteExponent) {
     ASSERT_EQ(c.value, nullptr);
 }
 
-TEST_F(ParserImplTest, ConstLiteral_InvalidFloat_TooSmallMagnitude) {
-    auto p = parser("1e-256");
-    auto c = p->const_literal();
-    EXPECT_FALSE(c.matched);
-    EXPECT_TRUE(c.errored);
-    EXPECT_EQ(p->error(), "1:1: value magnitude too small to be represented as 'f32'");
-    ASSERT_EQ(c.value, nullptr);
-}
-
-TEST_F(ParserImplTest, ConstLiteral_InvalidFloat_TooLargeNegative) {
-    auto p = parser("-1.2e+256");
-    auto c = p->const_literal();
-    EXPECT_FALSE(c.matched);
-    EXPECT_TRUE(c.errored);
-    EXPECT_EQ(p->error(), "1:1: value cannot be represented as 'f32'");
-    ASSERT_EQ(c.value, nullptr);
-}
-
-TEST_F(ParserImplTest, ConstLiteral_InvalidFloat_TooLargePositive) {
-    auto p = parser("1.2e+256");
-    auto c = p->const_literal();
-    EXPECT_FALSE(c.matched);
-    EXPECT_TRUE(c.errored);
-    EXPECT_EQ(p->error(), "1:1: value cannot be represented as 'f32'");
-    ASSERT_EQ(c.value, nullptr);
-}
-
 struct FloatLiteralTestCase {
     std::string input;
     double expected;
@@ -217,26 +162,54 @@ TEST_P(ParserImplFloatLiteralTest, Parse) {
         EXPECT_EQ(c->As<ast::FloatLiteralExpression>()->suffix,
                   ast::FloatLiteralExpression::Suffix::kNone);
     }
+    EXPECT_EQ(c->source.range, (Source::Range{{1u, 1u}, {1u, 1u + params.input.size()}}));
 }
 using FloatLiteralTestCaseList = std::vector<FloatLiteralTestCase>;
 
-FloatLiteralTestCaseList DecimalFloatCases() {
-    return FloatLiteralTestCaseList{
-        {"0.0", 0.0},                        // Zero
-        {"1.0", 1.0},                        // One
-        {"-1.0", -1.0},                      // MinusOne
-        {"1000000000.0", 1e9},               // Billion
-        {"-0.0", std::copysign(0.0, -5.0)},  // NegativeZero
-        {"0.0", MakeDouble(0, 0, 0)},        // Zero
-        {"-0.0", MakeDouble(1, 0, 0)},       // NegativeZero
-        {"1.0", MakeDouble(0, 1023, 0)},     // One
-        {"-1.0", MakeDouble(1, 1023, 0)},    // NegativeOne
-    };
-}
-
 INSTANTIATE_TEST_SUITE_P(ParserImplFloatLiteralTest_Float,
                          ParserImplFloatLiteralTest,
-                         testing::ValuesIn(DecimalFloatCases()));
+                         testing::ValuesIn(FloatLiteralTestCaseList{
+                             {"0.0", 0.0},                        // Zero
+                             {"1.0", 1.0},                        // One
+                             {"-1.0", -1.0},                      // MinusOne
+                             {"1000000000.0", 1e9},               // Billion
+                             {"-0.0", std::copysign(0.0, -5.0)},  // NegativeZero
+                             {"0.0", MakeDouble(0, 0, 0)},        // Zero
+                             {"-0.0", MakeDouble(1, 0, 0)},       // NegativeZero
+                             {"1.0", MakeDouble(0, 1023, 0)},     // One
+                             {"-1.0", MakeDouble(1, 1023, 0)},    // NegativeOne
+
+                             {"234.e12", 234.e12},
+                             {"234.e12f", static_cast<double>(234.e12f)},
+
+                             // Tiny cases
+                             {"1e-5000", 0.0},
+                             {"-1e-5000", 0.0},
+                             {"1e-5000f", 0.0},
+                             {"-1e-5000f", 0.0},
+                             {"1e-50f", 0.0},
+                             {"-1e-50f", 0.0},
+
+                             // Nearly overflow
+                             {"1.e308", 1.e308},
+                             {"-1.e308", -1.e308},
+                             {"1.8e307", 1.8e307},
+                             {"-1.8e307", -1.8e307},
+                             {"1.798e307", 1.798e307},
+                             {"-1.798e307", -1.798e307},
+                             {"1.7977e307", 1.7977e307},
+                             {"-1.7977e307", -1.7977e307},
+
+                             // Nearly overflow
+                             {"1e38f", static_cast<double>(1e38f)},
+                             {"-1e38f", static_cast<double>(-1e38f)},
+                             {"4.0e37f", static_cast<double>(4.0e37f)},
+                             {"-4.0e37f", static_cast<double>(-4.0e37f)},
+                             {"3.5e37f", static_cast<double>(3.5e37f)},
+                             {"-3.5e37f", static_cast<double>(-3.5e37f)},
+                             {"3.403e37f", static_cast<double>(3.403e37f)},
+                             {"-3.403e37f", static_cast<double>(-3.403e37f)},
+                         }));
 
 const double NegInf = MakeDouble(1, 0x7FF, 0);
 const double PosInf = MakeDouble(0, 0x7FF, 0);
@@ -535,7 +508,7 @@ INSTANTIATE_TEST_SUITE_P(
                      })));
 
 INSTANTIATE_TEST_SUITE_P(
-    NaNAFloat,
+    HexNaNAFloat,
     ParserImplInvalidLiteralTest,
     testing::Combine(testing::Values("1:1: value cannot be represented as 'abstract-float'"),
                      testing::ValuesIn(std::vector<const char*>{
@@ -552,7 +525,7 @@ INSTANTIATE_TEST_SUITE_P(
                      })));
 
 INSTANTIATE_TEST_SUITE_P(
-    NaNF32,
+    HexNaNF32,
     ParserImplInvalidLiteralTest,
     testing::Combine(testing::Values("1:1: value cannot be represented as 'f32'"),
                      testing::ValuesIn(std::vector<const char*>{
@@ -569,7 +542,7 @@ INSTANTIATE_TEST_SUITE_P(
                      })));
 
 INSTANTIATE_TEST_SUITE_P(
-    OverflowAFloat,
+    HexOverflowAFloat,
     ParserImplInvalidLiteralTest,
     testing::Combine(testing::Values("1:1: value cannot be represented as 'abstract-float'"),
                      testing::ValuesIn(std::vector<const char*>{
@@ -588,7 +561,7 @@ INSTANTIATE_TEST_SUITE_P(
                      })));
 
 INSTANTIATE_TEST_SUITE_P(
-    OverflowF32,
+    HexOverflowF32,
     ParserImplInvalidLiteralTest,
     testing::Combine(testing::Values("1:1: value cannot be represented as 'f32'"),
                      testing::ValuesIn(std::vector<const char*>{
@@ -604,6 +577,40 @@ INSTANTIATE_TEST_SUITE_P(
                          "-0x32p+500f",
                      })));
 
+INSTANTIATE_TEST_SUITE_P(
+    DecOverflowAFloat,
+    ParserImplInvalidLiteralTest,
+    testing::Combine(testing::Values("1:1: value cannot be represented as 'abstract-float'"),
+                     testing::ValuesIn(std::vector<const char*>{
+                         "1.e309",
+                         "-1.e309",
+                         "1.8e308",
+                         "-1.8e308",
+                         "1.798e308",
+                         "-1.798e308",
+                         "1.7977e308",
+                         "-1.7977e308",
+                         "1.2e+5000",
+                         "-1.2e+5000",
+                     })));
+
+INSTANTIATE_TEST_SUITE_P(
+    DecOverflowF32,
+    ParserImplInvalidLiteralTest,
+    testing::Combine(testing::Values("1:1: value cannot be represented as 'f32'"),
+                     testing::ValuesIn(std::vector<const char*>{
+                         "1e39f",
+                         "-1e39f",
+                         "4.0e38f",
+                         "-4.0e38f",
+                         "3.5e38f",
+                         "-3.5e38f",
+                         "3.403e38f",
+                         "-3.403e38f",
+                         "1.2e+256f",
+                         "-1.2e+256f",
+                     })));
+
 TEST_F(ParserImplTest, ConstLiteral_FloatHighest) {
     const auto highest = std::numeric_limits<float>::max();
     const auto expected_highest = 340282346638528859811704183484516925440.0f;