encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tint: Lex three types of integer literal 

Generate different tokens for:
• 'i' suffixed integer literals
• 'u' suffixed integer literals
• no-suffix integer literals

'i' and no-suffix are currently both treated as i32, but this is the
first step to supporting abstract integers.

Bug: tint:1504
Change-Id: Ib94652e0c829d7879ff594ff7efd279cb05010e6
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/88841
Commit-Queue: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
This commit is contained in:
Ben Clayton 2022-05-04 21:01:12 +00:00 committed by Dawn LUCI CQ
parent 085fcea6b7
commit f693488bff
7 changed files with 161 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
src/tint/reader/wgsl/lexer.cc
View File

@ -20,6 +20,7 @@
#include <limits>
#include <optional> // NOLINT(build/include_order)
#include <tuple>
#include <type_traits>
#include <utility>
#include "src/tint/debug.h"
@ -79,6 +80,29 @@ uint32_t hex_value(char c) {
return 0;
}
/// LimitCheck is the enumerator result of check_limits().
enum class LimitCheck {
/// The value was within the limits of the data type.
kWithinLimits,
/// The value was too small to fit within the data type.
kTooSmall,
/// The value was too large to fit within the data type.
kTooLarge,
};
/// Checks whether the value fits within the integer type `T`
template <typename T>
LimitCheck check_limits(int64_t value) {
static_assert(std::is_integral_v<T>, "T must be an integer");
if (value < static_cast<int64_t>(std::numeric_limits<T>::min())) {
return LimitCheck::kTooSmall;
}
if (value > static_cast<int64_t>(std::numeric_limits<T>::max())) {
return LimitCheck::kTooLarge;
}
return LimitCheck::kWithinLimits;
}
} // namespace
Lexer::Lexer(const Source::File* file) : file_(file), location_{1, 1} {}
@ -665,37 +689,49 @@ Token Lexer::build_token_from_int_if_possible(Source source,
size_t start,
size_t end,
int32_t base) {
auto res = strtoll(&at(start), nullptr, base);
int64_t res = strtoll(&at(start), nullptr, base);
auto str = [&] { return std::string{substr(start, end - start)}; };
if (matches(pos(), "u")) {
if (res < 0) {
return {Token::Type::kError, source,
"u32 (" + std::string{substr(start, end - start)} + ") must not be negative"};
}
if (static_cast<uint64_t>(res) >
static_cast<uint64_t>(std::numeric_limits<uint32_t>::max())) {
return {Token::Type::kError, source,
"u32 (" + std::string{substr(start, end - start)} + ") too large"};
}
switch (check_limits<uint32_t>(res)) {
case LimitCheck::kTooSmall:
return {Token::Type::kError, source, "unsigned literal cannot be negative"};
case LimitCheck::kTooLarge:
return {Token::Type::kError, source, str() + " too large for u32"};
default:
advance(1);
end_source(source);
return {source, static_cast<uint32_t>(res)};
return {Token::Type::kIntULiteral, source, res};
}
}
if (matches(pos(), "i")) {
switch (check_limits<int32_t>(res)) {
case LimitCheck::kTooSmall:
return {Token::Type::kError, source, str() + " too small for i32"};
case LimitCheck::kTooLarge:
return {Token::Type::kError, source, str() + " too large for i32"};
default:
break;
}
advance(1);
end_source(source);
return {Token::Type::kIntILiteral, source, res};
}
if (res < static_cast<int64_t>(std::numeric_limits<int32_t>::min())) {
return {Token::Type::kError, source,
"i32 (" + std::string{substr(start, end - start)} + ") too small"};
}
if (res > static_cast<int64_t>(std::numeric_limits<int32_t>::max())) {
return {Token::Type::kError, source,
"i32 (" + std::string{substr(start, end - start)} + ") too large"};
}
// TODO(crbug.com/tint/1504): Properly support abstract int:
// Change `AbstractIntType` to `int64_t`, update errors to say 'abstract int'.
using AbstractIntType = int32_t;
switch (check_limits<AbstractIntType>(res)) {
case LimitCheck::kTooSmall:
return {Token::Type::kError, source, str() + " too small for i32"};
case LimitCheck::kTooLarge:
return {Token::Type::kError, source, str() + " too large for i32"};
default:
end_source(source);
return {source, static_cast<int32_t>(res)};
return {Token::Type::kIntLiteral, source, res};
}
}
Token Lexer::try_hex_integer() {

41
src/tint/reader/wgsl/lexer_test.cc
View File

@ -588,18 +588,31 @@ inline std::ostream& operator<<(std::ostream& out, HexSignedIntData data) {
}
using IntegerTest_HexSigned = testing::TestWithParam<HexSignedIntData>;
TEST_P(IntegerTest_HexSigned, Matches) {
TEST_P(IntegerTest_HexSigned, NoSuffix) {
auto params = GetParam();
Source::File file("", params.input);
Lexer l(&file);
auto t = l.next();
EXPECT_TRUE(t.Is(Token::Type::kSintLiteral));
EXPECT_TRUE(t.Is(Token::Type::kIntLiteral));
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
EXPECT_EQ(t.to_i32(), params.result);
EXPECT_EQ(t.to_i64(), params.result);
}
TEST_P(IntegerTest_HexSigned, ISuffix) {
auto params = GetParam();
Source::File file("", std::string(params.input) + "i");
Lexer l(&file);
auto t = l.next();
EXPECT_TRUE(t.Is(Token::Type::kIntILiteral));
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 2u + strlen(params.input));
EXPECT_EQ(t.to_i64(), params.result);
}
INSTANTIATE_TEST_SUITE_P(
LexerTest,
@ -663,7 +676,7 @@ TEST_F(LexerTest, IntegerTest_HexSignedTooLarge) {
auto t = l.next();
ASSERT_TRUE(t.Is(Token::Type::kError));
EXPECT_EQ(t.to_str(), "i32 (0x80000000) too large");
EXPECT_EQ(t.to_str(), "0x80000000 too large for i32");
}
TEST_F(LexerTest, IntegerTest_HexSignedTooSmall) {
@ -672,7 +685,7 @@ TEST_F(LexerTest, IntegerTest_HexSignedTooSmall) {
auto t = l.next();
ASSERT_TRUE(t.Is(Token::Type::kError));
EXPECT_EQ(t.to_str(), "i32 (-0x8000000F) too small");
EXPECT_EQ(t.to_str(), "-0x8000000F too small for i32");
}
TEST_F(LexerTest, IntegerTest_HexSignedTooManyDigits) {
@ -703,18 +716,19 @@ inline std::ostream& operator<<(std::ostream& out, HexUnsignedIntData data) {
return out;
}
using IntegerTest_HexUnsigned = testing::TestWithParam<HexUnsignedIntData>;
// TODO(crbug.com/tint/1504): Split into NoSuffix and USuffix
TEST_P(IntegerTest_HexUnsigned, Matches) {
auto params = GetParam();
Source::File file("", params.input);
Lexer l(&file);
auto t = l.next();
EXPECT_TRUE(t.Is(Token::Type::kUintLiteral));
EXPECT_TRUE(t.Is(Token::Type::kIntULiteral));
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
EXPECT_EQ(t.source().range.end.column, 1u + strlen(params.input));
EXPECT_EQ(t.to_u32(), params.result);
EXPECT_EQ(t.to_i64(), params.result);
t = l.next();
EXPECT_TRUE(t.IsEof());
@ -752,8 +766,8 @@ TEST_P(IntegerTest_Unsigned, Matches) {
Lexer l(&file);
auto t = l.next();
EXPECT_TRUE(t.Is(Token::Type::kUintLiteral));
EXPECT_EQ(t.to_u32(), params.result);
EXPECT_TRUE(t.Is(Token::Type::kIntULiteral));
EXPECT_EQ(t.to_i64(), params.result);
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
@ -789,8 +803,8 @@ TEST_P(IntegerTest_Signed, Matches) {
Lexer l(&file);
auto t = l.next();
EXPECT_TRUE(t.Is(Token::Type::kSintLiteral));
EXPECT_EQ(t.to_i32(), params.result);
EXPECT_TRUE(t.Is(Token::Type::kIntLiteral));
EXPECT_EQ(t.to_i64(), params.result);
EXPECT_EQ(t.source().range.begin.line, 1u);
EXPECT_EQ(t.source().range.begin.column, 1u);
EXPECT_EQ(t.source().range.end.line, 1u);
@ -820,8 +834,9 @@ TEST_P(IntegerTest_Invalid, Parses) {
Lexer l(&file);
auto t = l.next();
EXPECT_FALSE(t.Is(Token::Type::kSintLiteral));
EXPECT_FALSE(t.Is(Token::Type::kUintLiteral));
EXPECT_FALSE(t.Is(Token::Type::kIntLiteral));
EXPECT_FALSE(t.Is(Token::Type::kIntULiteral));
EXPECT_FALSE(t.Is(Token::Type::kIntILiteral));
}
INSTANTIATE_TEST_SUITE_P(
LexerTest,

35
src/tint/reader/wgsl/parser_impl.cc
View File

@ -14,6 +14,8 @@
#include "src/tint/reader/wgsl/parser_impl.h"
#include <limits>
#include "src/tint/ast/array.h"
#include "src/tint/ast/assignment_statement.h"
#include "src/tint/ast/bitcast_expression.h"
@ -2776,21 +2778,24 @@ Maybe<const ast::Statement*> ParserImpl::assignment_stmt() {
// | FALSE
Maybe<const ast::LiteralExpression*> ParserImpl::const_literal() {
auto t = peek();
if (match(Token::Type::kIntLiteral)) {
return create<ast::SintLiteralExpression>(t.source(), static_cast<int32_t>(t.to_i64()));
}
if (match(Token::Type::kIntILiteral)) {
return create<ast::SintLiteralExpression>(t.source(), static_cast<int32_t>(t.to_i64()));
}
if (match(Token::Type::kIntULiteral)) {
return create<ast::UintLiteralExpression>(t.source(), static_cast<uint32_t>(t.to_i64()));
}
if (match(Token::Type::kFloatLiteral)) {
return create<ast::FloatLiteralExpression>(t.source(), t.to_f32());
}
if (match(Token::Type::kTrue)) {
return create<ast::BoolLiteralExpression>(t.source(), true);
}
if (match(Token::Type::kFalse)) {
return create<ast::BoolLiteralExpression>(t.source(), false);
}
if (match(Token::Type::kSintLiteral)) {
return create<ast::SintLiteralExpression>(t.source(), t.to_i32());
}
if (match(Token::Type::kUintLiteral)) {
return create<ast::UintLiteralExpression>(t.source(), t.to_u32());
}
if (match(Token::Type::kFloatLiteral)) {
return create<ast::FloatLiteralExpression>(t.source(), t.to_f32());
}
if (handle_error(t)) {
return Failure::kErrored;
}
@ -3119,11 +3124,19 @@ bool ParserImpl::expect(std::string_view use, Token::Type tok) {
Expect<int32_t> ParserImpl::expect_sint(std::string_view use) {
auto t = peek();
if (!t.Is(Token::Type::kSintLiteral))
if (!t.Is(Token::Type::kIntLiteral) && !t.Is(Token::Type::kIntILiteral)) {
return add_error(t.source(), "expected signed integer literal", use);
}
int64_t val = t.to_i64();
if ((val > std::numeric_limits<int32_t>::max()) ||
(val < std::numeric_limits<int32_t>::min())) {
// TODO(crbug.com/tint/1504): Test this when abstract int is implemented
return add_error(t.source(), "value overflows i32", use);
}
next();
return {t.to_i32(), t.source()};
return {static_cast<int32_t>(t.to_i64()), t.source()};
}
Expect<uint32_t> ParserImpl::expect_positive_sint(std::string_view use) {

2
src/tint/reader/wgsl/parser_impl_const_literal_test.cc
View File

@ -105,7 +105,7 @@ TEST_F(ParserImplTest, ConstLiteral_Uint_Negative) {
auto c = p->const_literal();
EXPECT_FALSE(c.matched);
EXPECT_TRUE(c.errored);
EXPECT_EQ(p->error(), "1:1: u32 (-234) must not be negative");
EXPECT_EQ(p->error(), "1:1: unsigned literal cannot be negative");
ASSERT_EQ(c.value, nullptr);
}

45
src/tint/reader/wgsl/token.cc
View File

@ -20,19 +20,21 @@ namespace tint::reader::wgsl {
std::string_view Token::TypeToName(Type type) {
switch (type) {
case Token::Type::kError:
return "kError";
return "error";
case Token::Type::kEOF:
return "kEOF";
return "end of file";
case Token::Type::kIdentifier:
return "kIdentifier";
return "identifier";
case Token::Type::kFloatLiteral:
return "kFloatLiteral";
case Token::Type::kSintLiteral:
return "kSintLiteral";
case Token::Type::kUintLiteral:
return "kUintLiteral";
return "float literal";
case Token::Type::kIntLiteral:
return "abstract integer literal";
case Token::Type::kIntILiteral:
return "'i'-suffixed integer literal";
case Token::Type::kIntULiteral:
return "'u'-suffixed integer literal";
case Token::Type::kUninitialized:
return "kUninitialized";
return "uninitialized";
case Token::Type::kAnd:
return "&";
@ -273,11 +275,8 @@ Token::Token(Type type, const Source& source, const std::string& str)
Token::Token(Type type, const Source& source, const char* str)
: type_(type), source_(source), value_(std::string_view(str)) {}
Token::Token(const Source& source, uint32_t val)
: type_(Type::kUintLiteral), source_(source), value_(val) {}
Token::Token(const Source& source, int32_t val)
: type_(Type::kSintLiteral), source_(source), value_(val) {}
Token::Token(Type type, const Source& source, int64_t val)
: type_(type), source_(source), value_(val) {}
Token::Token(const Source& source, float val)
: type_(Type::kFloatLiteral), source_(source), value_(val) {}
@ -306,10 +305,12 @@ std::string Token::to_str() const {
switch (type_) {
case Type::kFloatLiteral:
return std::to_string(std::get<float>(value_));
case Type::kSintLiteral:
return std::to_string(std::get<int32_t>(value_));
case Type::kUintLiteral:
return std::to_string(std::get<uint32_t>(value_));
case Type::kIntLiteral:
return std::to_string(std::get<int64_t>(value_));
case Type::kIntILiteral:
return std::to_string(std::get<int64_t>(value_)) + "i";
case Type::kIntULiteral:
return std::to_string(std::get<int64_t>(value_)) + "u";
case Type::kIdentifier:
case Type::kError:
if (auto* view = std::get_if<std::string_view>(&value_)) {
@ -325,12 +326,8 @@ float Token::to_f32() const {
return std::get<float>(value_);
}
uint32_t Token::to_u32() const {
return std::get<uint32_t>(value_);
}
int32_t Token::to_i32() const {
return std::get<int32_t>(value_);
int64_t Token::to_i64() const {
return std::get<int64_t>(value_);
}
} // namespace tint::reader::wgsl

38
src/tint/reader/wgsl/token.h
View File

@ -40,10 +40,12 @@ class Token {
kIdentifier,
/// A float value
kFloatLiteral,
/// An signed int value
kSintLiteral,
/// A unsigned int value
kUintLiteral,
/// An integer literal with no suffix
kIntLiteral,
/// An integer literal with an 'i' suffix
kIntILiteral,
/// An integer literal with a 'u' suffix
kIntULiteral,
/// A '&'
kAnd,
@ -297,14 +299,11 @@ class Token {
/// @param source the source of the token
/// @param str the source string for the token
Token(Type type, const Source& source, const char* str);
/// Create a unsigned integer Token
/// Create a integer Token of the given type
/// @param type the Token::Type of the token
/// @param source the source of the token
/// @param val the source unsigned for the token
Token(const Source& source, uint32_t val);
/// Create a signed integer Token
/// @param source the source of the token
/// @param val the source integer for the token
Token(const Source& source, int32_t val);
Token(Type type, const Source& source, int64_t val);
/// Create a float Token
/// @param source the source of the token
/// @param val the source float for the token
@ -340,8 +339,9 @@ class Token {
bool IsIdentifier() const { return type_ == Type::kIdentifier; }
/// @returns true if the token is a literal
bool IsLiteral() const {
return type_ == Type::kSintLiteral || type_ == Type::kFalse ||
type_ == Type::kUintLiteral || type_ == Type::kTrue || type_ == Type::kFloatLiteral;
return type_ == Type::kIntLiteral || type_ == Type::kIntILiteral ||
type_ == Type::kIntULiteral || type_ == Type::kFalse || type_ == Type::kTrue ||
type_ == Type::kFloatLiteral;
}
/// @returns true if token is a 'matNxM'
bool IsMatrix() const {
@ -381,14 +381,10 @@ class Token {
/// contain a float value.
/// @return float
float to_f32() const;
/// Returns the uint32 value of the token. 0 is returned if the token does not
/// contain a unsigned integer value.
/// @return uint32_t
uint32_t to_u32() const;
/// Returns the int32 value of the token. 0 is returned if the token does not
/// contain a signed integer value.
/// @return int32_t
int32_t to_i32() const;
/// Returns the int64_t value of the token. 0 is returned if the token does
/// not contain an integer value.
/// @return int64_t
int64_t to_i64() const;
/// @returns the token type as string
std::string_view to_name() const { return Token::TypeToName(type_); }
@ -399,7 +395,7 @@ class Token {
/// The source where the token appeared
Source source_;
/// The value represented by the token
std::variant<int32_t, uint32_t, float, std::string, std::string_view> value_;
std::variant<int64_t, float, std::string, std::string_view> value_;
};
#ifndef NDEBUG

26
src/tint/reader/wgsl/token_test.cc
View File

@ -32,31 +32,31 @@ TEST_F(TokenTest, ReturnsF32) {
}
TEST_F(TokenTest, ReturnsI32) {
Token t1(Source{}, -2345);
EXPECT_EQ(t1.to_i32(), -2345);
Token t1(Token::Type::kIntILiteral, Source{}, -2345);
EXPECT_EQ(t1.to_i64(), -2345);
Token t2(Source{}, 2345);
EXPECT_EQ(t2.to_i32(), 2345);
Token t2(Token::Type::kIntILiteral, Source{}, 2345);
EXPECT_EQ(t2.to_i64(), 2345);
}
TEST_F(TokenTest, HandlesMaxI32) {
Token t1(Source{}, std::numeric_limits<int32_t>::max());
EXPECT_EQ(t1.to_i32(), std::numeric_limits<int32_t>::max());
Token t1(Token::Type::kIntILiteral, Source{}, std::numeric_limits<int32_t>::max());
EXPECT_EQ(t1.to_i64(), std::numeric_limits<int32_t>::max());
}
TEST_F(TokenTest, HandlesMinI32) {
Token t1(Source{}, std::numeric_limits<int32_t>::min());
EXPECT_EQ(t1.to_i32(), std::numeric_limits<int32_t>::min());
Token t1(Token::Type::kIntILiteral, Source{}, std::numeric_limits<int32_t>::min());
EXPECT_EQ(t1.to_i64(), std::numeric_limits<int32_t>::min());
}
TEST_F(TokenTest, ReturnsU32) {
Token t2(Source{}, 2345u);
EXPECT_EQ(t2.to_u32(), 2345u);
Token t2(Token::Type::kIntULiteral, Source{}, 2345u);
EXPECT_EQ(t2.to_i64(), 2345u);
}
TEST_F(TokenTest, ReturnsMaxU32) {
Token t1(Source{}, std::numeric_limits<uint32_t>::max());
EXPECT_EQ(t1.to_u32(), std::numeric_limits<uint32_t>::max());
Token t1(Token::Type::kIntULiteral, Source{}, std::numeric_limits<uint32_t>::max());
EXPECT_EQ(t1.to_i64(), std::numeric_limits<uint32_t>::max());
}
TEST_F(TokenTest, Source) {
@ -64,7 +64,7 @@ TEST_F(TokenTest, Source) {
src.range.begin = Source::Location{3, 9};
src.range.end = Source::Location{4, 3};
Token t(Token::Type::kUintLiteral, src);
Token t(Token::Type::kIntLiteral, src);
EXPECT_EQ(t.source().range.begin.line, 3u);
EXPECT_EQ(t.source().range.begin.column, 9u);
EXPECT_EQ(t.source().range.end.line, 4u);