reader/wgsl: Optimize tokens by using string_view

Each Token was making a copy of the `val_str_`, despite the token being a slice on the original source.

Bug: tint:1383
Change-Id: I17b2da8f986ba105853aa47afe21bcc75f140f8e
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/78320
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
This commit is contained in:
Ben Clayton 2022-01-27 14:48:46 +00:00 committed by Tint LUCI CQ
parent e2f35ba8e0
commit 3cdb8e3c3e
4 changed files with 152 additions and 146 deletions

View File

@ -62,38 +62,31 @@ Lexer::Lexer(const std::string& file_path, const Source::FileContent* content)
Lexer::~Lexer() = default; Lexer::~Lexer() = default;
Token Lexer::next() { Token Lexer::next() {
auto t = skip_whitespace_and_comments(); if (auto t = skip_whitespace_and_comments(); !t.IsUninitialized()) {
if (!t.IsUninitialized()) {
return t; return t;
} }
t = try_hex_float(); if (auto t = try_hex_float(); !t.IsUninitialized()) {
if (!t.IsUninitialized()) {
return t; return t;
} }
t = try_hex_integer(); if (auto t = try_hex_integer(); !t.IsUninitialized()) {
if (!t.IsUninitialized()) {
return t; return t;
} }
t = try_float(); if (auto t = try_float(); !t.IsUninitialized()) {
if (!t.IsUninitialized()) {
return t; return t;
} }
t = try_integer(); if (auto t = try_integer(); !t.IsUninitialized()) {
if (!t.IsUninitialized()) {
return t; return t;
} }
t = try_ident(); if (auto t = try_ident(); !t.IsUninitialized()) {
if (!t.IsUninitialized()) {
return t; return t;
} }
t = try_punctuation(); if (auto t = try_punctuation(); !t.IsUninitialized()) {
if (!t.IsUninitialized()) {
return t; return t;
} }
@ -771,7 +764,7 @@ Token Lexer::try_ident() {
return t; return t;
} }
return {Token::Type::kIdentifier, source, std::string(str)}; return {Token::Type::kIdentifier, source, str};
} }
Token Lexer::try_punctuation() { Token Lexer::try_punctuation() {

View File

@ -125,28 +125,22 @@ const char kStrideDecoration[] = "stride";
const char kWorkgroupSizeDecoration[] = "workgroup_size"; const char kWorkgroupSizeDecoration[] = "workgroup_size";
bool is_decoration(Token t) { bool is_decoration(Token t) {
if (!t.IsIdentifier()) { return t == kAlignDecoration || t == kBindingDecoration ||
return false; t == kBlockDecoration || t == kBuiltinDecoration ||
} t == kGroupDecoration || t == kInterpolateDecoration ||
t == kLocationDecoration || t == kOverrideDecoration ||
auto s = t.to_str(); t == kSizeDecoration || t == kStageDecoration ||
return s == kAlignDecoration || s == kBindingDecoration || t == kStrideDecoration || t == kWorkgroupSizeDecoration;
s == kBlockDecoration || s == kBuiltinDecoration ||
s == kGroupDecoration || s == kInterpolateDecoration ||
s == kLocationDecoration || s == kOverrideDecoration ||
s == kSizeDecoration || s == kStageDecoration ||
s == kStrideDecoration || s == kWorkgroupSizeDecoration;
} }
// https://gpuweb.github.io/gpuweb/wgsl.html#reserved-keywords // https://gpuweb.github.io/gpuweb/wgsl.html#reserved-keywords
bool is_reserved(Token t) { bool is_reserved(Token t) {
auto s = t.to_str(); return t == "asm" || t == "bf16" || t == "const" || t == "do" ||
return s == "asm" || s == "bf16" || s == "const" || s == "do" || t == "enum" || t == "f16" || t == "f64" || t == "handle" ||
s == "enum" || s == "f16" || s == "f64" || s == "handle" || t == "i8" || t == "i16" || t == "i64" || t == "mat" ||
s == "i8" || s == "i16" || s == "i64" || s == "mat" || t == "premerge" || t == "regardless" || t == "typedef" || t == "u8" ||
s == "premerge" || s == "regardless" || s == "typedef" || s == "u8" || t == "u16" || t == "u64" || t == "unless" || t == "using" ||
s == "u16" || s == "u64" || s == "unless" || s == "using" || t == "vec" || t == "void" || t == "while";
s == "vec" || s == "void" || s == "while";
} }
/// Enter-exit counters for block token types. /// Enter-exit counters for block token types.
@ -760,59 +754,56 @@ Maybe<const ast::Type*> ParserImpl::depth_texture_type() {
// | 'rgba32sint' // | 'rgba32sint'
// | 'rgba32float' // | 'rgba32float'
Expect<ast::TexelFormat> ParserImpl::expect_texel_format(std::string_view use) { Expect<ast::TexelFormat> ParserImpl::expect_texel_format(std::string_view use) {
auto tok = next(); auto t = next();
if (tok.IsIdentifier()) { if (t == "rgba8unorm") {
auto s = tok.to_str();
if (s == "rgba8unorm") {
return ast::TexelFormat::kRgba8Unorm; return ast::TexelFormat::kRgba8Unorm;
} }
if (s == "rgba8snorm") { if (t == "rgba8snorm") {
return ast::TexelFormat::kRgba8Snorm; return ast::TexelFormat::kRgba8Snorm;
} }
if (s == "rgba8uint") { if (t == "rgba8uint") {
return ast::TexelFormat::kRgba8Uint; return ast::TexelFormat::kRgba8Uint;
} }
if (s == "rgba8sint") { if (t == "rgba8sint") {
return ast::TexelFormat::kRgba8Sint; return ast::TexelFormat::kRgba8Sint;
} }
if (s == "rgba16uint") { if (t == "rgba16uint") {
return ast::TexelFormat::kRgba16Uint; return ast::TexelFormat::kRgba16Uint;
} }
if (s == "rgba16sint") { if (t == "rgba16sint") {
return ast::TexelFormat::kRgba16Sint; return ast::TexelFormat::kRgba16Sint;
} }
if (s == "rgba16float") { if (t == "rgba16float") {
return ast::TexelFormat::kRgba16Float; return ast::TexelFormat::kRgba16Float;
} }
if (s == "r32uint") { if (t == "r32uint") {
return ast::TexelFormat::kR32Uint; return ast::TexelFormat::kR32Uint;
} }
if (s == "r32sint") { if (t == "r32sint") {
return ast::TexelFormat::kR32Sint; return ast::TexelFormat::kR32Sint;
} }
if (s == "r32float") { if (t == "r32float") {
return ast::TexelFormat::kR32Float; return ast::TexelFormat::kR32Float;
} }
if (s == "rg32uint") { if (t == "rg32uint") {
return ast::TexelFormat::kRg32Uint; return ast::TexelFormat::kRg32Uint;
} }
if (s == "rg32sint") { if (t == "rg32sint") {
return ast::TexelFormat::kRg32Sint; return ast::TexelFormat::kRg32Sint;
} }
if (s == "rg32float") { if (t == "rg32float") {
return ast::TexelFormat::kRg32Float; return ast::TexelFormat::kRg32Float;
} }
if (s == "rgba32uint") { if (t == "rgba32uint") {
return ast::TexelFormat::kRgba32Uint; return ast::TexelFormat::kRgba32Uint;
} }
if (s == "rgba32sint") { if (t == "rgba32sint") {
return ast::TexelFormat::kRgba32Sint; return ast::TexelFormat::kRgba32Sint;
} }
if (s == "rgba32float") { if (t == "rgba32float") {
return ast::TexelFormat::kRgba32Float; return ast::TexelFormat::kRgba32Float;
} }
} return add_error(t.source(), "invalid format", use);
return add_error(tok.source(), "invalid format", use);
} }
// variable_ident_decl // variable_ident_decl
@ -1431,24 +1422,18 @@ Expect<ast::Variable*> ParserImpl::expect_param() {
// | COMPUTE // | COMPUTE
Expect<ast::PipelineStage> ParserImpl::expect_pipeline_stage() { Expect<ast::PipelineStage> ParserImpl::expect_pipeline_stage() {
auto t = peek(); auto t = peek();
if (!t.IsIdentifier()) { if (t == kVertexStage) {
return add_error(t, "invalid value for stage decoration");
}
auto s = t.to_str();
if (s == kVertexStage) {
next(); // Consume the peek next(); // Consume the peek
return {ast::PipelineStage::kVertex, t.source()}; return {ast::PipelineStage::kVertex, t.source()};
} }
if (s == kFragmentStage) { if (t == kFragmentStage) {
next(); // Consume the peek next(); // Consume the peek
return {ast::PipelineStage::kFragment, t.source()}; return {ast::PipelineStage::kFragment, t.source()};
} }
if (s == kComputeStage) { if (t == kComputeStage) {
next(); // Consume the peek next(); // Consume the peek
return {ast::PipelineStage::kCompute, t.source()}; return {ast::PipelineStage::kCompute, t.source()};
} }
return add_error(peek(), "invalid value for stage decoration"); return add_error(peek(), "invalid value for stage decoration");
} }
@ -2932,9 +2917,7 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
return Failure::kNoMatch; return Failure::kNoMatch;
} }
auto s = t.to_str(); if (t == kLocationDecoration) {
if (s == kLocationDecoration) {
const char* use = "location decoration"; const char* use = "location decoration";
return expect_paren_block(use, [&]() -> Result { return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use); auto val = expect_positive_sint(use);
@ -2945,7 +2928,7 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kBindingDecoration) { if (t == kBindingDecoration) {
const char* use = "binding decoration"; const char* use = "binding decoration";
return expect_paren_block(use, [&]() -> Result { return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use); auto val = expect_positive_sint(use);
@ -2956,7 +2939,7 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kGroupDecoration) { if (t == kGroupDecoration) {
const char* use = "group decoration"; const char* use = "group decoration";
return expect_paren_block(use, [&]() -> Result { return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use); auto val = expect_positive_sint(use);
@ -2967,18 +2950,17 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kInterpolateDecoration) { if (t == kInterpolateDecoration) {
return expect_paren_block("interpolate decoration", [&]() -> Result { return expect_paren_block("interpolate decoration", [&]() -> Result {
ast::InterpolationType type; ast::InterpolationType type;
ast::InterpolationSampling sampling = ast::InterpolationSampling::kNone; ast::InterpolationSampling sampling = ast::InterpolationSampling::kNone;
auto type_tok = next(); auto type_tok = next();
auto type_str = type_tok.to_str(); if (type_tok == "perspective") {
if (type_str == "perspective") {
type = ast::InterpolationType::kPerspective; type = ast::InterpolationType::kPerspective;
} else if (type_str == "linear") { } else if (type_tok == "linear") {
type = ast::InterpolationType::kLinear; type = ast::InterpolationType::kLinear;
} else if (type_str == "flat") { } else if (type_tok == "flat") {
type = ast::InterpolationType::kFlat; type = ast::InterpolationType::kFlat;
} else { } else {
return add_error(type_tok, "invalid interpolation type"); return add_error(type_tok, "invalid interpolation type");
@ -2986,12 +2968,11 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
if (match(Token::Type::kComma)) { if (match(Token::Type::kComma)) {
auto sampling_tok = next(); auto sampling_tok = next();
auto sampling_str = sampling_tok.to_str(); if (sampling_tok == "center") {
if (sampling_str == "center") {
sampling = ast::InterpolationSampling::kCenter; sampling = ast::InterpolationSampling::kCenter;
} else if (sampling_str == "centroid") { } else if (sampling_tok == "centroid") {
sampling = ast::InterpolationSampling::kCentroid; sampling = ast::InterpolationSampling::kCentroid;
} else if (sampling_str == "sample") { } else if (sampling_tok == "sample") {
sampling = ast::InterpolationSampling::kSample; sampling = ast::InterpolationSampling::kSample;
} else { } else {
return add_error(sampling_tok, "invalid interpolation sampling"); return add_error(sampling_tok, "invalid interpolation sampling");
@ -3002,11 +2983,11 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kInvariantDecoration) { if (t == kInvariantDecoration) {
return create<ast::InvariantDecoration>(t.source()); return create<ast::InvariantDecoration>(t.source());
} }
if (s == kBuiltinDecoration) { if (t == kBuiltinDecoration) {
return expect_paren_block("builtin decoration", [&]() -> Result { return expect_paren_block("builtin decoration", [&]() -> Result {
auto builtin = expect_builtin(); auto builtin = expect_builtin();
if (builtin.errored) if (builtin.errored)
@ -3016,7 +2997,7 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kWorkgroupSizeDecoration) { if (t == kWorkgroupSizeDecoration) {
return expect_paren_block("workgroup_size decoration", [&]() -> Result { return expect_paren_block("workgroup_size decoration", [&]() -> Result {
const ast::Expression* x = nullptr; const ast::Expression* x = nullptr;
const ast::Expression* y = nullptr; const ast::Expression* y = nullptr;
@ -3054,7 +3035,7 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kStageDecoration) { if (t == kStageDecoration) {
return expect_paren_block("stage decoration", [&]() -> Result { return expect_paren_block("stage decoration", [&]() -> Result {
auto stage = expect_pipeline_stage(); auto stage = expect_pipeline_stage();
if (stage.errored) if (stage.errored)
@ -3064,12 +3045,12 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kBlockDecoration) { if (t == kBlockDecoration) {
deprecated(t.source(), "[[block]] attributes have been removed from WGSL"); deprecated(t.source(), "[[block]] attributes have been removed from WGSL");
return create<ast::StructBlockDecoration>(t.source()); return create<ast::StructBlockDecoration>(t.source());
} }
if (s == kStrideDecoration) { if (t == kStrideDecoration) {
const char* use = "stride decoration"; const char* use = "stride decoration";
return expect_paren_block(use, [&]() -> Result { return expect_paren_block(use, [&]() -> Result {
auto val = expect_nonzero_positive_sint(use); auto val = expect_nonzero_positive_sint(use);
@ -3082,7 +3063,7 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kSizeDecoration) { if (t == kSizeDecoration) {
const char* use = "size decoration"; const char* use = "size decoration";
return expect_paren_block(use, [&]() -> Result { return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use); auto val = expect_positive_sint(use);
@ -3093,7 +3074,7 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kAlignDecoration) { if (t == kAlignDecoration) {
const char* use = "align decoration"; const char* use = "align decoration";
return expect_paren_block(use, [&]() -> Result { return expect_paren_block(use, [&]() -> Result {
auto val = expect_positive_sint(use); auto val = expect_positive_sint(use);
@ -3104,7 +3085,7 @@ Maybe<const ast::Decoration*> ParserImpl::decoration() {
}); });
} }
if (s == kOverrideDecoration) { if (t == kOverrideDecoration) {
const char* use = "override decoration"; const char* use = "override decoration";
if (peek_is(Token::Type::kParenLeft)) { if (peek_is(Token::Type::kParenLeft)) {

View File

@ -254,19 +254,25 @@ std::string_view Token::TypeToName(Type type) {
Token::Token() : type_(Type::kUninitialized) {} Token::Token() : type_(Type::kUninitialized) {}
Token::Token(Type type, const Source& source, const std::string& val) Token::Token(Type type, const Source& source, const std::string_view& view)
: type_(type), source_(source), val_str_(val) {} : type_(type), source_(source), value_(view) {}
Token::Token(Type type, const Source& source, const std::string& str)
: type_(type), source_(source), value_(str) {}
Token::Token(Type type, const Source& source, const char* str)
: type_(type), source_(source), value_(std::string_view(str)) {}
Token::Token(const Source& source, uint32_t val) Token::Token(const Source& source, uint32_t val)
: type_(Type::kUintLiteral), source_(source), val_uint_(val) {} : type_(Type::kUintLiteral), source_(source), value_(val) {}
Token::Token(const Source& source, int32_t val) Token::Token(const Source& source, int32_t val)
: type_(Type::kSintLiteral), source_(source), val_int_(val) {} : type_(Type::kSintLiteral), source_(source), value_(val) {}
Token::Token(const Source& source, float val) Token::Token(const Source& source, float val)
: type_(Type::kFloatLiteral), source_(source), val_float_(val) {} : type_(Type::kFloatLiteral), source_(source), value_(val) {}
Token::Token(Type type, const Source& source) : Token(type, source, "") {} Token::Token(Type type, const Source& source) : type_(type), source_(source) {}
Token::Token(Token&&) = default; Token::Token(Token&&) = default;
@ -274,31 +280,47 @@ Token::Token(const Token&) = default;
Token::~Token() = default; Token::~Token() = default;
Token& Token::operator=(const Token&) = default; Token& Token::operator=(const Token& rhs) = default;
bool Token::operator==(std::string_view ident) {
if (type_ != Type::kIdentifier) {
return false;
}
if (auto* view = std::get_if<std::string_view>(&value_)) {
return *view == ident;
}
return std::get<std::string>(value_) == ident;
}
std::string Token::to_str() const { std::string Token::to_str() const {
if (type_ == Type::kFloatLiteral) { switch (type_) {
return std::to_string(val_float_); case Type::kFloatLiteral:
return std::to_string(std::get<float>(value_));
case Type::kSintLiteral:
return std::to_string(std::get<int32_t>(value_));
case Type::kUintLiteral:
return std::to_string(std::get<uint32_t>(value_));
case Type::kIdentifier:
case Type::kError:
if (auto* view = std::get_if<std::string_view>(&value_)) {
return std::string(*view);
} }
if (type_ == Type::kSintLiteral) { return std::get<std::string>(value_);
return std::to_string(val_int_); default:
return "";
} }
if (type_ == Type::kUintLiteral) {
return std::to_string(val_uint_);
}
return val_str_;
} }
float Token::to_f32() const { float Token::to_f32() const {
return val_float_; return std::get<float>(value_);
} }
uint32_t Token::to_u32() const { uint32_t Token::to_u32() const {
return val_uint_; return std::get<uint32_t>(value_);
} }
int32_t Token::to_i32() const { int32_t Token::to_i32() const {
return val_int_; return std::get<int32_t>(value_);
} }
} // namespace wgsl } // namespace wgsl

View File

@ -17,6 +17,7 @@
#include <string> #include <string>
#include <string_view> #include <string_view>
#include <variant> // NOLINT: cpplint doesn't recognise this
#include "src/source.h" #include "src/source.h"
@ -273,8 +274,18 @@ class Token {
/// Create a string Token /// Create a string Token
/// @param type the Token::Type of the token /// @param type the Token::Type of the token
/// @param source the source of the token /// @param source the source of the token
/// @param val the source string for the token /// @param view the source string view for the token
Token(Type type, const Source& source, const std::string& val); Token(Type type, const Source& source, const std::string_view& view);
/// Create a string Token
/// @param type the Token::Type of the token
/// @param source the source of the token
/// @param str the source string for the token
Token(Type type, const Source& source, const std::string& str);
/// Create a string Token
/// @param type the Token::Type of the token
/// @param source the source of the token
/// @param str the source string for the token
Token(Type type, const Source& source, const char* str);
/// Create a unsigned integer Token /// Create a unsigned integer Token
/// @param source the source of the token /// @param source the source of the token
/// @param val the source unsigned for the token /// @param val the source unsigned for the token
@ -298,6 +309,11 @@ class Token {
/// @return Token /// @return Token
Token& operator=(const Token& b); Token& operator=(const Token& b);
/// Equality operator with an identifier
/// @param ident the identifier string
/// @return true if this token is an identifier and is equal to ident.
bool operator==(std::string_view ident);
/// Returns true if the token is of the given type /// Returns true if the token is of the given type
/// @param t the type to check against. /// @param t the type to check against.
/// @returns true if the token is of type `t` /// @returns true if the token is of type `t`
@ -378,14 +394,8 @@ class Token {
Type type_ = Type::kError; Type type_ = Type::kError;
/// The source where the token appeared /// The source where the token appeared
Source source_; Source source_;
/// The string represented by the token /// The value represented by the token
std::string val_str_; std::variant<int32_t, uint32_t, float, std::string, std::string_view> value_;
/// The signed integer represented by the token
int32_t val_int_ = 0;
/// The unsigned integer represented by the token
uint32_t val_uint_ = 0;
/// The float value represented by the token
float val_float_ = 0.0;
}; };
#ifndef NDEBUG #ifndef NDEBUG