From 4ff2645d1620d65aae6d847f191ae0ca3ee27e22 Mon Sep 17 00:00:00 2001 From: Ben Clayton Date: Tue, 24 Jan 2023 17:56:57 +0000 Subject: [PATCH] tint/reader/wgsl: Add ClassifyTemplateArguments() Applies a heuristic to disambiguate less-than / greater-than from template argument lists. This function is not currently used by the parser. Bug: tint:1810 Change-Id: Ibd72dbae53b3159282177bf79c00ad0808b123a2 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/117208 Kokoro: Ben Clayton Reviewed-by: Dan Sinclair Kokoro: Kokoro Reviewed-by: David Neto Commit-Queue: Ben Clayton --- src/tint/BUILD.gn | 3 + src/tint/CMakeLists.txt | 3 + .../reader/wgsl/classify_template_args.cc | 168 ++++++ src/tint/reader/wgsl/classify_template_args.h | 28 + .../wgsl/classify_template_args_test.cc | 483 ++++++++++++++++++ src/tint/reader/wgsl/token.cc | 4 + src/tint/reader/wgsl/token.h | 8 +- 7 files changed, 694 insertions(+), 3 deletions(-) create mode 100644 src/tint/reader/wgsl/classify_template_args.cc create mode 100644 src/tint/reader/wgsl/classify_template_args.h create mode 100644 src/tint/reader/wgsl/classify_template_args_test.cc diff --git a/src/tint/BUILD.gn b/src/tint/BUILD.gn index 83962cbab5..451a38f903 100644 --- a/src/tint/BUILD.gn +++ b/src/tint/BUILD.gn @@ -908,6 +908,8 @@ libtint_source_set("libtint_spv_writer_src") { libtint_source_set("libtint_wgsl_reader_src") { sources = [ + "reader/wgsl/classify_template_args.cc", + "reader/wgsl/classify_template_args.h", "reader/wgsl/lexer.cc", "reader/wgsl/lexer.h", "reader/wgsl/parser.cc", @@ -1570,6 +1572,7 @@ if (tint_build_unittests) { tint_unittests_source_set("tint_unittests_wgsl_reader_src") { sources = [ + "reader/wgsl/classify_template_args_test.cc", "reader/wgsl/lexer_test.cc", "reader/wgsl/parser_impl_additive_expression_test.cc", "reader/wgsl/parser_impl_address_space_test.cc", diff --git a/src/tint/CMakeLists.txt b/src/tint/CMakeLists.txt index f1f438094b..9fac34dbeb 100644 --- a/src/tint/CMakeLists.txt +++ b/src/tint/CMakeLists.txt @@ -616,6 +616,8 @@ endif() if(${TINT_BUILD_WGSL_READER}) list(APPEND TINT_LIB_SRCS + reader/wgsl/classify_template_args.cc + reader/wgsl/classify_template_args.h reader/wgsl/lexer.cc reader/wgsl/lexer.h reader/wgsl/parser.cc @@ -1073,6 +1075,7 @@ if(TINT_BUILD_TESTS) if(${TINT_BUILD_WGSL_READER}) list(APPEND TINT_TEST_SRCS + reader/wgsl/classify_template_args_test.cc reader/wgsl/lexer_test.cc reader/wgsl/parser_test.cc reader/wgsl/parser_impl_additive_expression_test.cc diff --git a/src/tint/reader/wgsl/classify_template_args.cc b/src/tint/reader/wgsl/classify_template_args.cc new file mode 100644 index 0000000000..136ff18600 --- /dev/null +++ b/src/tint/reader/wgsl/classify_template_args.cc @@ -0,0 +1,168 @@ +// Copyright 2023 The Tint Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/tint/reader/wgsl/classify_template_args.h" + +#include + +#include "src/tint/debug.h" +#include "src/tint/utils/vector.h" + +namespace tint::reader::wgsl { + +namespace { + +/// If the token at index @p idx is a '>>', '>=' or '>>=', then the token is split into two, with +/// the first being '>', otherwise MaybeSplit() will be a no-op. +/// @param tokens the vector of tokens +/// @param idx the index of the token to (maybe) split +void MaybeSplit(std::vector& tokens, size_t idx) { + Token* token = &tokens[idx]; + switch (token->type()) { + case Token::Type::kShiftRight: // '>>' + TINT_ASSERT(Reader, token[1].type() == Token::Type::kPlaceholder); + token[0].SetType(Token::Type::kGreaterThan); + token[1].SetType(Token::Type::kGreaterThan); + break; + case Token::Type::kGreaterThanEqual: // '>=' + TINT_ASSERT(Reader, token[1].type() == Token::Type::kPlaceholder); + token[0].SetType(Token::Type::kGreaterThan); + token[1].SetType(Token::Type::kEqual); + break; + case Token::Type::kShiftRightEqual: // '>>=' + TINT_ASSERT(Reader, token[1].type() == Token::Type::kPlaceholder); + token[0].SetType(Token::Type::kGreaterThan); + token[1].SetType(Token::Type::kGreaterThanEqual); + break; + default: + break; + } +} + +} // namespace + +void ClassifyTemplateArguments(std::vector& tokens) { + const size_t count = tokens.size(); + + // The current expression nesting depth. + // Each '(', '[' increments the depth. + // Each ')', ']' decrements the depth. + uint64_t expr_depth = 0; + + // A stack of '<' tokens. + // Used to pair '<' and '>' tokens at the same expression depth. + struct StackEntry { + Token* token; // A pointer to the opening '<' token + uint64_t expr_depth; // The value of 'expr_depth' for the opening '<' + }; + utils::Vector stack; + + for (size_t i = 0; i < count - 1; i++) { + switch (tokens[i].type()) { + // + all type / builtin keywords that will become identifiers. + case Token::Type::kIdentifier: + case Token::Type::kArray: + case Token::Type::kAtomic: + case Token::Type::kBitcast: + case Token::Type::kMat2x2: + case Token::Type::kMat2x3: + case Token::Type::kMat2x4: + case Token::Type::kMat3x2: + case Token::Type::kMat3x3: + case Token::Type::kMat3x4: + case Token::Type::kMat4x2: + case Token::Type::kMat4x3: + case Token::Type::kMat4x4: + case Token::Type::kPtr: + case Token::Type::kTextureMultisampled2d: + case Token::Type::kTextureSampled1d: + case Token::Type::kTextureSampled2d: + case Token::Type::kTextureSampled2dArray: + case Token::Type::kTextureSampled3d: + case Token::Type::kTextureSampledCube: + case Token::Type::kTextureSampledCubeArray: + case Token::Type::kTextureStorage1d: + case Token::Type::kTextureStorage2d: + case Token::Type::kTextureStorage2dArray: + case Token::Type::kVec2: + case Token::Type::kVec3: + case Token::Type::kVec4: + case Token::Type::kTextureStorage3d: { + auto& next = tokens[i + 1]; + if (next.type() == Token::Type::kLessThan) { + // ident '<' + // Push this '<' to the stack, along with the current nesting expr_depth. + stack.Push(StackEntry{&tokens[i + 1], expr_depth}); + i++; // Skip the '<' + } + break; + } + case Token::Type::kGreaterThan: // '>' + case Token::Type::kShiftRight: // '>>' + case Token::Type::kGreaterThanEqual: // '>=' + case Token::Type::kShiftRightEqual: // '>>=' + if (!stack.IsEmpty() && stack.Back().expr_depth == expr_depth) { + // '<' and '>' at same expr_depth, and no terminating tokens in-between. + // Consider both as a template argument list. + MaybeSplit(tokens, i); + stack.Pop().token->SetType(Token::Type::kTemplateArgsLeft); + tokens[i].SetType(Token::Type::kTemplateArgsRight); + } + break; + + case Token::Type::kParenLeft: // '(' + case Token::Type::kBracketLeft: // '[' + // Entering a nested expression + expr_depth++; + break; + + case Token::Type::kParenRight: // ')' + case Token::Type::kBracketRight: // ']' + // Exiting a nested expression + // Pop the stack until we return to the current expression expr_depth + while (!stack.IsEmpty() && stack.Back().expr_depth == expr_depth) { + stack.Pop(); + } + if (expr_depth > 0) { + expr_depth--; + } + break; + + case Token::Type::kSemicolon: // ';' + case Token::Type::kBraceLeft: // '{' + case Token::Type::kEqual: // '=' + case Token::Type::kColon: // ':' + // Expression terminating tokens. No opening template list can hold these tokens, so + // clear the stack and expression depth. + expr_depth = 0; + stack.Clear(); + break; + + case Token::Type::kOrOr: // '||' + case Token::Type::kAndAnd: // '&&' + // Treat 'a < b || c > d' as a logical binary operator of two comparison operators + // instead of a single template argument 'b||c'. + // Use parentheses around 'b||c' to parse as a template argument list. + while (!stack.IsEmpty() && stack.Back().expr_depth == expr_depth) { + stack.Pop(); + } + break; + + default: + break; + } + } +} + +} // namespace tint::reader::wgsl diff --git a/src/tint/reader/wgsl/classify_template_args.h b/src/tint/reader/wgsl/classify_template_args.h new file mode 100644 index 0000000000..92d3eb78a4 --- /dev/null +++ b/src/tint/reader/wgsl/classify_template_args.h @@ -0,0 +1,28 @@ +// Copyright 2023 The Tint Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SRC_TINT_READER_WGSL_CLASSIFY_TEMPLATE_ARGS_H_ +#define SRC_TINT_READER_WGSL_CLASSIFY_TEMPLATE_ARGS_H_ + +#include + +#include "src/tint/reader/wgsl/token.h" + +namespace tint::reader::wgsl { + +void ClassifyTemplateArguments(std::vector& tokens); + +} // namespace tint::reader::wgsl + +#endif // SRC_TINT_READER_WGSL_CLASSIFY_TEMPLATE_ARGS_H_ diff --git a/src/tint/reader/wgsl/classify_template_args_test.cc b/src/tint/reader/wgsl/classify_template_args_test.cc new file mode 100644 index 0000000000..fb9bcf9282 --- /dev/null +++ b/src/tint/reader/wgsl/classify_template_args_test.cc @@ -0,0 +1,483 @@ +// Copyright 2023 The Tint Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gmock/gmock.h" + +#include "src/tint/reader/wgsl/classify_template_args.h" +#include "src/tint/reader/wgsl/lexer.h" +#include "src/tint/utils/transform.h" + +namespace tint::reader::wgsl { +namespace { + +using T = Token::Type; + +struct Case { + const char* wgsl; + std::vector tokens; +}; + +static std::ostream& operator<<(std::ostream& out, const Case& c) { + return out << "'" << c.wgsl << "'"; +} + +using ClassifyTemplateArgsTest = testing::TestWithParam; + +TEST_P(ClassifyTemplateArgsTest, Classify) { + auto& params = GetParam(); + Source::File file("", params.wgsl); + Lexer l(&file); + auto tokens = l.Lex(); + ClassifyTemplateArguments(tokens); + auto types = utils::Transform(tokens, [&](const Token& t) { return t.type(); }); + EXPECT_THAT(types, testing::ContainerEq(params.tokens)); +} + +INSTANTIATE_TEST_SUITE_P(NonTemplate, + ClassifyTemplateArgsTest, + testing::ValuesIn(std::vector{ + { + "", + {T::kEOF}, + }, + { + "abc", + {T::kIdentifier, T::kEOF}, + }, + { + "ab", + {T::kIdentifier, T::kGreaterThan, T::kIdentifier, T::kEOF}, + }, + { + "(ac", + { + T::kParenLeft, // ( + T::kIdentifier, // a + T::kLessThan, // < + T::kIdentifier, // b + T::kParenRight, // ) + T::kGreaterThan, // > + T::kIdentifier, // c + T::kEOF, + }, + }, + { + "a<(b>c)", + { + T::kIdentifier, // a + T::kLessThan, // < + T::kParenLeft, // ( + T::kIdentifier, // b + T::kGreaterThan, // > + T::kIdentifier, // c + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a((b(e))", + { + T::kIdentifier, // a + T::kParenLeft, // ( + T::kParenLeft, // ( + T::kIdentifier, // b + T::kLessThan, // < + T::kIdentifier, // c + T::kParenRight, // ) + T::kComma, // , + T::kIdentifier, // d + T::kGreaterThan, // > + T::kParenLeft, // ( + T::kIdentifier, // e + T::kParenRight, // ) + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a(d)]", + { + T::kIdentifier, // a + T::kLessThan, // < + T::kIdentifier, // b + T::kBracketLeft, // [ + T::kIdentifier, // c + T::kGreaterThan, // > + T::kParenLeft, // ( + T::kIdentifier, // d + T::kParenRight, // ) + T::kBracketRight, // ] + T::kEOF, + }, + }, + { + "ad()", + { + T::kIdentifier, // a + T::kLessThan, // < + T::kIdentifier, // b + T::kSemicolon, // ; + T::kIdentifier, // c + T::kGreaterThan, // > + T::kIdentifier, // d + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "if a < b {} else if c > d {}", + { + T::kIf, // a + T::kIdentifier, // a + T::kLessThan, // < + T::kIdentifier, // b + T::kBraceLeft, // { + T::kBraceRight, // } + T::kElse, // else + T::kIf, // if + T::kIdentifier, // c + T::kGreaterThan, // > + T::kIdentifier, // d + T::kBraceLeft, // { + T::kBraceRight, // } + T::kEOF, + }, + }, + { + "ad", + { + T::kIdentifier, // a + T::kLessThan, // < + T::kIdentifier, // b + T::kAndAnd, // && + T::kPlaceholder, // + T::kIdentifier, // c + T::kGreaterThan, // > + T::kIdentifier, // d + T::kEOF, + }, + }, + { + "ad", + { + T::kIdentifier, // a + T::kLessThan, // < + T::kIdentifier, // b + T::kOrOr, // || + T::kIdentifier, // c + T::kGreaterThan, // > + T::kIdentifier, // d + T::kEOF, + }, + }, + { + "a>", + { + T::kIdentifier, // a + T::kLessThan, // < + T::kIdentifier, // b + T::kLessThan, // < + T::kIdentifier, // c + T::kOrOr, // || + T::kIdentifier, // d + T::kShiftRight, // >> + T::kPlaceholder, // + T::kEOF, + }, + }, + })); + +INSTANTIATE_TEST_SUITE_P(Template, + ClassifyTemplateArgsTest, + testing::ValuesIn(std::vector{ + { + "a()", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kTemplateArgsRight, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "ac", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kTemplateArgsRight, // > + T::kIdentifier, // c + T::kEOF, + }, + }, + { + "vec3", + { + T::kVec3, // vec3 + T::kTemplateArgsLeft, // < + T::kI32, // i32 + T::kTemplateArgsRight, // > + T::kEOF, + }, + }, + { + "vec3()", + { + T::kVec3, // vec3 + T::kTemplateArgsLeft, // < + T::kI32, // i32 + T::kTemplateArgsRight, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "array,5>", + { + T::kArray, // array + T::kTemplateArgsLeft, // < + T::kVec3, // vec3 + T::kTemplateArgsLeft, // < + T::kI32, // i32 + T::kTemplateArgsRight, // > + T::kComma, // , + T::kIntLiteral, // 5 + T::kTemplateArgsRight, // > + T::kEOF, + }, + }, + { + "a(b(e))", + { + T::kIdentifier, // a + T::kParenLeft, // ( + T::kIdentifier, // b + T::kTemplateArgsLeft, // < + T::kIdentifier, // c + T::kComma, // , + T::kIdentifier, // d + T::kTemplateArgsRight, // > + T::kParenLeft, // ( + T::kIdentifier, // e + T::kParenRight, // ) + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a<1+2>()", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIntLiteral, // 1 + T::kPlus, // + + T::kIntLiteral, // 2 + T::kTemplateArgsRight, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a<1,b>()", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIntLiteral, // 1 + T::kComma, // , + T::kIdentifier, // b + T::kTemplateArgsRight, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a=d", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kComma, // , + T::kIdentifier, // c + T::kTemplateArgsRight, // > + T::kEqual, // = + T::kIdentifier, // d + T::kEOF, + }, + }, + { + "a=d>()", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kComma, // , + T::kIdentifier, // c + T::kTemplateArgsRight, // > + T::kEqual, // = + T::kIdentifier, // d + T::kGreaterThan, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a>=", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kTemplateArgsLeft, // < + T::kIdentifier, // c + T::kTemplateArgsRight, // > + T::kTemplateArgsRight, // > + T::kEqual, // = + T::kEOF, + }, + }, + { + "ac>()", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kTemplateArgsRight, // > + T::kIdentifier, // c + T::kGreaterThan, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a()", + { + T::kIdentifier, // a + T::kLessThan, // < + T::kIdentifier, // b + T::kTemplateArgsLeft, // < + T::kIdentifier, // c + T::kTemplateArgsRight, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a>()", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kTemplateArgsLeft, // < + T::kIdentifier, // c + T::kTemplateArgsRight, // > + T::kTemplateArgsRight, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a()>()", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kTemplateArgsLeft, // < + T::kIdentifier, // c + T::kTemplateArgsRight, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kTemplateArgsRight, // > + T::kParenLeft, // ( + T::kParenRight, // ) + T::kEOF, + }, + }, + { + "a.c", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kTemplateArgsRight, // > + T::kPeriod, // . + T::kIdentifier, // c + T::kEOF, + }, + }, + { + "a<(b&&c)>d", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kParenLeft, // ( + T::kIdentifier, // b + T::kAndAnd, // && + T::kPlaceholder, // + T::kIdentifier, // c + T::kParenRight, // ) + T::kTemplateArgsRight, // > + T::kIdentifier, // d + T::kEOF, + }, + }, + { + "a<(b||c)>d", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kParenLeft, // ( + T::kIdentifier, // b + T::kOrOr, // || + T::kIdentifier, // c + T::kParenRight, // ) + T::kTemplateArgsRight, // > + T::kIdentifier, // d + T::kEOF, + }, + }, + { + "a>", + { + T::kIdentifier, // a + T::kTemplateArgsLeft, // < + T::kIdentifier, // b + T::kTemplateArgsLeft, // < + T::kParenLeft, // ( + T::kIdentifier, // c + T::kOrOr, // || + T::kIdentifier, // d + T::kParenRight, // ) + T::kTemplateArgsRight, // > + T::kTemplateArgsRight, // > + T::kEOF, + }, + }, + })); + +} // namespace +} // namespace tint::reader::wgsl diff --git a/src/tint/reader/wgsl/token.cc b/src/tint/reader/wgsl/token.cc index c85bbf0a43..52706d60ad 100644 --- a/src/tint/reader/wgsl/token.cc +++ b/src/tint/reader/wgsl/token.cc @@ -70,12 +70,16 @@ std::string_view Token::TypeToName(Type type) { return "="; case Token::Type::kEqualEqual: return "=="; + case Token::Type::kTemplateArgsRight: + return "> (closing template argument list)"; case Token::Type::kGreaterThan: return ">"; case Token::Type::kGreaterThanEqual: return ">="; case Token::Type::kShiftRight: return ">>"; + case Token::Type::kTemplateArgsLeft: + return "< (opening template argument list)"; case Token::Type::kLessThan: return "<"; case Token::Type::kLessThanEqual: diff --git a/src/tint/reader/wgsl/token.h b/src/tint/reader/wgsl/token.h index 1aeab04748..7935cc347a 100644 --- a/src/tint/reader/wgsl/token.h +++ b/src/tint/reader/wgsl/token.h @@ -32,7 +32,7 @@ class Token { kError = -2, /// Uninitialized token kUninitialized = 0, - /// Placeholder token which maybe fillled in later + /// Placeholder token which maybe filled in later kPlaceholder = 1, /// End of input string reached kEOF, @@ -80,12 +80,16 @@ class Token { kEqual, /// A '==' kEqualEqual, + /// A '>' (post template-args classification) + kTemplateArgsRight, /// A '>' kGreaterThan, /// A '>=' kGreaterThanEqual, /// A '>>' kShiftRight, + /// A '<' (post template-args classification) + kTemplateArgsLeft, /// A '<' kLessThan, /// A '<=' @@ -456,12 +460,10 @@ class Token { std::variant value_; }; -#ifndef NDEBUG inline std::ostream& operator<<(std::ostream& out, Token::Type type) { out << Token::TypeToName(type); return out; } -#endif // NDEBUG } // namespace tint::reader::wgsl