encounter/dawn-cmake
1
0
Fork 0
mirror of https://github.com/encounter/dawn-cmake.git synced 2025-06-23 23:13:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

tint/reader/wgsl: Add ClassifyTemplateArguments()

Browse Source 
Applies a heuristic to disambiguate less-than / greater-than from template argument lists.

This function is not currently used by the parser.

Bug: tint:1810
Change-Id: Ibd72dbae53b3159282177bf79c00ad0808b123a2
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/117208
Kokoro: Ben Clayton <bclayton@chromium.org>
Reviewed-by: Dan Sinclair <dsinclair@chromium.org>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: David Neto <dneto@google.com>
Commit-Queue: Ben Clayton <bclayton@chromium.org>
This commit is contained in:
Ben Clayton 2023-01-24 17:56:57 +00:00 committed by Dawn LUCI CQ
parent d4f318d0f3
commit 4ff2645d16
7 changed files with 694 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/tint/BUILD.gn
View File

@ -908,6 +908,8 @@ libtint_source_set("libtint_spv_writer_src") {
libtint_source_set("libtint_wgsl_reader_src") { libtint_source_set("libtint_wgsl_reader_src") {
sources = [ sources = [
"reader/wgsl/classify_template_args.cc",
"reader/wgsl/classify_template_args.h",
"reader/wgsl/lexer.cc", "reader/wgsl/lexer.cc",
"reader/wgsl/lexer.h", "reader/wgsl/lexer.h",
"reader/wgsl/parser.cc", "reader/wgsl/parser.cc",
@ -1570,6 +1572,7 @@ if (tint_build_unittests) {
tint_unittests_source_set("tint_unittests_wgsl_reader_src") { tint_unittests_source_set("tint_unittests_wgsl_reader_src") {
sources = [ sources = [
"reader/wgsl/classify_template_args_test.cc",
"reader/wgsl/lexer_test.cc", "reader/wgsl/lexer_test.cc",
"reader/wgsl/parser_impl_additive_expression_test.cc", "reader/wgsl/parser_impl_additive_expression_test.cc",
"reader/wgsl/parser_impl_address_space_test.cc", "reader/wgsl/parser_impl_address_space_test.cc",

3
src/tint/CMakeLists.txt
View File

@ -616,6 +616,8 @@ endif()
if(${TINT_BUILD_WGSL_READER}) if(${TINT_BUILD_WGSL_READER})
list(APPEND TINT_LIB_SRCS list(APPEND TINT_LIB_SRCS
reader/wgsl/classify_template_args.cc
reader/wgsl/classify_template_args.h
reader/wgsl/lexer.cc reader/wgsl/lexer.cc
reader/wgsl/lexer.h reader/wgsl/lexer.h
reader/wgsl/parser.cc reader/wgsl/parser.cc
@ -1073,6 +1075,7 @@ if(TINT_BUILD_TESTS)
if(${TINT_BUILD_WGSL_READER}) if(${TINT_BUILD_WGSL_READER})
list(APPEND TINT_TEST_SRCS list(APPEND TINT_TEST_SRCS
reader/wgsl/classify_template_args_test.cc
reader/wgsl/lexer_test.cc reader/wgsl/lexer_test.cc
reader/wgsl/parser_test.cc reader/wgsl/parser_test.cc
reader/wgsl/parser_impl_additive_expression_test.cc reader/wgsl/parser_impl_additive_expression_test.cc

168
src/tint/reader/wgsl/classify_template_args.cc Normal file
View File

@ -0,0 +1,168 @@
// Copyright 2023 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "src/tint/reader/wgsl/classify_template_args.h"
#include <vector>
#include "src/tint/debug.h"
#include "src/tint/utils/vector.h"
namespace tint::reader::wgsl {
namespace {
/// If the token at index @p idx is a '>>', '>=' or '>>=', then the token is split into two, with
/// the first being '>', otherwise MaybeSplit() will be a no-op.
/// @param tokens the vector of tokens
/// @param idx the index of the token to (maybe) split
void MaybeSplit(std::vector<Token>& tokens, size_t idx) {
Token* token = &tokens[idx];
switch (token->type()) {
case Token::Type::kShiftRight: // '>>'
TINT_ASSERT(Reader, token[1].type() == Token::Type::kPlaceholder);
token[0].SetType(Token::Type::kGreaterThan);
token[1].SetType(Token::Type::kGreaterThan);
break;
case Token::Type::kGreaterThanEqual: // '>='
TINT_ASSERT(Reader, token[1].type() == Token::Type::kPlaceholder);
token[0].SetType(Token::Type::kGreaterThan);
token[1].SetType(Token::Type::kEqual);
break;
case Token::Type::kShiftRightEqual: // '>>='
TINT_ASSERT(Reader, token[1].type() == Token::Type::kPlaceholder);
token[0].SetType(Token::Type::kGreaterThan);
token[1].SetType(Token::Type::kGreaterThanEqual);
break;
default:
break;
}
}
} // namespace
void ClassifyTemplateArguments(std::vector<Token>& tokens) {
const size_t count = tokens.size();
// The current expression nesting depth.
// Each '(', '[' increments the depth.
// Each ')', ']' decrements the depth.
uint64_t expr_depth = 0;
// A stack of '<' tokens.
// Used to pair '<' and '>' tokens at the same expression depth.
struct StackEntry {
Token* token; // A pointer to the opening '<' token
uint64_t expr_depth; // The value of 'expr_depth' for the opening '<'
};
utils::Vector<StackEntry, 16> stack;
for (size_t i = 0; i < count - 1; i++) {
switch (tokens[i].type()) {
// <identifier> + all type / builtin keywords that will become identifiers.
case Token::Type::kIdentifier:
case Token::Type::kArray:
case Token::Type::kAtomic:
case Token::Type::kBitcast:
case Token::Type::kMat2x2:
case Token::Type::kMat2x3:
case Token::Type::kMat2x4:
case Token::Type::kMat3x2:
case Token::Type::kMat3x3:
case Token::Type::kMat3x4:
case Token::Type::kMat4x2:
case Token::Type::kMat4x3:
case Token::Type::kMat4x4:
case Token::Type::kPtr:
case Token::Type::kTextureMultisampled2d:
case Token::Type::kTextureSampled1d:
case Token::Type::kTextureSampled2d:
case Token::Type::kTextureSampled2dArray:
case Token::Type::kTextureSampled3d:
case Token::Type::kTextureSampledCube:
case Token::Type::kTextureSampledCubeArray:
case Token::Type::kTextureStorage1d:
case Token::Type::kTextureStorage2d:
case Token::Type::kTextureStorage2dArray:
case Token::Type::kVec2:
case Token::Type::kVec3:
case Token::Type::kVec4:
case Token::Type::kTextureStorage3d: {
auto& next = tokens[i + 1];
if (next.type() == Token::Type::kLessThan) {
// ident '<'
// Push this '<' to the stack, along with the current nesting expr_depth.
stack.Push(StackEntry{&tokens[i + 1], expr_depth});
i++; // Skip the '<'
}
break;
}
case Token::Type::kGreaterThan: // '>'
case Token::Type::kShiftRight: // '>>'
case Token::Type::kGreaterThanEqual: // '>='
case Token::Type::kShiftRightEqual: // '>>='
if (!stack.IsEmpty() && stack.Back().expr_depth == expr_depth) {
// '<' and '>' at same expr_depth, and no terminating tokens in-between.
// Consider both as a template argument list.
MaybeSplit(tokens, i);
stack.Pop().token->SetType(Token::Type::kTemplateArgsLeft);
tokens[i].SetType(Token::Type::kTemplateArgsRight);
}
break;
case Token::Type::kParenLeft: // '('
case Token::Type::kBracketLeft: // '['
// Entering a nested expression
expr_depth++;
break;
case Token::Type::kParenRight: // ')'
case Token::Type::kBracketRight: // ']'
// Exiting a nested expression
// Pop the stack until we return to the current expression expr_depth
while (!stack.IsEmpty() && stack.Back().expr_depth == expr_depth) {
stack.Pop();
}
if (expr_depth > 0) {
expr_depth--;
}
break;
case Token::Type::kSemicolon: // ';'
case Token::Type::kBraceLeft: // '{'
case Token::Type::kEqual: // '='
case Token::Type::kColon: // ':'
// Expression terminating tokens. No opening template list can hold these tokens, so
// clear the stack and expression depth.
expr_depth = 0;
stack.Clear();
break;
case Token::Type::kOrOr: // '||'
case Token::Type::kAndAnd: // '&&'
// Treat 'a < b || c > d' as a logical binary operator of two comparison operators
// instead of a single template argument 'b||c'.
// Use parentheses around 'b||c' to parse as a template argument list.
while (!stack.IsEmpty() && stack.Back().expr_depth == expr_depth) {
stack.Pop();
}
break;
default:
break;
}
}
}
} // namespace tint::reader::wgsl

28
src/tint/reader/wgsl/classify_template_args.h Normal file
View File

@ -0,0 +1,28 @@
// Copyright 2023 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SRC_TINT_READER_WGSL_CLASSIFY_TEMPLATE_ARGS_H_
#define SRC_TINT_READER_WGSL_CLASSIFY_TEMPLATE_ARGS_H_
#include <vector>
#include "src/tint/reader/wgsl/token.h"
namespace tint::reader::wgsl {
void ClassifyTemplateArguments(std::vector<Token>& tokens);
} // namespace tint::reader::wgsl
#endif // SRC_TINT_READER_WGSL_CLASSIFY_TEMPLATE_ARGS_H_

483
src/tint/reader/wgsl/classify_template_args_test.cc Normal file
View File

@ -0,0 +1,483 @@
// Copyright 2023 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "gmock/gmock.h"
#include "src/tint/reader/wgsl/classify_template_args.h"
#include "src/tint/reader/wgsl/lexer.h"
#include "src/tint/utils/transform.h"
namespace tint::reader::wgsl {
namespace {
using T = Token::Type;
struct Case {
const char* wgsl;
std::vector<T> tokens;
};
static std::ostream& operator<<(std::ostream& out, const Case& c) {
return out << "'" << c.wgsl << "'";
}
using ClassifyTemplateArgsTest = testing::TestWithParam<Case>;
TEST_P(ClassifyTemplateArgsTest, Classify) {
auto& params = GetParam();
Source::File file("", params.wgsl);
Lexer l(&file);
auto tokens = l.Lex();
ClassifyTemplateArguments(tokens);
auto types = utils::Transform(tokens, [&](const Token& t) { return t.type(); });
EXPECT_THAT(types, testing::ContainerEq(params.tokens));
}
INSTANTIATE_TEST_SUITE_P(NonTemplate,
ClassifyTemplateArgsTest,
testing::ValuesIn(std::vector<Case>{
{
"",
{T::kEOF},
},
{
"abc",
{T::kIdentifier, T::kEOF},
},
{
"a<b",
{T::kIdentifier, T::kLessThan, T::kIdentifier, T::kEOF},
},
{
"a>b",
{T::kIdentifier, T::kGreaterThan, T::kIdentifier, T::kEOF},
},
{
"(a<b)>c",
{
T::kParenLeft, // (
T::kIdentifier, // a
T::kLessThan, // <
T::kIdentifier, // b
T::kParenRight, // )
T::kGreaterThan, // >
T::kIdentifier, // c
T::kEOF,
},
},
{
"a<(b>c)",
{
T::kIdentifier, // a
T::kLessThan, // <
T::kParenLeft, // (
T::kIdentifier, // b
T::kGreaterThan, // >
T::kIdentifier, // c
T::kParenRight, // )
T::kEOF,
},
},
{
"a((b<c), d>(e))",
{
T::kIdentifier, // a
T::kParenLeft, // (
T::kParenLeft, // (
T::kIdentifier, // b
T::kLessThan, // <
T::kIdentifier, // c
T::kParenRight, // )
T::kComma, // ,
T::kIdentifier, // d
T::kGreaterThan, // >
T::kParenLeft, // (
T::kIdentifier, // e
T::kParenRight, // )
T::kParenRight, // )
T::kEOF,
},
},
{
"a<b[c>(d)]",
{
T::kIdentifier, // a
T::kLessThan, // <
T::kIdentifier, // b
T::kBracketLeft, // [
T::kIdentifier, // c
T::kGreaterThan, // >
T::kParenLeft, // (
T::kIdentifier, // d
T::kParenRight, // )
T::kBracketRight, // ]
T::kEOF,
},
},
{
"a<b;c>d()",
{
T::kIdentifier, // a
T::kLessThan, // <
T::kIdentifier, // b
T::kSemicolon, // ;
T::kIdentifier, // c
T::kGreaterThan, // >
T::kIdentifier, // d
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"if a < b {} else if c > d {}",
{
T::kIf, // a
T::kIdentifier, // a
T::kLessThan, // <
T::kIdentifier, // b
T::kBraceLeft, // {
T::kBraceRight, // }
T::kElse, // else
T::kIf, // if
T::kIdentifier, // c
T::kGreaterThan, // >
T::kIdentifier, // d
T::kBraceLeft, // {
T::kBraceRight, // }
T::kEOF,
},
},
{
"a<b&&c>d",
{
T::kIdentifier, // a
T::kLessThan, // <
T::kIdentifier, // b
T::kAndAnd, // &&
T::kPlaceholder, // <placeholder>
T::kIdentifier, // c
T::kGreaterThan, // >
T::kIdentifier, // d
T::kEOF,
},
},
{
"a<b||c>d",
{
T::kIdentifier, // a
T::kLessThan, // <
T::kIdentifier, // b
T::kOrOr, // ||
T::kIdentifier, // c
T::kGreaterThan, // >
T::kIdentifier, // d
T::kEOF,
},
},
{
"a<b<c||d>>",
{
T::kIdentifier, // a
T::kLessThan, // <
T::kIdentifier, // b
T::kLessThan, // <
T::kIdentifier, // c
T::kOrOr, // ||
T::kIdentifier, // d
T::kShiftRight, // >>
T::kPlaceholder, // <placeholder>
T::kEOF,
},
},
}));
INSTANTIATE_TEST_SUITE_P(Template,
ClassifyTemplateArgsTest,
testing::ValuesIn(std::vector<Case>{
{
"a<b>()",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kTemplateArgsRight, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"a<b>c",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kTemplateArgsRight, // >
T::kIdentifier, // c
T::kEOF,
},
},
{
"vec3<i32>",
{
T::kVec3, // vec3
T::kTemplateArgsLeft, // <
T::kI32, // i32
T::kTemplateArgsRight, // >
T::kEOF,
},
},
{
"vec3<i32>()",
{
T::kVec3, // vec3
T::kTemplateArgsLeft, // <
T::kI32, // i32
T::kTemplateArgsRight, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"array<vec3<i32>,5>",
{
T::kArray, // array
T::kTemplateArgsLeft, // <
T::kVec3, // vec3
T::kTemplateArgsLeft, // <
T::kI32, // i32
T::kTemplateArgsRight, // >
T::kComma, // ,
T::kIntLiteral, // 5
T::kTemplateArgsRight, // >
T::kEOF,
},
},
{
"a(b<c, d>(e))",
{
T::kIdentifier, // a
T::kParenLeft, // (
T::kIdentifier, // b
T::kTemplateArgsLeft, // <
T::kIdentifier, // c
T::kComma, // ,
T::kIdentifier, // d
T::kTemplateArgsRight, // >
T::kParenLeft, // (
T::kIdentifier, // e
T::kParenRight, // )
T::kParenRight, // )
T::kEOF,
},
},
{
"a<1+2>()",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIntLiteral, // 1
T::kPlus, // +
T::kIntLiteral, // 2
T::kTemplateArgsRight, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"a<1,b>()",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIntLiteral, // 1
T::kComma, // ,
T::kIdentifier, // b
T::kTemplateArgsRight, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"a<b,c>=d",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kComma, // ,
T::kIdentifier, // c
T::kTemplateArgsRight, // >
T::kEqual, // =
T::kIdentifier, // d
T::kEOF,
},
},
{
"a<b,c>=d>()",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kComma, // ,
T::kIdentifier, // c
T::kTemplateArgsRight, // >
T::kEqual, // =
T::kIdentifier, // d
T::kGreaterThan, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"a<b<c>>=",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kTemplateArgsLeft, // <
T::kIdentifier, // c
T::kTemplateArgsRight, // >
T::kTemplateArgsRight, // >
T::kEqual, // =
T::kEOF,
},
},
{
"a<b>c>()",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kTemplateArgsRight, // >
T::kIdentifier, // c
T::kGreaterThan, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"a<b<c>()",
{
T::kIdentifier, // a
T::kLessThan, // <
T::kIdentifier, // b
T::kTemplateArgsLeft, // <
T::kIdentifier, // c
T::kTemplateArgsRight, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"a<b<c>>()",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kTemplateArgsLeft, // <
T::kIdentifier, // c
T::kTemplateArgsRight, // >
T::kTemplateArgsRight, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"a<b<c>()>()",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kTemplateArgsLeft, // <
T::kIdentifier, // c
T::kTemplateArgsRight, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kTemplateArgsRight, // >
T::kParenLeft, // (
T::kParenRight, // )
T::kEOF,
},
},
{
"a<b>.c",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kTemplateArgsRight, // >
T::kPeriod, // .
T::kIdentifier, // c
T::kEOF,
},
},
{
"a<(b&&c)>d",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kParenLeft, // (
T::kIdentifier, // b
T::kAndAnd, // &&
T::kPlaceholder, // <placeholder>
T::kIdentifier, // c
T::kParenRight, // )
T::kTemplateArgsRight, // >
T::kIdentifier, // d
T::kEOF,
},
},
{
"a<(b||c)>d",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kParenLeft, // (
T::kIdentifier, // b
T::kOrOr, // ||
T::kIdentifier, // c
T::kParenRight, // )
T::kTemplateArgsRight, // >
T::kIdentifier, // d
T::kEOF,
},
},
{
"a<b<(c||d)>>",
{
T::kIdentifier, // a
T::kTemplateArgsLeft, // <
T::kIdentifier, // b
T::kTemplateArgsLeft, // <
T::kParenLeft, // (
T::kIdentifier, // c
T::kOrOr, // ||
T::kIdentifier, // d
T::kParenRight, // )
T::kTemplateArgsRight, // >
T::kTemplateArgsRight, // >
T::kEOF,
},
},
}));
} // namespace
} // namespace tint::reader::wgsl

4
src/tint/reader/wgsl/token.cc
View File

@ -70,12 +70,16 @@ std::string_view Token::TypeToName(Type type) {
return "="; return "=";
case Token::Type::kEqualEqual: case Token::Type::kEqualEqual:
return "=="; return "==";
case Token::Type::kTemplateArgsRight:
return "> (closing template argument list)";
case Token::Type::kGreaterThan: case Token::Type::kGreaterThan:
return ">"; return ">";
case Token::Type::kGreaterThanEqual: case Token::Type::kGreaterThanEqual:
return ">="; return ">=";
case Token::Type::kShiftRight: case Token::Type::kShiftRight:
return ">>"; return ">>";
case Token::Type::kTemplateArgsLeft:
return "< (opening template argument list)";
case Token::Type::kLessThan: case Token::Type::kLessThan:
return "<"; return "<";
case Token::Type::kLessThanEqual: case Token::Type::kLessThanEqual:

8
src/tint/reader/wgsl/token.h
View File

@ -32,7 +32,7 @@ class Token {
kError = -2, kError = -2,
/// Uninitialized token /// Uninitialized token
kUninitialized = 0, kUninitialized = 0,
/// Placeholder token which maybe fillled in later /// Placeholder token which maybe filled in later
kPlaceholder = 1, kPlaceholder = 1,
/// End of input string reached /// End of input string reached
kEOF, kEOF,
@ -80,12 +80,16 @@ class Token {
kEqual, kEqual,
/// A '==' /// A '=='
kEqualEqual, kEqualEqual,
/// A '>' (post template-args classification)
kTemplateArgsRight,
/// A '>' /// A '>'
kGreaterThan, kGreaterThan,
/// A '>=' /// A '>='
kGreaterThanEqual, kGreaterThanEqual,
/// A '>>' /// A '>>'
kShiftRight, kShiftRight,
/// A '<' (post template-args classification)
kTemplateArgsLeft,
/// A '<' /// A '<'
kLessThan, kLessThan,
/// A '<=' /// A '<='
@ -456,12 +460,10 @@ class Token {
std::variant<int64_t, double, std::string, std::string_view> value_; std::variant<int64_t, double, std::string, std::string_view> value_;
}; };
#ifndef NDEBUG
inline std::ostream& operator<<(std::ostream& out, Token::Type type) { inline std::ostream& operator<<(std::ostream& out, Token::Type type) {
out << Token::TypeToName(type); out << Token::TypeToName(type);
return out; return out;
} }
#endif // NDEBUG
} // namespace tint::reader::wgsl } // namespace tint::reader::wgsl