2021-06-19 17:34:35 +00:00
|
|
|
ByteAddressBuffer firstMatrix : register(t0, space0);
|
|
|
|
ByteAddressBuffer secondMatrix : register(t1, space0);
|
|
|
|
RWByteAddressBuffer resultMatrix : register(u2, space0);
|
|
|
|
cbuffer cbuffer_uniforms : register(b3, space0) {
|
|
|
|
uint4 uniforms[1];
|
|
|
|
};
|
|
|
|
|
|
|
|
float mm_readA(uint row, uint col) {
|
2021-07-13 12:18:13 +00:00
|
|
|
bool tint_tmp = (row < uniforms[0].x);
|
2021-06-19 17:34:35 +00:00
|
|
|
if (tint_tmp) {
|
2021-07-13 12:18:13 +00:00
|
|
|
tint_tmp = (col < uniforms[0].y);
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
2021-06-26 11:11:55 +00:00
|
|
|
if ((tint_tmp)) {
|
2021-07-13 12:18:13 +00:00
|
|
|
const float result = asfloat(firstMatrix.Load((4u * ((row * uniforms[0].y) + col))));
|
2021-06-19 17:34:35 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
return 0.0f;
|
|
|
|
}
|
|
|
|
|
|
|
|
float mm_readB(uint row, uint col) {
|
2021-07-13 12:18:13 +00:00
|
|
|
bool tint_tmp_1 = (row < uniforms[0].y);
|
2021-06-19 17:34:35 +00:00
|
|
|
if (tint_tmp_1) {
|
2021-07-13 12:18:13 +00:00
|
|
|
tint_tmp_1 = (col < uniforms[0].z);
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
2021-06-26 11:11:55 +00:00
|
|
|
if ((tint_tmp_1)) {
|
2021-07-13 12:18:13 +00:00
|
|
|
const float result = asfloat(secondMatrix.Load((4u * ((row * uniforms[0].z) + col))));
|
2021-06-19 17:34:35 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
return 0.0f;
|
|
|
|
}
|
|
|
|
|
|
|
|
void mm_write(uint row, uint col, float value) {
|
2021-07-13 12:18:13 +00:00
|
|
|
bool tint_tmp_2 = (row < uniforms[0].x);
|
2021-06-19 17:34:35 +00:00
|
|
|
if (tint_tmp_2) {
|
2021-07-13 12:18:13 +00:00
|
|
|
tint_tmp_2 = (col < uniforms[0].z);
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
2021-06-26 11:11:55 +00:00
|
|
|
if ((tint_tmp_2)) {
|
2021-07-13 12:18:13 +00:00
|
|
|
const uint index = (col + (row * uniforms[0].z));
|
2021-06-19 17:34:35 +00:00
|
|
|
resultMatrix.Store((4u * index), asuint(value));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static const uint RowPerThread = 4u;
|
|
|
|
static const uint ColPerThread = 4u;
|
|
|
|
static const uint TileAOuter = 64u;
|
|
|
|
static const uint TileBOuter = 64u;
|
|
|
|
static const uint TileInner = 64u;
|
2021-07-05 15:20:57 +00:00
|
|
|
groupshared float mm_Asub[64][64];
|
|
|
|
groupshared float mm_Bsub[64][64];
|
2021-06-19 17:34:35 +00:00
|
|
|
|
|
|
|
struct tint_symbol_1 {
|
|
|
|
uint3 local_id : SV_GroupThreadID;
|
|
|
|
uint local_invocation_index : SV_GroupIndex;
|
|
|
|
uint3 global_id : SV_DispatchThreadID;
|
|
|
|
};
|
|
|
|
|
2021-08-04 22:15:28 +00:00
|
|
|
void main_inner(uint3 local_id, uint3 global_id, uint local_invocation_index) {
|
2021-07-30 14:08:06 +00:00
|
|
|
{
|
|
|
|
for(uint idx = local_invocation_index; (idx < 4096u); idx = (idx + 256u)) {
|
|
|
|
const uint i = (idx / 64u);
|
|
|
|
const uint i_1 = (idx % 64u);
|
|
|
|
mm_Asub[i][i_1] = 0.0f;
|
|
|
|
mm_Bsub[i][i_1] = 0.0f;
|
2021-07-05 17:18:16 +00:00
|
|
|
}
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
|
|
|
GroupMemoryBarrierWithGroupSync();
|
|
|
|
const uint tileRow = (local_id.y * RowPerThread);
|
|
|
|
const uint tileCol = (local_id.x * ColPerThread);
|
|
|
|
const uint globalRow = (global_id.y * RowPerThread);
|
|
|
|
const uint globalCol = (global_id.x * ColPerThread);
|
2021-07-13 12:18:13 +00:00
|
|
|
const uint numTiles = (((uniforms[0].y - 1u) / TileInner) + 1u);
|
2021-07-05 15:20:57 +00:00
|
|
|
float acc[16] = (float[16])0;
|
2021-06-19 17:34:35 +00:00
|
|
|
float ACached = 0.0f;
|
2021-07-05 15:20:57 +00:00
|
|
|
float BCached[4] = (float[4])0;
|
2021-06-19 17:34:35 +00:00
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint index = 0u; (index < (RowPerThread * ColPerThread)); index = (index + 1u)) {
|
2021-07-05 15:20:57 +00:00
|
|
|
acc[index] = 0.0f;
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
const uint ColPerThreadA = (TileInner / 16u);
|
|
|
|
const uint tileColA = (local_id.x * ColPerThreadA);
|
|
|
|
const uint RowPerThreadB = (TileInner / 16u);
|
|
|
|
const uint tileRowB = (local_id.y * RowPerThreadB);
|
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint t = 0u; (t < numTiles); t = (t + 1u)) {
|
2021-06-19 17:34:35 +00:00
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint innerRow = 0u; (innerRow < RowPerThread); innerRow = (innerRow + 1u)) {
|
2021-06-19 17:34:35 +00:00
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint innerCol = 0u; (innerCol < ColPerThreadA); innerCol = (innerCol + 1u)) {
|
2021-06-19 17:34:35 +00:00
|
|
|
const uint inputRow = (tileRow + innerRow);
|
|
|
|
const uint inputCol = (tileColA + innerCol);
|
2021-07-05 15:20:57 +00:00
|
|
|
mm_Asub[inputRow][inputCol] = mm_readA((globalRow + innerRow), ((t * TileInner) + inputCol));
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint innerRow = 0u; (innerRow < RowPerThreadB); innerRow = (innerRow + 1u)) {
|
2021-06-19 17:34:35 +00:00
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint innerCol = 0u; (innerCol < ColPerThread); innerCol = (innerCol + 1u)) {
|
2021-06-19 17:34:35 +00:00
|
|
|
const uint inputRow = (tileRowB + innerRow);
|
|
|
|
const uint inputCol = (tileCol + innerCol);
|
2021-07-05 15:20:57 +00:00
|
|
|
mm_Bsub[innerCol][inputCol] = mm_readB(((t * TileInner) + inputRow), (globalCol + innerCol));
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
GroupMemoryBarrierWithGroupSync();
|
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint k = 0u; (k < TileInner); k = (k + 1u)) {
|
2021-06-19 17:34:35 +00:00
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint inner = 0u; (inner < ColPerThread); inner = (inner + 1u)) {
|
2021-07-05 15:20:57 +00:00
|
|
|
BCached[inner] = mm_Bsub[k][(tileCol + inner)];
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint innerRow = 0u; (innerRow < RowPerThread); innerRow = (innerRow + 1u)) {
|
2021-07-05 15:20:57 +00:00
|
|
|
ACached = mm_Asub[(tileRow + innerRow)][k];
|
2021-06-19 17:34:35 +00:00
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint innerCol = 0u; (innerCol < ColPerThread); innerCol = (innerCol + 1u)) {
|
2021-06-19 17:34:35 +00:00
|
|
|
const uint index = ((innerRow * ColPerThread) + innerCol);
|
2021-07-05 15:20:57 +00:00
|
|
|
acc[index] = (acc[index] + (ACached * BCached[innerCol]));
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
GroupMemoryBarrierWithGroupSync();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint innerRow = 0u; (innerRow < RowPerThread); innerRow = (innerRow + 1u)) {
|
2021-06-19 17:34:35 +00:00
|
|
|
{
|
2021-07-08 21:23:33 +00:00
|
|
|
for(uint innerCol = 0u; (innerCol < ColPerThread); innerCol = (innerCol + 1u)) {
|
2021-06-19 17:34:35 +00:00
|
|
|
const uint index = ((innerRow * ColPerThread) + innerCol);
|
2021-07-05 15:20:57 +00:00
|
|
|
mm_write((globalRow + innerRow), (globalCol + innerCol), acc[index]);
|
2021-06-19 17:34:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-08-04 22:15:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
[numthreads(16, 16, 1)]
|
|
|
|
void main(tint_symbol_1 tint_symbol) {
|
|
|
|
main_inner(tint_symbol.local_id, tint_symbol.global_id, tint_symbol.local_invocation_index);
|
2021-06-19 17:34:35 +00:00
|
|
|
return;
|
|
|
|
}
|