From 9346afbcac30524cb377341662216bfe294e85f5 Mon Sep 17 00:00:00 2001 From: Jack Elliott Date: Thu, 20 Aug 2026 08:29:19 +1200 Subject: [PATCH 1/2] [HLSL] Add LinAlg matrix accumulation contention coverage The existing matrix accumulation tests run a single Wave in a single group, so they demonstrate that an accumulation lands but never that it lands atomically. An implementation that dropped or duplicated concurrent accumulations would pass all of them. These four cases run several Waves across several groups against one destination. Each Wave contributes a distinct amount rather than the same one. Wave w repeats its additions w plus one times in the descriptor cases, and scales the value it accumulates by w plus one in the group-shared cases, so the expected total is weighted by Wave index. Had every Wave contributed the same amount the total would only pin the number of additions applied, and a dropped update from one Wave would be cancelled exactly by a duplicated update from another. Weighting also rejects a lowering that assumes the participating Waves hold the same operand and applies one representative value scaled by the participant count. Both shaders already selected a single Wave with a hardcoded index test, so the change is to compare against an ACTIVE_WAVE_COUNT define instead and let the existing helpers take a Wave count and a dispatch width. The existing callers pass one Wave and one group, which is what they did before. The group-shared oracle is now computed per element as InitialValue + WaveWeightSum * (AccumulateStartingValue + Index), where WaveWeightSum is the sum of one through the active Wave count. Both that and the descriptor count reduce to their previous values when one Wave is active, so the two existing cases are unchanged. The counts are chosen so every intermediate value is exactly representable, which is what lets the result be compared for equality rather than within a tolerance. The descriptor cases accumulate a fill of seven or one across eighty additions, and the group-shared cases reach at most three hundred and twenty-seven, both well inside the range where F16 represents integers exactly. Because every partial sum is exact, the F32 case cannot depend on the order the hardware happens to apply the additions, which is what OrderInvariant in its name refers to. The group-shared helper previously rejected anything that was not F16 through one compound condition covering four unrelated parameters. It now accepts I32 as well and reports each rejected parameter separately, so a mistake in a future case says which parameter was wrong. Verified by making every Wave contribute the same amount while leaving the oracles expecting the weighted total: the two cases that run on WARP fail, and every other test is unaffected, including the two single-Wave cases the weighting is designed not to disturb. The two I32 cases skip on WARP because it reports no I32 accumulation support. This does not prove the Waves physically overlap in time. Nothing forces an implementation to run them concurrently, so one that serialises them passes. WARP reports a Wave size of four, so the contention depth actually exercised here is low and real depth is untested until this runs on hardware. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83725f5d-8e98-4c1d-91ee-ad47629e007b Assisted-by: GitHub Copilot --- .../clang/unittests/HLSLExec/LinAlgTests.cpp | 256 +++++++++++++++--- 1 file changed, 222 insertions(+), 34 deletions(-) diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index e77f0a6064..bf004d4782 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -3122,6 +3122,8 @@ class DxilConf_SM610_LinAlg { TEST_METHOD(StoreDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView); TEST_METHOD(SplatStore_Wave_16x16_F16); TEST_METHOD(AccumulateDescriptor_Wave_16x16_F16); + TEST_METHOD(AccumulateDescriptorContention_Wave_4x8_I32); + TEST_METHOD(AccumulateDescriptorContention_Wave_4x8_F32_OrderInvariant); // Load/Store/Accumulate Memory TEST_METHOD(LoadMemory_Wave_16x16_F16); @@ -3130,6 +3132,8 @@ class DxilConf_SM610_LinAlg { TEST_METHOD(LoadStoreMemory_Wave_4x8_F16_RowMajorOffsetPadded); TEST_METHOD(LoadStoreMemory_Wave_4x8_F32_ColumnMajorOffsetPadded); TEST_METHOD(LoadStoreMemory_ThreadGroup_4x8_F16); + TEST_METHOD(AccumulateMemoryContention_Wave_4x8_F16); + TEST_METHOD(AccumulateMemoryContention_Wave_4x8_I32); // Element access TEST_METHOD(ElementAccess_Wave_16x16_F16); @@ -3928,6 +3932,8 @@ void DxilConf_SM610_LinAlg::SplatStore_Wave_16x16_F16() { SelectedWaveSize); } +static constexpr UINT DescriptorAccumulatesPerWave = 2; + static const char AccumulateDescriptorShader[] = R"( #define USE_ACC 2 @@ -3941,7 +3947,7 @@ static const char AccumulateDescriptorShader[] = R"( #endif [numthreads(NUMTHREADS, 1, 1)] void main() { - if (GetGroupWaveIndex() != 0) + if (GetGroupWaveIndex() >= ACTIVE_WAVE_COUNT) return; __builtin_LinAlgMatrix @@ -3949,21 +3955,31 @@ static const char AccumulateDescriptorShader[] = R"( Mat; __builtin_LinAlg_MatrixLoadFromDescriptor( Mat, Input, 0, STRIDE, LAYOUT, 128); - __builtin_LinAlg_MatrixAccumulateToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); - __builtin_LinAlg_MatrixAccumulateToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); + // Repeating once per Wave index makes each Wave contribute a distinct + // amount, so a dropped update cannot cancel a duplicated one. + for (uint Repeat = 0; Repeat <= GetGroupWaveIndex(); ++Repeat) { + __builtin_LinAlg_MatrixAccumulateToDescriptor( + Mat, Output, 0, STRIDE, LAYOUT, 128); + __builtin_LinAlg_MatrixAccumulateToDescriptor( + Mat, Output, 0, STRIDE, LAYOUT, 128); + } } )"; static void runAccumulateDescriptor(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, const MatrixParams &Params, int FillValue, - bool Verbose, UINT ForcedWaveSize = 0) { + bool Verbose, UINT ForcedWaveSize = 0, + UINT ActiveWaveCount = 1, + UINT DispatchX = 1) { const size_t NumElements = Params.totalElements(); const size_t BufferSize = Params.totalBytes(); + const UINT AccumulationCount = DescriptorAccumulatesPerWave * + (ActiveWaveCount * (ActiveWaveCount + 1) / 2) * + DispatchX; std::stringstream ExtraDefs; + ExtraDefs << " -DACTIVE_WAVE_COUNT=" << ActiveWaveCount; if (ForcedWaveSize != 0) ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; @@ -3973,10 +3989,16 @@ static void runAccumulateDescriptor(ID3D12Device *Device, Verbose); auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, - static_cast(FillValue) * 2, false); + static_cast(FillValue) * + static_cast(AccumulationCount), + false); + hlsl_test::LogCommentFmt( + L"Descriptor accumulation issues %u atomic matrix additions in total " + L"across %u Waves in each of %u groups", + AccumulationCount, ActiveWaveCount, DispatchX); auto Op = createComputeOp(AccumulateDescriptorShader, "cs_6_10", - "SRV(t0), UAV(u1)", Args.c_str()); + "SRV(t0), UAV(u1)", Args.c_str(), DispatchX); addSRVBuffer(Op.get(), "Input", BufferSize, "byname"); addUAVBuffer(Op.get(), "Output", BufferSize, true); addRootView(Op.get(), 0, "Input"); @@ -4025,6 +4047,50 @@ void DxilConf_SM610_LinAlg::AccumulateDescriptor_Wave_16x16_F16() { SelectedWaveSize); } +static void runAccumulateDescriptorContention( + ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, + ComponentType CompType, int FillValue, LPCWSTR CaseName, bool Verbose) { + MatrixParams Params = {}; + Params.CompType = CompType; + Params.M = 4; + Params.N = 8; + Params.Use = MatrixUse::Accumulator; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 512; + Params.Enable16Bit = false; + + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(Device, Params, {Params.Use}, CaseName, + SelectedWaveSize)) + return; + if (!accumulateStoreApplicable( + Device, Params.CompType, + linalg_test::AtomicDestination::RWByteAddressBuffer, CaseName)) + return; + + constexpr UINT ActiveWaveCount = 4; + constexpr UINT DispatchX = 4; + Params.NumThreads = static_cast(SelectedWaveSize * ActiveWaveCount); + + runAccumulateDescriptor(Device, DxcSupport, Params, FillValue, Verbose, + SelectedWaveSize, ActiveWaveCount, DispatchX); +} + +void DxilConf_SM610_LinAlg::AccumulateDescriptorContention_Wave_4x8_I32() { + runAccumulateDescriptorContention( + D3DDevice, DxcSupport, ComponentType::I32, /*FillValue=*/7, + L"AccumulateDescriptorContention_Wave_4x8_I32", VerboseLogging); +} + +void DxilConf_SM610_LinAlg:: + AccumulateDescriptorContention_Wave_4x8_F32_OrderInvariant() { + runAccumulateDescriptorContention( + D3DDevice, DxcSupport, ComponentType::F32, /*FillValue=*/1, + L"AccumulateDescriptorContention_Wave_4x8_F32_OrderInvariant", + VerboseLogging); +} + // Element access constructs a wave-scope matrix and then reads or writes its // components, so applicability is exactly MatrixConstruction for the tile the // case declares. D3D12LinearAlgebraRuntimeFeatureSupport.md guarantees only @@ -7536,7 +7602,7 @@ static const char GroupSharedAccumulateShader[] = R"( GroupMemoryBarrierWithGroupSync(); - if (GetGroupWaveIndex() == 0) { + if (GetGroupWaveIndex() < ACTIVE_WAVE_COUNT) { __builtin_LinAlgMatrix [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] Mat; @@ -7545,7 +7611,8 @@ static const char GroupSharedAccumulateShader[] = R"( uint2 Coord = __builtin_LinAlg_MatrixGetCoordinate(Mat, I); __builtin_LinAlg_MatrixSetElement( Mat, Mat, I, - (ELEM_TYPE)(ACCUMULATE_START + Coord.x * N_DIM + Coord.y)); + (ELEM_TYPE)((GetGroupWaveIndex() + 1) * + (ACCUMULATE_START + Coord.x * N_DIM + Coord.y))); } __builtin_LinAlg_MatrixAccumulateToMemory( Mat, GsData, COMP_TYPE, MEM_OFFSET, MEM_STRIDE, MEM_LAYOUT); @@ -7564,16 +7631,32 @@ static void runGroupSharedAccumulate( ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, const MatrixParams &Params, const cpu_oracle::MatrixBufferLayout &MemoryLayout, uint32_t InitialValue, - uint32_t AccumulateStartingValue, bool Verbose, UINT ForcedWaveSize = 0) { - if (!Device || Params.CompType != ComponentType::F16 || - Params.Scope != MatrixScope::Wave || - Params.Use != MatrixUse::Accumulator || - InitialValue > - (std::numeric_limits::max)() - AccumulateStartingValue) { - VERIFY_IS_TRUE(false, "Invalid group-shared accumulate parameters"); + uint32_t AccumulateStartingValue, bool Verbose, UINT ForcedWaveSize = 0, + UINT ActiveWaveCount = 1) { + if (!Device) { + hlsl_test::LogErrorFmt(L"Group-shared accumulation has no device"); + VERIFY_FAIL(L"Invalid group-shared accumulation device"); + return; + } + if (Params.CompType != ComponentType::F16 && + Params.CompType != ComponentType::I32) { + hlsl_test::LogErrorFmt( + L"Group-shared accumulation does not support component type %u", + static_cast(Params.CompType)); + VERIFY_FAIL(L"Unsupported group-shared accumulation component type"); + return; + } + if (Params.Scope != MatrixScope::Wave) { + hlsl_test::LogErrorFmt(L"Group-shared accumulation requires Wave scope"); + VERIFY_FAIL(L"Invalid group-shared accumulation scope"); + return; + } + if (Params.Use != MatrixUse::Accumulator) { + hlsl_test::LogErrorFmt( + L"Group-shared accumulation requires Accumulator use"); + VERIFY_FAIL(L"Invalid group-shared accumulation matrix use"); return; } - size_t BufferSize; UINT NumElements; if (!getGroupSharedBufferDescription(Params, MemoryLayout, BufferSize, @@ -7583,24 +7666,74 @@ static void runGroupSharedAccumulate( } const size_t MatrixElements = Params.totalElements(); - std::optional InitialMatrix = - cpu_oracle::makeTypedMatrix( - Params.M, Params.N, - std::vector( - MatrixElements, HLSLHalf_t(static_cast(InitialValue)))); - std::optional ExpectedMatrix = - cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N, - InitialValue + AccumulateStartingValue); + // Wave w contributes w + 1 times its element value, matching the shader, so + // the weighted total distinguishes a dropped update from a duplicated one. + const uint32_t WaveWeightSum = ActiveWaveCount * (ActiveWaveCount + 1) / 2; + std::optional InitialMatrix; + std::optional ExpectedMatrix; + if (Params.CompType == ComponentType::F16) { + std::vector InitialValues( + MatrixElements, HLSLHalf_t(static_cast(InitialValue))); + std::vector ExpectedValues; + ExpectedValues.reserve(MatrixElements); + for (size_t I = 0; I < MatrixElements; ++I) { + const uint32_t Value = + InitialValue + + WaveWeightSum * (AccumulateStartingValue + static_cast(I)); + ExpectedValues.emplace_back(static_cast(Value)); + } + InitialMatrix = cpu_oracle::makeTypedMatrix(Params.M, Params.N, + std::move(InitialValues)); + ExpectedMatrix = cpu_oracle::makeTypedMatrix(Params.M, Params.N, + std::move(ExpectedValues)); + } else { + std::vector InitialValues(MatrixElements, + static_cast(InitialValue)); + std::vector ExpectedValues; + ExpectedValues.reserve(MatrixElements); + for (size_t I = 0; I < MatrixElements; ++I) { + const uint32_t Value = + InitialValue + + WaveWeightSum * (AccumulateStartingValue + static_cast(I)); + ExpectedValues.push_back(static_cast(Value)); + } + InitialMatrix = cpu_oracle::makeTypedMatrix(Params.M, Params.N, + std::move(InitialValues)); + ExpectedMatrix = cpu_oracle::makeTypedMatrix(Params.M, Params.N, + std::move(ExpectedValues)); + } + if (!InitialMatrix || !ExpectedMatrix) { + hlsl_test::LogErrorFmt( + L"Failed to build %ux%u group-shared accumulation matrices for " + L"component type %u", + Params.M, Params.N, static_cast(Params.CompType)); + VERIFY_FAIL(L"Invalid group-shared accumulation matrices"); + return; + } + std::optional> Initial = makeGroupSharedTypedBuffer(Params.CompType, BufferSize, 90); std::optional> Expected = makeGroupSharedTypedBuffer(Params.CompType, BufferSize, 90); - if (!InitialMatrix.has_value() || !ExpectedMatrix.has_value() || - !Initial.has_value() || !Expected.has_value() || - !cpu_oracle::writeMatrixBuffer(*InitialMatrix, MemoryLayout, *Initial) || - !cpu_oracle::writeMatrixBuffer(*ExpectedMatrix, MemoryLayout, + if (!Initial || !Expected) { + hlsl_test::LogErrorFmt( + L"Failed to allocate %zu-byte group-shared accumulation buffers for " + L"component type %u", + BufferSize, static_cast(Params.CompType)); + VERIFY_FAIL(L"Invalid group-shared accumulation buffers"); + return; + } + if (!cpu_oracle::writeMatrixBuffer(*InitialMatrix, MemoryLayout, *Initial)) { + hlsl_test::LogErrorFmt( + L"Failed to write the initial group-shared accumulation matrix"); + VERIFY_FAIL(L"Invalid initial group-shared accumulation layout"); + return; + } + if (!cpu_oracle::writeMatrixBuffer(*ExpectedMatrix, MemoryLayout, *Expected)) { - VERIFY_IS_TRUE(false, "Failed to build group-shared accumulate oracle"); + hlsl_test::LogErrorFmt( + L"Failed to write the expected group-shared accumulation matrix"); + VERIFY_FAIL(L"Invalid expected group-shared accumulation layout"); return; } @@ -7611,6 +7744,7 @@ static void runGroupSharedAccumulate( ExtraDefs << " -DMEM_STRIDE=" << MemoryLayout.StrideBytes / ElementBytes; ExtraDefs << " -DMEM_LAYOUT=" << static_cast(MemoryLayout.Layout); ExtraDefs << " -DACCUMULATE_START=" << AccumulateStartingValue; + ExtraDefs << " -DACTIVE_WAVE_COUNT=" << ActiveWaveCount; if (ForcedWaveSize != 0) ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; @@ -7634,11 +7768,16 @@ static void runGroupSharedAccumulate( MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); + std::wstringstream PublicRule; + PublicRule << L"MatrixAccumulateToMemory atomically adds each logical " + L"matrix value from " + << ActiveWaveCount << L" active Wave"; + if (ActiveWaveCount != 1) + PublicRule << L"s"; + PublicRule << L", preserving every padding byte and guard"; VERIFY_IS_TRUE(verifyGroupSharedTypedBuffer( Params.CompType, OutData.data(), OutData.size(), *Expected, - L"MatrixAccumulateToMemory adds each logical matrix value to 12 while " - L"preserving exact padding and guards", - Verbose)); + PublicRule.str().c_str(), Verbose)); } static void runPaddedGroupSharedAccumulateCase( @@ -7714,6 +7853,55 @@ static void runPaddedGroupSharedAccumulateCase( SelectedWaveSize); } +static void runGroupSharedAccumulateContention( + ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, + ComponentType CompType, LPCWSTR CaseName, bool Verbose) { + MatrixParams Params = {}; + Params.CompType = CompType; + Params.M = 4; + Params.N = 8; + Params.Use = MatrixUse::Accumulator; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 512; + Params.Enable16Bit = CompType == ComponentType::F16; + + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(Device, Params, {Params.Use}, CaseName, + SelectedWaveSize)) + return; + if (!accumulateStoreApplicable(Device, Params.CompType, + linalg_test::AtomicDestination::GroupShared, + CaseName)) + return; + + constexpr UINT ActiveWaveCount = 4; + Params.NumThreads = static_cast(SelectedWaveSize * ActiveWaveCount); + + const size_t ElementBytes = elementSize(CompType); + const cpu_oracle::MatrixBufferLayout Memory = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/4 * ElementBytes, + /*StrideBytes=*/Params.N * ElementBytes + 8, + }; + runGroupSharedAccumulate(Device, DxcSupport, Params, Memory, + /*InitialValue=*/7, + /*AccumulateStartingValue=*/1, Verbose, + SelectedWaveSize, ActiveWaveCount); +} + +void DxilConf_SM610_LinAlg::AccumulateMemoryContention_Wave_4x8_F16() { + runGroupSharedAccumulateContention(D3DDevice, DxcSupport, ComponentType::F16, + L"AccumulateMemoryContention_Wave_4x8_F16", + VerboseLogging); +} + +void DxilConf_SM610_LinAlg::AccumulateMemoryContention_Wave_4x8_I32() { + runGroupSharedAccumulateContention(D3DDevice, DxcSupport, ComponentType::I32, + L"AccumulateMemoryContention_Wave_4x8_I32", + VerboseLogging); +} + static const char ConvertShader[] = R"( #define CT_F16 8 #define CT_F32 9 From 2b56bb465486625fe641ce2523d8b7db9c0e2683 Mon Sep 17 00:00:00 2001 From: Jack Elliott Date: Sat, 22 Aug 2026 11:04:26 +1200 Subject: [PATCH 2/2] [HLSL] Weight LinAlg descriptor contention by a global Wave index The descriptor cases varied how many times each Wave added the same matrix rather than what each Wave added. Every one of the additions was therefore identical, so the total pinned only how many were applied and a dropped update could still be cancelled exactly by a duplicated one. That is the property the cases were added to reject, so they did not reject it. Each Wave now scales what it accumulates and issues a fixed two additions. The scale is one plus a global index built from SV_GroupID and the Wave index, so it is distinct across every Wave in every group and the total pins which Waves landed. A Wave index on its own would repeat in each group, and an update dropped in one group could be cancelled by a duplicate in another. The group-shared shader already scaled the value it accumulated rather than repeating it, and its destination is group-shared so contention is confined to one group where Wave indices are already distinct. It is unchanged. The oracle becomes the fill times two times the sum of one through the number of contending Waves, which is sixteen Waves and so 272 times the fill. The I32 case reaches 1904 and the F32 case 272, and every partial sum along the way is a smaller integer, so both stay exact and the results can still be compared for equality. One Wave in one group gives a scale of one and two additions, which is what the expression produced before, so the existing single-Wave case is unchanged by construction rather than by a preprocessor guard. Verified by dropping the SV_GroupID term so the scale repeats in each group: the F32 contention case reaches 80 times the fill instead of 272 and fails, while the single-Wave case is unaffected because its scale is one either way. The I32 contention case skips on WARP, which reports no I32 accumulation support. Reported by Copilot code review on the pull request. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83725f5d-8e98-4c1d-91ee-ad47629e007b Assisted-by: GitHub Copilot --- .../clang/unittests/HLSLExec/LinAlgTests.cpp | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index bf004d4782..325d6eba06 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -3946,7 +3946,7 @@ static const char AccumulateDescriptorShader[] = R"( [WaveSize(4, 128)] #endif [numthreads(NUMTHREADS, 1, 1)] - void main() { + void main(uint3 GroupID : SV_GroupID) { if (GetGroupWaveIndex() >= ACTIVE_WAVE_COUNT) return; @@ -3955,14 +3955,22 @@ static const char AccumulateDescriptorShader[] = R"( Mat; __builtin_LinAlg_MatrixLoadFromDescriptor( Mat, Input, 0, STRIDE, LAYOUT, 128); - // Repeating once per Wave index makes each Wave contribute a distinct - // amount, so a dropped update cannot cancel a duplicated one. - for (uint Repeat = 0; Repeat <= GetGroupWaveIndex(); ++Repeat) { - __builtin_LinAlg_MatrixAccumulateToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); - __builtin_LinAlg_MatrixAccumulateToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); + + // The index is distinct across every Wave in every group, so the total + // pins which Waves landed rather than only how many additions were + // applied. A Wave index alone would repeat in each group. + uint Weight = GroupID.x * ACTIVE_WAVE_COUNT + GetGroupWaveIndex() + 1; + for (uint I = 0; I < __builtin_LinAlg_MatrixLength(Mat); ++I) { + ELEM_TYPE Elem; + __builtin_LinAlg_MatrixGetElement(Elem, Mat, I); + __builtin_LinAlg_MatrixSetElement( + Mat, Mat, I, (ELEM_TYPE)(Weight * Elem)); } + + __builtin_LinAlg_MatrixAccumulateToDescriptor( + Mat, Output, 0, STRIDE, LAYOUT, 128); + __builtin_LinAlg_MatrixAccumulateToDescriptor( + Mat, Output, 0, STRIDE, LAYOUT, 128); } )"; @@ -3974,9 +3982,10 @@ static void runAccumulateDescriptor(ID3D12Device *Device, UINT DispatchX = 1) { const size_t NumElements = Params.totalElements(); const size_t BufferSize = Params.totalBytes(); - const UINT AccumulationCount = DescriptorAccumulatesPerWave * - (ActiveWaveCount * (ActiveWaveCount + 1) / 2) * - DispatchX; + const UINT ContendingWaves = ActiveWaveCount * DispatchX; + const UINT AccumulationCount = DescriptorAccumulatesPerWave * ContendingWaves; + const UINT WeightSum = ContendingWaves * (ContendingWaves + 1) / 2; + const UINT WeightedAccumulation = DescriptorAccumulatesPerWave * WeightSum; std::stringstream ExtraDefs; ExtraDefs << " -DACTIVE_WAVE_COUNT=" << ActiveWaveCount; @@ -3990,13 +3999,13 @@ static void runAccumulateDescriptor(ID3D12Device *Device, auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, static_cast(FillValue) * - static_cast(AccumulationCount), + static_cast(WeightedAccumulation), false); hlsl_test::LogCommentFmt( L"Descriptor accumulation issues %u atomic matrix additions in total " - L"across %u Waves in each of %u groups", - AccumulationCount, ActiveWaveCount, DispatchX); + L"across %u Waves in each of %u groups, weighted to %u times the fill", + AccumulationCount, ActiveWaveCount, DispatchX, WeightedAccumulation); auto Op = createComputeOp(AccumulateDescriptorShader, "cs_6_10", "SRV(t0), UAV(u1)", Args.c_str(), DispatchX); addSRVBuffer(Op.get(), "Input", BufferSize, "byname");