diff --git a/tools/clang/unittests/HLSLExec/HlslExecTestUtils.cpp b/tools/clang/unittests/HLSLExec/HlslExecTestUtils.cpp index 8715ea1ac4..9453eb13b5 100644 --- a/tools/clang/unittests/HLSLExec/HlslExecTestUtils.cpp +++ b/tools/clang/unittests/HLSLExec/HlslExecTestUtils.cpp @@ -725,12 +725,15 @@ void addRootView(st::ShaderOp *Op, UINT Index, const char *ResName) { std::shared_ptr runShaderOp(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, std::unique_ptr Op, - st::ShaderOpTest::TInitCallbackFn InitCallback) { + st::ShaderOpTest::TInitCallbackFn InitCallback, + st::ShaderOpTest::TCommandCallbackFn PostExecuteCallback) { auto OpSet = std::make_shared(); OpSet->ShaderOps.push_back(std::move(Op)); return st::RunShaderOpTestAfterParse( - Device, DxcSupport, nullptr, std::move(InitCallback), std::move(OpSet)); + Device, DxcSupport, nullptr, std::move(InitCallback), + /*pShaderCallback=*/nullptr, std::move(PostExecuteCallback), + std::move(OpSet)); } void compileShader(dxc::SpecificDllLoader &DxcSupport, const char *Source, @@ -796,3 +799,63 @@ void compileShader(dxc::SpecificDllLoader &DxcSupport, const char *Source, VERIFY_SUCCEEDED(HR); } } + +#if HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE +UINT getLinAlgMatrixByteSize(ID3D12Device *Device, UINT NumRows, + UINT NumColumns, + D3D12_LINEAR_ALGEBRA_DATATYPE DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT Layout, + UINT Stride) { + CComPtr DevicePreview; + VERIFY_SUCCEEDED(Device->QueryInterface(IID_PPV_ARGS(&DevicePreview))); + + D3D12_LINEAR_ALGEBRA_MATRIX_CONVERSION_DEST_INFO Info = {}; + Info.DestSize = 0; + Info.DestLayout = Layout; + Info.DestStride = Stride; + Info.NumRows = NumRows; + Info.NumColumns = NumColumns; + Info.DestDataType = DataType; + DevicePreview->GetLinearAlgebraMatrixConversionDestinationInfo(&Info); + return Info.DestSize; +} + +void recordLinAlgMatrixConversion( + ID3D12GraphicsCommandList *List, ID3D12Resource *SrcBuffer, UINT SrcSize, + ID3D12Resource *DestBuffer, UINT DestSize, UINT NumRows, UINT NumColumns, + D3D12_LINEAR_ALGEBRA_DATATYPE DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT SrcLayout, UINT SrcStride, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT DestLayout, UINT DestStride) { + CComPtr PreviewList; + VERIFY_SUCCEEDED(List->QueryInterface(IID_PPV_ARGS(&PreviewList))); + + // Per the linear-algebra spec, ConvertLinearAlgebraMatrix (legacy barriers) + // requires the source buffer in NON_PIXEL_SHADER_RESOURCE and the destination + // in UNORDERED_ACCESS. The caller passes both in UNORDERED_ACCESS (the + // ShaderOp default UAV state), so transition the source to the required read + // state; the destination is already in UNORDERED_ACCESS. + D3D12_RESOURCE_BARRIER Barrier = {}; + Barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + Barrier.Transition.pResource = SrcBuffer; + Barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + Barrier.Transition.StateBefore = D3D12_RESOURCE_STATE_UNORDERED_ACCESS; + Barrier.Transition.StateAfter = + D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE; + List->ResourceBarrier(1, &Barrier); + + D3D12_LINEAR_ALGEBRA_MATRIX_CONVERSION_INFO Info = {}; + Info.DestInfo.DestSize = DestSize; + Info.DestInfo.DestLayout = DestLayout; + Info.DestInfo.DestStride = DestStride; + Info.DestInfo.NumRows = NumRows; + Info.DestInfo.NumColumns = NumColumns; + Info.DestInfo.DestDataType = DataType; + Info.SrcInfo.SrcSize = SrcSize; + Info.SrcInfo.SrcDataType = DataType; + Info.SrcInfo.SrcLayout = SrcLayout; + Info.SrcInfo.SrcStride = SrcStride; + Info.DataDesc.DestVA = DestBuffer->GetGPUVirtualAddress(); + Info.DataDesc.SrcVA = SrcBuffer->GetGPUVirtualAddress(); + PreviewList->ConvertLinearAlgebraMatrix(&Info, 1); +} +#endif // HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE diff --git a/tools/clang/unittests/HLSLExec/HlslExecTestUtils.h b/tools/clang/unittests/HLSLExec/HlslExecTestUtils.h index 435a091a54..863fd9eac6 100644 --- a/tools/clang/unittests/HLSLExec/HlslExecTestUtils.h +++ b/tools/clang/unittests/HLSLExec/HlslExecTestUtils.h @@ -99,7 +99,8 @@ void addRootView(st::ShaderOp *Op, UINT Index, const char *ResName); std::shared_ptr runShaderOp(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, std::unique_ptr Op, - st::ShaderOpTest::TInitCallbackFn InitCallback = nullptr); + st::ShaderOpTest::TInitCallbackFn InitCallback = nullptr, + st::ShaderOpTest::TCommandCallbackFn PostExecuteCallback = nullptr); /// Compiles an HLSL shader using the DXC API to verify it is well-formed. /// Fails the test on compile error. @@ -107,4 +108,40 @@ void compileShader(dxc::SpecificDllLoader &DxcSupport, const char *Source, const char *Target, const std::string &Args, bool VerboseLogging = false); +// Host-side linear-algebra matrix-conversion helpers. These need the D3D12 +// linear-algebra API (the D3D12_LINEAR_ALGEBRA_* types and the conversion +// methods on ID3D12DevicePreview / ID3D12GraphicsCommandListPreview). When +// absent, these helpers and the tests using them are compiled out (they Skip at +// runtime). +#if defined(DIRECT3D_LINEAR_ALGEBRA) +#define HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE 1 +#else +#define HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE 0 +#endif + +#if HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE +/// Query the number of bytes required to store an NumRows x NumColumns matrix +/// of the given datatype in the specified device layout. +UINT getLinAlgMatrixByteSize(ID3D12Device *Device, UINT NumRows, + UINT NumColumns, + D3D12_LINEAR_ALGEBRA_DATATYPE DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT Layout, + UINT Stride); + +/// Record a GPU matrix layout conversion onto \p List using +/// ID3D12GraphicsCommandListPreview::ConvertLinearAlgebraMatrix. Both +/// \p SrcBuffer (in \p SrcLayout) and \p DestBuffer (receiving \p DestLayout) +/// must be passed in the D3D12_RESOURCE_STATE_UNORDERED_ACCESS state; the +/// conversion requires the source in NON_PIXEL_SHADER_RESOURCE, so this helper +/// transitions it and leaves the destination in UNORDERED_ACCESS. The caller is +/// responsible for ensuring that writes to \p SrcBuffer have completed before +/// this conversion reads it. +void recordLinAlgMatrixConversion( + ID3D12GraphicsCommandList *List, ID3D12Resource *SrcBuffer, UINT SrcSize, + ID3D12Resource *DestBuffer, UINT DestSize, UINT NumRows, UINT NumColumns, + D3D12_LINEAR_ALGEBRA_DATATYPE DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT SrcLayout, UINT SrcStride, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT DestLayout, UINT DestStride); +#endif // HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE + #endif // HLSLEXECTESTUTILS_H diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index b3c4b761ea..cb7ab9f09b 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -349,9 +349,7 @@ class DxilConf_SM610_LinAlg { // Matrix Vector Arithmetic TEST_METHOD(MatVecMul_Thread_16x16_F16); TEST_METHOD(MatVecMulAdd_Thread_16x16_F16); -#if 0 TEST_METHOD(OuterProduct_Thread_16x16_F16); -#endif // Query Accumulator Layout TEST_METHOD(QueryAccumLayout); @@ -1321,7 +1319,29 @@ void DxilConf_SM610_LinAlg::MatVecMulAdd_Thread_16x16_F16() { ComponentType::F16); } -#if 0 +// Map a DXIL ComponentType to the D3D12 linear-algebra datatype used by the +// host-side matrix conversion API. +#if HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE +static D3D12_LINEAR_ALGEBRA_DATATYPE toLinAlgDataType(ComponentType CT) { + switch (CT) { + case ComponentType::F16: + return D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16; + case ComponentType::F32: + return D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32; + case ComponentType::I16: + return D3D12_LINEAR_ALGEBRA_DATATYPE_SINT16; + case ComponentType::U16: + return D3D12_LINEAR_ALGEBRA_DATATYPE_UINT16; + case ComponentType::I32: + return D3D12_LINEAR_ALGEBRA_DATATYPE_SINT32; + case ComponentType::U32: + return D3D12_LINEAR_ALGEBRA_DATATYPE_UINT32; + default: + VERIFY_IS_TRUE(false, "Unsupported component type for linalg conversion"); + return D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16; + } +} + static const char OuterProductShader[] = R"( #define USE_A 0 #define SCOPE_THREAD 0 @@ -1348,8 +1368,11 @@ static const char OuterProductShader[] = R"( Mat; __builtin_LinAlg_MatrixOuterProduct(Mat, VecA, VecB); + // Outer product accumulators are stored in the OuterProductOptimal layout + // with stride 0 and no alignment requirement (align 0), matching the + // dx::linalg header's thread-scoped InterlockedAccumulate. __builtin_LinAlg_MatrixAccumulateToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); + Mat, Output, 0, STRIDE, LAYOUT, 0); } )"; @@ -1359,7 +1382,18 @@ static void runOuterProduct(ID3D12Device *Device, const size_t NumVecElements = Params.M + Params.N; const size_t InBuffSize = NumVecElements * elementSize(Params.CompType); const size_t NumMatElements = Params.totalElements(); - const size_t OutBufferSize = Params.totalBytes(); + const D3D12_LINEAR_ALGEBRA_DATATYPE DataType = + toLinAlgDataType(Params.CompType); + + const UINT OutBufferSize = getLinAlgMatrixByteSize( + Device, Params.M, Params.N, DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_OUTER_PRODUCT_OPTIMAL, /*Stride=*/0); + + const UINT RowMajorStride = + static_cast(Params.N * elementSize(Params.CompType)); + const UINT RowMajorSize = getLinAlgMatrixByteSize( + Device, Params.M, Params.N, DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_ROW_MAJOR, RowMajorStride); std::string Args = buildCompilerArgs(Params); @@ -1371,7 +1405,8 @@ static void runOuterProduct(ID3D12Device *Device, auto Op = createComputeOp(OuterProductShader, "cs_6_10", "UAV(u0), UAV(u1)", Args.c_str()); addUAVBuffer(Op.get(), "Input", InBuffSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", OutBufferSize, true); + addUAVBuffer(Op.get(), "Output", OutBufferSize, /*ReadBack=*/false); + addUAVBuffer(Op.get(), "OutputRowMajor", RowMajorSize, /*ReadBack=*/true); addRootView(Op.get(), 0, "Input"); addRootView(Op.get(), 1, "Output"); @@ -1383,27 +1418,49 @@ static void runOuterProduct(ID3D12Device *Device, NumVecElements, /*StartingVal=*/2, /*Increment=*/false), "Saw unsupported component type"); + }, + [OutBufferSize, RowMajorSize, RowMajorStride, DataType, + Params](ID3D12GraphicsCommandList *List, st::ShaderOpTest *Test) { + ID3D12Resource *OptimalBuffer = nullptr; + ID3D12Resource *RowMajorBuffer = nullptr; + Test->GetResource("Output", &OptimalBuffer); + Test->GetResource("OutputRowMajor", &RowMajorBuffer); + recordLinAlgMatrixConversion( + List, OptimalBuffer, OutBufferSize, RowMajorBuffer, RowMajorSize, + Params.M, Params.N, DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_OUTER_PRODUCT_OPTIMAL, + /*SrcStride=*/0, D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_ROW_MAJOR, + RowMajorStride); }); MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); + Result->Test->GetReadBackData("OutputRowMajor", &OutData); VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), Expected, NumMatElements, Verbose)); } +#endif // HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE void DxilConf_SM610_LinAlg::OuterProduct_Thread_16x16_F16() { +#if HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; Params.Scope = MatrixScope::Thread; - Params.Layout = LinalgMatrixLayout::RowMajor; + Params.Layout = LinalgMatrixLayout::OuterProductOptimal; Params.NumThreads = 1; Params.Enable16Bit = true; runOuterProduct(D3DDevice, DxcSupport, Params, VerboseLogging); +#else + WEX::Logging::Log::Comment( + L"Skipping OuterProduct_Thread_16x16_F16: built against a D3D12 SDK " + L"without the linear-algebra matrix-conversion API " + L"(DIRECT3D_LINEAR_ALGEBRA undefined); the host-side conversion helpers " + L"are compiled out."); + WEX::Logging::Log::Result(WEX::Logging::TestResults::Skipped); +#endif // HLSL_EXEC_LINALG_HOST_CONVERSION_AVAILABLE } -#endif static const char QueryAccumLayoutShader[] = R"( RWByteAddressBuffer Output : register(u0); diff --git a/tools/clang/unittests/HLSLExec/ShaderOpTest.cpp b/tools/clang/unittests/HLSLExec/ShaderOpTest.cpp index 7542c050ba..7296165cc2 100644 --- a/tools/clang/unittests/HLSLExec/ShaderOpTest.cpp +++ b/tools/clang/unittests/HLSLExec/ShaderOpTest.cpp @@ -928,6 +928,13 @@ void ShaderOpTest::GetReadBackData(LPCSTR pResourceName, MappedData *pData) { pData->reset(D.ReadBack, sizeInBytes); } +void ShaderOpTest::GetResource(LPCSTR pResourceName, + ID3D12Resource **ppResource) { + pResourceName = m_pShaderOp->Strings.insert(pResourceName); // Unique + auto It = m_ResourceData.find(pResourceName); + *ppResource = It == m_ResourceData.end() ? nullptr : It->second.Resource.p; +} + static void SetDescriptorHeaps(ID3D12GraphicsCommandList *pList, std::vector &heaps) { if (heaps.empty()) @@ -1059,6 +1066,27 @@ void ShaderOpTest::RunCommandList() { WaitForSignal(m_CommandList.Queue, m_pFence, m_hFence, m_FenceValue++); } +// Runs the post-execute callback on a dedicated DIRECT command list that is +// submitted after the compute/graphics command list has completed. This lets +// the callback record DIRECT-only commands (e.g. ConvertLinearAlgebraMatrix) +// while the shader itself still runs on its native (compute) queue. The +// fence wait at the end of RunCommandList orders the two submissions. +void ShaderOpTest::RunPostExecuteCommandList() { + if (!m_PostExecuteCallbackFn) + return; + + CommandListRefs DirectCommandList; + DirectCommandList.CreateForDevice(m_pDevice, /*compute=*/false); + ID3D12GraphicsCommandList *pList = DirectCommandList.List; + pList->SetName(L"ShaderOpTest PostExecute CommandList"); + + m_PostExecuteCallbackFn(pList, this); + + CHECK_HR(pList->Close()); + ExecuteCommandList(DirectCommandList.Queue, pList); + WaitForSignal(DirectCommandList.Queue, m_pFence, m_hFence, m_FenceValue++); +} + void ShaderOpTest::RunShaderOp(ShaderOp *pShaderOp) { m_pShaderOp = pShaderOp; @@ -1068,6 +1096,7 @@ void ShaderOpTest::RunShaderOp(ShaderOp *pShaderOp) { CreatePipelineState(); CreateCommandList(); RunCommandList(); + RunPostExecuteCommandList(); CopyBackResources(); } @@ -1150,6 +1179,10 @@ void ShaderOpTest::SetInitCallback(TInitCallbackFn InitCallbackFn) { void ShaderOpTest::SetShaderCallback(TShaderCallbackFn ShaderCallbackFn) { m_ShaderCallbackFn = ShaderCallbackFn; } +void ShaderOpTest::SetPostExecuteCallback( + TCommandCallbackFn PostExecuteCallbackFn) { + m_PostExecuteCallbackFn = PostExecuteCallbackFn; +} void ShaderOpTest::SetupRenderTarget(ShaderOp *pShaderOp, ID3D12Device *pDevice, ID3D12CommandQueue *pCommandQueue, @@ -2750,12 +2783,12 @@ bool ShaderOpParser::ReadAtElementName(IXmlReader *pReader, LPCWSTR pName) { } } -std::shared_ptr -RunShaderOpTestAfterParse(ID3D12Device *pDevice, - dxc::SpecificDllLoader &support, LPCSTR pName, - st::ShaderOpTest::TInitCallbackFn pInitCallback, - st::ShaderOpTest::TShaderCallbackFn pShaderCallback, - std::shared_ptr ShaderOpSet) { +std::shared_ptr RunShaderOpTestAfterParse( + ID3D12Device *pDevice, dxc::SpecificDllLoader &support, LPCSTR pName, + st::ShaderOpTest::TInitCallbackFn pInitCallback, + st::ShaderOpTest::TShaderCallbackFn pShaderCallback, + st::ShaderOpTest::TCommandCallbackFn pPostExecuteCallback, + std::shared_ptr ShaderOpSet) { st::ShaderOp *pShaderOp; if (pName == nullptr) { if (ShaderOpSet->ShaderOps.size() != 1) { @@ -2786,6 +2819,7 @@ RunShaderOpTestAfterParse(ID3D12Device *pDevice, test->SetSpecificDllLoader(&support); test->SetInitCallback(pInitCallback); test->SetShaderCallback(pShaderCallback); + test->SetPostExecuteCallback(pPostExecuteCallback); test->SetDevice(pDevice); test->RunShaderOp(pShaderOp); @@ -2797,13 +2831,23 @@ RunShaderOpTestAfterParse(ID3D12Device *pDevice, return result; } +std::shared_ptr +RunShaderOpTestAfterParse(ID3D12Device *pDevice, + dxc::SpecificDllLoader &support, LPCSTR pName, + st::ShaderOpTest::TInitCallbackFn pInitCallback, + st::ShaderOpTest::TShaderCallbackFn pShaderCallback, + std::shared_ptr ShaderOpSet) { + return RunShaderOpTestAfterParse(pDevice, support, pName, pInitCallback, + pShaderCallback, nullptr, ShaderOpSet); +} + std::shared_ptr RunShaderOpTestAfterParse(ID3D12Device *pDevice, dxc::SpecificDllLoader &support, LPCSTR pName, st::ShaderOpTest::TInitCallbackFn pInitCallback, std::shared_ptr ShaderOpSet) { return RunShaderOpTestAfterParse(pDevice, support, pName, pInitCallback, - nullptr, ShaderOpSet); + nullptr, nullptr, ShaderOpSet); } std::shared_ptr diff --git a/tools/clang/unittests/HLSLExec/ShaderOpTest.h b/tools/clang/unittests/HLSLExec/ShaderOpTest.h index 23078f5303..3513a06ec8 100644 --- a/tools/clang/unittests/HLSLExec/ShaderOpTest.h +++ b/tools/clang/unittests/HLSLExec/ShaderOpTest.h @@ -276,17 +276,22 @@ class ShaderOpTest { typedef std::function TShaderCallbackFn; + typedef std::function + TCommandCallbackFn; ShaderOpTest(); ~ShaderOpTest(); void GetPipelineStats(D3D12_QUERY_DATA_PIPELINE_STATISTICS *pStats); void GetReadBackData(LPCSTR pResourceName, MappedData *pData); + void GetResource(LPCSTR pResourceName, ID3D12Resource **ppResource); void RunShaderOp(ShaderOp *pShaderOp); void RunShaderOp(std::shared_ptr pShaderOp); void SetDevice(ID3D12Device *pDevice); void SetSpecificDllLoader(dxc::SpecificDllLoader *pDxcSupport); void SetInitCallback(TInitCallbackFn InitCallbackFn); void SetShaderCallback(TShaderCallbackFn ShaderCallbackFn); + void SetPostExecuteCallback(TCommandCallbackFn PostExecuteCallbackFn); void SetupRenderTarget(ShaderOp *pShaderOp, ID3D12Device *pDevice, ID3D12CommandQueue *pCommandQueue, ID3D12Resource *pRenderTarget); @@ -327,6 +332,7 @@ class ShaderOpTest { std::shared_ptr m_OrigShaderOp; TInitCallbackFn m_InitCallbackFn = nullptr; TShaderCallbackFn m_ShaderCallbackFn = nullptr; + TCommandCallbackFn m_PostExecuteCallbackFn = nullptr; void CopyBackResources(); void CreateCommandList(); void CreateDescriptorHeaps(); @@ -336,6 +342,7 @@ class ShaderOpTest { void CreateRootSignature(); void CreateShaders(); void RunCommandList(); + void RunPostExecuteCommandList(); void SetRootValues(ID3D12GraphicsCommandList *pList, bool isCompute); }; @@ -353,6 +360,13 @@ struct ShaderOpTestResult { std::shared_ptr Test; }; +std::shared_ptr RunShaderOpTestAfterParse( + ID3D12Device *pDevice, dxc::SpecificDllLoader &support, LPCSTR pName, + st::ShaderOpTest::TInitCallbackFn pInitCallback, + st::ShaderOpTest::TShaderCallbackFn pShaderCallback, + st::ShaderOpTest::TCommandCallbackFn pPostExecuteCallback, + std::shared_ptr ShaderOpSet); + std::shared_ptr RunShaderOpTestAfterParse(ID3D12Device *pDevice, dxc::SpecificDllLoader &support, LPCSTR pName,