diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index d5f1b68dff..872f1676fd 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -22,6 +22,8 @@ #include "simplnx/DataStructure/IDataArray.hpp" #include "simplnx/DataStructure/IDataStore.hpp" #include "simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp" +#include "simplnx/DataStructure/ListStore.hpp" +#include "simplnx/DataStructure/ScalarData.hpp" #include "simplnx/Filter/Arguments.hpp" #include "simplnx/Filter/FilterHandle.hpp" #include "simplnx/Parameters/Dream3dImportParameter.hpp" @@ -59,6 +61,45 @@ const fs::path k_ExportFilename2 = "export2.dream3d"; const fs::path k_MultiExportFilename1 = "multi_export1.dream3d"; const fs::path k_MultiExportFilename2 = "multi_export2.dream3d"; const fs::path k_MultiExportFilename3 = "multi_export3.dream3d"; + +constexpr StringLiteral k_CellData = "Cell Data"; +constexpr StringLiteral k_DataContainer = "Data Container"; +constexpr StringLiteral k_EdgeGeom = "EdgeGeom"; +constexpr StringLiteral k_ImageGeom = "ImageGeom"; +constexpr StringLiteral k_RectGridGeom = "RectGrid"; +constexpr StringLiteral k_HexGeom = "HexahedralGeom"; +constexpr StringLiteral k_QuadGeom = "QuadGeom"; +constexpr StringLiteral k_TetrahedralGeom = "TetrahedralGeom"; +constexpr StringLiteral k_TriangleGeom = "TriangleGeom"; +constexpr StringLiteral k_VertexGeom = "VertexGeom"; +constexpr StringLiteral k_XBounds = "X Bounds"; +constexpr StringLiteral k_YBounds = "Y Bounds"; +constexpr StringLiteral k_ZBounds = "Z Bounds"; + +constexpr StringLiteral k_DynamicListArray = "DynamicList"; +constexpr StringLiteral k_NeighborList = "NeighborList"; +constexpr StringLiteral k_StringArray = "String Array"; +constexpr StringLiteral k_VertexList = "Vertices"; +constexpr StringLiteral k_Edges = "Edges"; +constexpr StringLiteral k_Faces = "Faces"; +constexpr StringLiteral k_Polyhedra = "Polyhedra"; +constexpr StringLiteral k_HexaArray = "Hexahedra Array"; +constexpr StringLiteral k_TetraArray = "Tetrahedra Array"; +constexpr StringLiteral k_QuadArray = "Quad Array"; +constexpr StringLiteral k_Int8 = "Int 8"; +constexpr StringLiteral k_Int16 = "Int 16"; +constexpr StringLiteral k_Int32 = "Int 32"; +constexpr StringLiteral k_Int64 = "Int 64"; +constexpr StringLiteral k_UInt8 = "UInt 8"; +constexpr StringLiteral k_UInt16 = "UInt 16"; +constexpr StringLiteral k_UInt32 = "UInt 32"; +constexpr StringLiteral k_UInt64 = "UInt 64"; +constexpr StringLiteral k_Float32 = "Float 32"; +constexpr StringLiteral k_Float64 = "Float 64"; +constexpr int64 k_ScalarValue = 3; +const ShapeType k_TupleShape{3, 2, 1}; +const SizeVec3 k_ImageShape{1, 2, 3}; +constexpr int16 k_ListCount = 6; } // namespace Constants std::mutex m_DataMutex; @@ -97,6 +138,13 @@ fs::path GetIODataPath() return GetDataDir(*app) / Constants::k_Dream3dFilename; } +fs::path GetXdmfPath() +{ + fs::path filePath = GetIODataPath(); + filePath.replace_extension(".xdmf"); + return filePath; +} + fs::path GetExportDataPath() { auto app = Application::Instance(); @@ -152,6 +200,261 @@ fs::path GetReMultiExportDataPath() return GetDataDir(*app) / Constants::k_MultiExportFilename3; } +/** + * @brief Creates and sets arrays for a 2D geometry of type T. + * @param dataStructure + * @param name + * @param vertexArray + * @param edgeList + * @param faceList + * @return T* + */ +template +T* Create2DGeom(DataStructure& dataStructure, const std::string& name, const IGeometry::SharedVertexList& vertexArray, const IGeometry::SharedEdgeList& edgeList, + const IGeometry::SharedFaceList& faceList) +{ + auto* geom = T::Create(dataStructure, name); + geom->setVertices(vertexArray); + geom->setEdgeList(edgeList); + geom->setFaceList(faceList); + return geom; +} + +/** + * @brief Creates and sets arrays for a 3D geometry of type T. + * @param dataStructure + * @param name + * @param vertexArray + * @param edgeList + * @param faceList + * @param polyArray + * @return T* + */ +template +T* Create3DGeom(DataStructure& dataStructure, const std::string& name, const IGeometry::SharedVertexList& vertexArray, const IGeometry::SharedEdgeList& edgeList, + const IGeometry::SharedFaceList& faceList, const IGeometry::SharedHexList& polyArray) +{ + auto* geom = T::Create(dataStructure, name); + geom->setVertices(vertexArray); + geom->setEdgeList(edgeList); + geom->setFaceList(faceList); + geom->setPolyhedraList(polyArray); + return geom; +} + +/** + * @brief Fill the data store with values, but do not use values greater than the number of tuples. + * @param dataStore + */ +template +void FillDataStore(AbstractDataStore& dataStore) +{ + const auto numTuples = dataStore.getNumberOfTuples(); + const auto numComponents = dataStore.getNumberOfComponents(); + + for(usize i = 0; i < numTuples; i++) + { + const usize offset = i * numComponents; + for(usize j = 0; j < numComponents; j++) + { + usize value = (i + j) % numTuples; + dataStore[offset + j] = static_cast(value); + } + } +} + +template +void CheckDataStore(const AbstractDataStore& dataStore, usize requiredComponents) +{ + const auto numTuples = dataStore.getNumberOfTuples(); + const auto numComponents = dataStore.getNumberOfComponents(); + REQUIRE(numComponents == requiredComponents); + + for(usize i = 0; i < numTuples; i++) + { + const usize offset = i * numComponents; + for(usize j = 0; j < numComponents; j++) + { + usize value = (i + j) % numTuples; + REQUIRE(dataStore[offset + j] == static_cast(value)); + } + } +} + +void CheckGeom0D(const INodeGeometry0D* geom, DataObject::IdType vertexId) +{ + REQUIRE(geom != nullptr); + REQUIRE(geom->getVertexListId() == vertexId); +} + +void CheckGeom1D(const INodeGeometry1D* geom, DataObject::IdType vertexId, DataObject::IdType edgeId) +{ + REQUIRE(geom != nullptr); + REQUIRE(geom->getEdgeListId() == edgeId); + CheckGeom0D(geom, vertexId); +} + +void CheckGeom2D(const INodeGeometry2D* geom, DataObject::IdType vertexId, DataObject::IdType edgeId, DataObject::IdType faceId) +{ + REQUIRE(geom != nullptr); + REQUIRE(geom->getFaceListId() == faceId); + CheckGeom1D(geom, vertexId, edgeId); +} + +void CheckGeom3D(const INodeGeometry3D* geom, DataObject::IdType vertexId, DataObject::IdType edgeId, DataObject::IdType faceId, DataObject::IdType polyhedraId) +{ + REQUIRE(geom != nullptr); + REQUIRE(geom->getPolyhedronListId().has_value()); + REQUIRE(geom->getPolyhedronListId().value() == polyhedraId); + CheckGeom2D(geom, vertexId, edgeId, faceId); +} + +template +void CheckScalarData(const DataStructure& dataStructure, const DataPath& path) +{ + auto* scalarData = dataStructure.getDataAs>(path); + REQUIRE(scalarData != nullptr); + REQUIRE(scalarData->getValue() == Approx(static_cast(Constants::k_ScalarValue))); +} + +void CheckNeighborListStore(const AbstractListStore& store) +{ + REQUIRE(store.getNumberOfTuples() == Constants::k_ListCount); + for(usize i = 0; i < Constants::k_ListCount; i++) + { + auto list = store.getList(i); + REQUIRE(list.size() == 2); + REQUIRE(list[0] == 1); + REQUIRE(list[1] == 2); + } +} + +void CheckTestDataStructure(const DataStructure& dataStructure) +{ + DataPath dataGroupPath({Constants::k_DataContainer}); + REQUIRE(dataStructure.getDataAs(dataGroupPath) != nullptr); + + const auto* neighborList = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_NeighborList)); + REQUIRE(neighborList != nullptr); + const auto storePtr = neighborList->getStore(); + REQUIRE(storePtr != nullptr); + CheckNeighborListStore(*storePtr); + + const auto* vertexArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_VertexList)); + REQUIRE(vertexArray != nullptr); + const auto& vertices = vertexArray->getDataStoreRef(); + CheckDataStore(vertices, 3); + + const auto* edgeArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_Edges)); + REQUIRE(edgeArray != nullptr); + const auto& edges = edgeArray->getDataStoreRef(); + CheckDataStore(edges, 2); + + const auto* faceArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_Faces)); + REQUIRE(faceArray != nullptr); + const auto& faces = faceArray->getDataStoreRef(); + CheckDataStore(faces, 3); + + const auto* quadArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_QuadArray)); + REQUIRE(quadArray != nullptr); + const auto& quads = quadArray->getDataStoreRef(); + CheckDataStore(quads, 4); + + const auto* hexaArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_HexaArray)); + REQUIRE(hexaArray != nullptr); + const auto& hexa = hexaArray->getDataStoreRef(); + CheckDataStore(hexa, 8); + + const auto* tetraArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_TetraArray)); + REQUIRE(tetraArray != nullptr); + const auto& tetra = tetraArray->getDataStoreRef(); + CheckDataStore(tetra, 4); + + const auto* polyArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_Polyhedra)); + REQUIRE(polyArray != nullptr); + const auto& polyhedra = polyArray->getDataStoreRef(); + CheckDataStore(polyhedra, 4); + + const auto* stringArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_StringArray)); + REQUIRE(stringArray != nullptr); + auto stringCount = stringArray->getNumberOfTuples(); + REQUIRE(stringCount == 6); + REQUIRE(stringArray->at(0) == "1"); + REQUIRE(stringArray->at(1) == "2"); + REQUIRE(stringArray->at(2) == "3"); + REQUIRE(stringArray->at(3) == "4"); + REQUIRE(stringArray->at(4) == "5"); + REQUIRE(stringArray->at(5) == "6"); + + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Int8)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Int16)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Int32)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Int64)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_UInt8)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_UInt16)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_UInt32)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_UInt64)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Float32)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Float64)); + + const auto* vertexGeom = dataStructure.getDataAs(DataPath({Constants::k_VertexGeom})); + CheckGeom0D(vertexGeom, vertexArray->getId()); + + const auto* edgeGeom = dataStructure.getDataAs(DataPath({Constants::k_EdgeGeom})); + CheckGeom1D(edgeGeom, vertexArray->getId(), edgeArray->getId()); + + const auto* quadGeom = dataStructure.getDataAs(DataPath({Constants::k_QuadGeom})); + CheckGeom2D(quadGeom, vertexArray->getId(), edgeArray->getId(), quadArray->getId()); + + const auto* triGeom = dataStructure.getDataAs(DataPath({Constants::k_TriangleGeom})); + CheckGeom2D(triGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId()); + + const auto* hexGeom = dataStructure.getDataAs(DataPath({Constants::k_HexGeom})); + CheckGeom3D(hexGeom, vertexArray->getId(), edgeArray->getId(), hexaArray->getId(), polyArray->getId()); + + const auto* tetraGeom = dataStructure.getDataAs(DataPath({Constants::k_TetrahedralGeom})); + CheckGeom3D(tetraGeom, vertexArray->getId(), edgeArray->getId(), tetraArray->getId(), polyArray->getId()); + + // Image Geom + DataPath imageGeomPath({Constants::k_ImageGeom}); + const auto* imageGeom = dataStructure.getDataAs(imageGeomPath); + REQUIRE(imageGeom != nullptr); + REQUIRE(imageGeom->getDimensions() == Constants::k_ImageShape); + const auto* cellData = dataStructure.getDataAs(imageGeomPath.createChildPath(Constants::k_CellData)); + REQUIRE(cellData != nullptr); + REQUIRE(imageGeom->getCellDataId() == cellData->getId()); + REQUIRE(cellData->getShape() == Constants::k_TupleShape); + + // RectGrid Geom + DataPath rectGridGeomPath({Constants::k_RectGridGeom}); + const auto* rectGrid = dataStructure.getDataAs(rectGridGeomPath); + REQUIRE(rectGrid != nullptr); + REQUIRE(rectGrid->getCellDataId() == cellData->getId()); + auto rectDims = rectGrid->getDimensions(); + REQUIRE(rectDims[0] == Constants::k_TupleShape[0]); + REQUIRE(rectDims[1] == Constants::k_TupleShape[1]); + REQUIRE(rectDims[2] == Constants::k_TupleShape[2]); + DataPath xPath = rectGridGeomPath.createChildPath(Constants::k_XBounds); + const auto* xBoundsArray = dataStructure.getDataAs(xPath); + REQUIRE(xBoundsArray != nullptr); + DataPath yPath = rectGridGeomPath.createChildPath(Constants::k_YBounds); + const auto* yBoundsArray = dataStructure.getDataAs(yPath); + REQUIRE(yBoundsArray != nullptr); + DataPath zPath = rectGridGeomPath.createChildPath(Constants::k_ZBounds); + const auto* zBoundsArray = dataStructure.getDataAs(zPath); + REQUIRE(zBoundsArray != nullptr); + REQUIRE(rectGrid->getXBoundsId() == xBoundsArray->getId()); + REQUIRE(rectGrid->getYBoundsId() == yBoundsArray->getId()); + REQUIRE(rectGrid->getZBoundsId() == zBoundsArray->getId()); +} + +template +void CreateScalarData(DataStructure& dataStructure, const std::string& name, DataObject::IdType parentId) +{ + auto* scalarData = ScalarData::Create(dataStructure, name, static_cast(Constants::k_ScalarValue), parentId); + REQUIRE(scalarData != nullptr); +} + DataStructure CreateTestDataStructure() { DataStructure dataStructure; @@ -165,6 +468,115 @@ DataStructure CreateTestDataStructure() Result<> arrayCreationResults = ArrayCreationUtilities::CreateArray(dataStructure, tupleShape, std::vector{1}, DataPath({DataNames::k_Group1Name, DataNames::k_AttributeMatrixName, DataNames::k_Array2Name}), IDataAction::Mode::Execute, ArrayCreationUtilities::k_DefaultDataFormat, "1"); + + // Create Arrays and DataGroup + auto* dataGroup = DataGroup::Create(dataStructure, Constants::k_DataContainer); + + auto listStorePtr = std::make_shared>(Constants::k_TupleShape); + listStorePtr->setList(0, std::vector{1, 2}); + listStorePtr->setList(1, std::vector{1, 2}); + listStorePtr->setList(2, std::vector{1, 2}); + listStorePtr->setList(3, std::vector{1, 2}); + listStorePtr->setList(4, std::vector{1, 2}); + listStorePtr->setList(5, std::vector{1, 2}); + auto* neighborList = Int16NeighborList::Create(dataStructure, Constants::k_NeighborList, listStorePtr, dataGroup->getId()); + REQUIRE(neighborList != nullptr); + + auto vertices = std::make_shared(Constants::k_TupleShape, ShapeType{3}, 0.0f); + auto* vertexArray = Float32Array::Create(dataStructure, Constants::k_VertexList, vertices, dataGroup->getId()); + FillDataStore(*vertices.get()); + + auto edges = std::make_shared(Constants::k_TupleShape, ShapeType{2}, 0); + auto* edgesArray = IGeometry::SharedEdgeList::Create(dataStructure, Constants::k_Edges, edges, dataGroup->getId()); + FillDataStore(*edges.get()); + + auto triangles = std::make_shared(Constants::k_TupleShape, ShapeType{3}, 0); + auto* trianglesArray = IGeometry::SharedTriList::Create(dataStructure, Constants::k_Faces, triangles, dataGroup->getId()); + FillDataStore(*triangles.get()); + + auto polyhedra = std::make_shared(Constants::k_TupleShape, ShapeType{4}, 0); + auto* polyhedraArray = IGeometry::SharedFaceList::Create(dataStructure, Constants::k_Polyhedra, polyhedra, dataGroup->getId()); + FillDataStore(*polyhedra.get()); + + auto quadStore = std::make_shared(Constants::k_TupleShape, ShapeType{4}, 0); + auto* quadArray = IGeometry::SharedFaceList::Create(dataStructure, Constants::k_QuadArray, quadStore, dataGroup->getId()); + FillDataStore(*quadStore.get()); + + auto hexStore = std::make_shared(Constants::k_TupleShape, ShapeType{8}, 0); + auto* hexArray = IGeometry::SharedHexList::Create(dataStructure, Constants::k_HexaArray, hexStore, dataGroup->getId()); + FillDataStore(*hexStore.get()); + + auto tetraStore = std::make_shared(Constants::k_TupleShape, ShapeType{4}, 0); + auto* tetraArray = IGeometry::SharedTetList::Create(dataStructure, Constants::k_TetraArray, tetraStore, dataGroup->getId()); + FillDataStore(*tetraStore.get()); + + StringArray::collection_type strings = {"1", "2", "3", "4", "5", "6"}; + auto* stringArray = StringArray::CreateWithValues(dataStructure, Constants::k_StringArray, Constants::k_TupleShape, strings, dataGroup->getId()); + REQUIRE(stringArray != nullptr); + + CreateScalarData(dataStructure, Constants::k_Int8, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Int16, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Int32, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Int64, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_UInt8, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_UInt16, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_UInt32, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_UInt64, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Float32, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Float64, dataGroup->getId()); + + // Create Geometries and make sure special arrays are set. + auto* imageGeom = ImageGeom::Create(dataStructure, Constants::k_ImageGeom); + REQUIRE(imageGeom != nullptr); + auto* cellMatrix = AttributeMatrix::Create(dataStructure, Constants::k_CellData, Constants::k_TupleShape, imageGeom->getId()); + REQUIRE(cellMatrix != nullptr); + imageGeom->setCellData(cellMatrix->getId()); + imageGeom->setDimensions(Constants::k_ImageShape); + + // RectGrid Data + auto* rectGridGeom = RectGridGeom::Create(dataStructure, Constants::k_RectGridGeom); + REQUIRE(rectGridGeom != nullptr); + ShapeType xShape{Constants::k_TupleShape[0]}; + ShapeType componentBounds{1}; + auto xBounds = std::make_shared(xShape, componentBounds, 0.0f); + auto* xBoundsArray = Float32Array::Create(dataStructure, Constants::k_XBounds, xBounds, rectGridGeom->getId()); + FillDataStore(*xBounds.get()); + ShapeType yShape{Constants::k_TupleShape[1]}; + auto yBounds = std::make_shared(yShape, componentBounds, 0.0f); + auto* yBoundsArray = Float32Array::Create(dataStructure, Constants::k_YBounds, yBounds, rectGridGeom->getId()); + FillDataStore(*yBounds.get()); + ShapeType zShape{Constants::k_TupleShape[1]}; + auto zBounds = std::make_shared(zShape, componentBounds, 0.0f); + auto* zBoundsArray = Float32Array::Create(dataStructure, Constants::k_ZBounds, zBounds, rectGridGeom->getId()); + FillDataStore(*zBounds.get()); + + rectGridGeom->setDimensions(Constants::k_TupleShape); + rectGridGeom->setCellData(cellMatrix->getId()); + rectGridGeom->setBounds(xBoundsArray, yBoundsArray, zBoundsArray); + + // 0D Geometry + auto* vertexGeom = VertexGeom::Create(dataStructure, Constants::k_VertexGeom); + REQUIRE(vertexGeom != nullptr); + vertexGeom->setVertices(*vertexArray); + + // 1D Geometry + auto* edgeGeom = EdgeGeom::Create(dataStructure, Constants::k_EdgeGeom); + REQUIRE(edgeGeom != nullptr); + edgeGeom->setVertices(*vertexArray); + edgeGeom->setEdgeList(*edgesArray); + + // 2D Geometries + auto* quadGeom = Create2DGeom(dataStructure, Constants::k_QuadGeom, *vertexArray, *edgesArray, *quadArray); + REQUIRE(quadGeom != nullptr); + auto* triangleGeom = Create2DGeom(dataStructure, Constants::k_TriangleGeom, *vertexArray, *edgesArray, *trianglesArray); + REQUIRE(triangleGeom != nullptr); + + // 3D Geometries + auto* hexGeom = Create3DGeom(dataStructure, Constants::k_HexGeom, *vertexArray, *edgesArray, *hexArray, *polyhedraArray); + REQUIRE(hexGeom != nullptr); + auto* tetrahedralGeom = Create3DGeom(dataStructure, Constants::k_TetrahedralGeom, *vertexArray, *edgesArray, *tetraArray, *polyhedraArray); + REQUIRE(tetrahedralGeom != nullptr); + return dataStructure; } @@ -449,18 +861,134 @@ GeometryTestCase MakeGeometryTestCase(std::string typeName, std::function lock(m_DataMutex); + + DataStructure dataStructure = CreateTestDataStructure(); + Arguments args; + WriteDREAM3DFilter filter; + + args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(GetIODataPath())); + args.insertOrAssign(WriteDREAM3DFilter::k_WriteXdmf, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(1)); + + SECTION("Empty FilePath") + { + args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(std::filesystem::path())); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_INVALID(preflightResult.outputActions); + } + + SECTION("Bad Compression Level") + { + args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(true)); + args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(0)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_INVALID(preflightResult.outputActions); + } +} + +TEST_CASE("WriteDREAM3DFilter:Valid Parameters", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") +{ + UnitTest::LoadPlugins(); + std::lock_guard lock(m_DataMutex); + + { + DataStructure dataStructure = CreateTestDataStructure(); + Arguments args; + WriteDREAM3DFilter filter; + + args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(GetIODataPath())); + args.insertOrAssign(WriteDREAM3DFilter::k_WriteXdmf, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(1)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + + auto result = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + } + + // Check that the output file exists + REQUIRE(fs::exists(GetIODataPath())); + + // Check that the file can be read back in and that the imported DataStructure matches expected values. + { + auto fileReader = HDF5::FileIO::ReadFile(GetIODataPath()); + auto fileResult = DREAM3D::ReadFile(fileReader); + SIMPLNX_RESULT_REQUIRE_VALID(fileResult); + + auto [pipeline, dataStructureRead] = fileResult.value(); + + CheckTestDataStructure(dataStructureRead); + } +} + +void CheckXdmfFile() +{ + auto filepath = GetXdmfPath(); + REQUIRE(fs::exists(filepath)); +} + +TEST_CASE("WriteDREAM3D:Pipeline / WriteXdmf combinations", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") +{ + UnitTest::LoadPlugins(); + std::lock_guard lock(m_DataMutex); + + bool writeXdmf = GENERATE(true, false); + Pipeline exportPipeline = GENERATE(CreateExportPipeline(), Pipeline()); + + auto writeResult = DREAM3D::WriteFile(GetIODataPath(), CreateTestDataStructure(), exportPipeline, writeXdmf); + SIMPLNX_RESULT_REQUIRE_VALID(writeResult); + + if(writeXdmf) + { + CheckXdmfFile(); + } +} + +TEST_CASE("WriteDREAM3D:Invalid File", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") +{ + UnitTest::LoadPlugins(); + std::lock_guard lock(m_DataMutex); + + bool writeXdmf = GENERATE(true, false); + Pipeline exportPipeline = GENERATE(CreateExportPipeline(), Pipeline()); + + auto writeResult = DREAM3D::WriteFile(fs::path(), CreateTestDataStructure(), exportPipeline, writeXdmf); + SIMPLNX_RESULT_REQUIRE_INVALID(writeResult); + + if(writeXdmf) + { + CheckXdmfFile(); + } +} + TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test", "[WriteDREAM3DFilter]") { UnitTest::LoadPlugins(); std::lock_guard lock(m_DataMutex); + + bool writeXdmf = GENERATE(true, false); // Write .dream3d file { - auto fileData = CreateFileData(); - auto fileWriter = HDF5::FileIO::WriteFile(GetIODataPath()); - - auto writeResult = DREAM3D::WriteFile(fileWriter, fileData); + auto writeResult = DREAM3D::WriteFile(GetIODataPath(), CreateTestDataStructure(), CreateExportPipeline(), writeXdmf); SIMPLNX_RESULT_REQUIRE_VALID(writeResult); + + if(writeXdmf) + { + CheckXdmfFile(); + } } // Read .dream3d file @@ -471,6 +999,8 @@ TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test", "[WriteDREAM3DFilter]") auto [pipeline, dataStructure] = fileResult.value(); + CheckTestDataStructure(dataStructure); + // Test reading the DataStructure REQUIRE(dataStructure.getData(DataPath({DataNames::k_Group1Name})) != nullptr); REQUIRE(dataStructure.getData(DataPath({DataNames::k_Group1Name, DataNames::k_Group2Name})) != nullptr); @@ -887,6 +1417,7 @@ TEST_CASE("SimplnxCore::WriteDREAM3DFilter: SIMPL Backwards Compatibility", "[Si const Arguments args = pipelineFilter->getArguments(); CHECK(args.value(WriteDREAM3DFilter::k_ExportFilePath) == fs::path("/test/path/output.dream3d")); CHECK(args.value(WriteDREAM3DFilter::k_WriteXdmf) == true); + CHECK(args.value(WriteDREAM3DFilter::k_UseCompression) == false); } } } diff --git a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md new file mode 100644 index 0000000000..74fad98955 --- /dev/null +++ b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md @@ -0,0 +1,127 @@ +# V&V Report: WriteDREAM3DFilter + +| | | +|-----------------------------|-----------------------------------------------------------------------------| +| Plugin | SimplnxCore | +| SIMPLNX UUID | `b3a95784-2ced-41ec-8d3d-0242ac130003` | +| SIMPLNX Human Name | Write DREAM3D-NX File | +| DREAM3D 6.5.171 equivalent | `DataContainerWriter` — SIMPL UUID `3fcd4c43-9d75-5b86-aad4-4441bc914f37` | +| Verified commit | ** | +| Status | **READY FOR REVIEW** (second-engineer review outstanding — see V&V phase) | +| Sign-off | *pending second-engineer review* | + +## At a glance + +| Aspect | Current state | +|------------------------|---------------| +| Algorithm Relationship | **Rewrite** — same UUID/role as legacy `DataContainerWriter`, but the on-disk format is entirely new (v8 `DataStructure` HDF5 layout + `AtomicFile` atomic-write + optional gzip compression), not a translation of the legacy writer's code. | +| Oracle (confirmed) | **Class 1 (Analytical)** — expected content is the hand-built in-memory `DataStructure`/`Pipeline` the test itself constructed; expected HDF5 physical layout (contiguous vs. chunked+deflate) is a closed-form function of array byte-size and the two compression parameters. 17 fixtures across `DREAM3DFileTest.cpp`, all pass. | +| Code paths enumerated | **14 of 19** exercised; 5 gaps are defensive/unreachable-via-public-API guards (see table). | +| Tests today | **17 TEST_CASEs** (some with `GENERATE`/`DYNAMIC_SECTION` multiplying cases) — preflight validation, full round-trip content fidelity across every geometry/DataObject type, SIMPL args backward-compat, and a 6-test compression sub-suite (layout, bypass threshold, level monotonicity). | +| Exemplar archive | **None.** Every test builds its `DataStructure` inline in C++ and round-trips it through `WriteFile`/`ReadFile` in the same run — no cached `.tar.gz` golden file is used or needed for a Class 1 oracle. | +| Legacy comparison | **Not run — and not applicable.** The two writers target deliberately different on-disk contracts, so a byte/dataset-level A/B against 6.5.171 `DataContainerWriter` output would be 100% noise by design, not signal. `ReadDREAM3DFilter` is the only tool in either codebase that understands both formats; fidelity is instead verified independently via round-trip Class 1 tests. | +| Bug flags | None. | +| V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design, of the 5 uncovered defensive paths, and of the `DynamicListArray` IO gap (Known limitations). | + +## Summary + +`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 17 test cases pass; no bugs were found. `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, a bare `DynamicListArray` (as opposed to its `NeighborList` specialization) has no HDF5 IO factory at all in the current codebase and cannot be written by this or any filter (see Known limitations). + +## Algorithm Relationship + +*Classification:* **Rewrite** ~~| Port | Minor changes | New filter~~ + +`WriteDREAM3DFilter` keeps the SIMPL UUID mapping (`3fcd4c43-9d75-5b86-aad4-4441bc914f37` → `WriteDREAM3DFilter`, `SimplnxCoreLegacyUUIDMapping.hpp:170`) and the legacy `DataContainerWriter` role, but the algorithm (`Algorithms/WriteDREAM3D.cpp`, 82 lines) was designed from the start for the current v8 `DataStructure` HDF5 layout, `AtomicFile`-based atomic writes, and (as of PR #1606) optional gzip compression — none of which exist in the legacy 6.5.171 writer. This has never been a line-by-line port of the legacy C++; the file format itself is a clean-sheet design (`k_CurrentFileVersion = "8.0"` vs. legacy's `"7.0"`/`DataContainers` group tag, see `Dream3dIO.hpp:31`). + +*Evidence:* `parametersVersion()` is at 2 (compression parameters added after the filter's initial release); `git log --follow` on the algorithm/filter files shows the write path has been restructured multiple times since inception (out-of-core support #1253, atomic-file rework #900, algorithm-class extraction #1544) without ever tracking legacy DataContainerWriter's implementation. + +*Port-time deltas (SIMPL → SIMPLNX argument mapping, `WriteDREAM3DFilter::FromSIMPLJson`):* + +1. `OutputFile` → `export_file_path`, `WriteXdmfFile` → `write_xdmf_file`: direct 1:1 mapping, no behavior change. +2. SIMPL's "Write Time Series" parameter has no SIMPLNX equivalent (dropped) — legacy time-series writing is not part of this filter's scope in NX; not a regression since no NX pipeline concept maps to it. +3. `use_compression` is force-overridden to `false` for any pipeline converted from SIMPL JSON (`WriteDREAM3DFilter.cpp:130`), even though SIMPLNX's own default is `true`. This is deliberate: SIMPL v6 pipelines never wrote compressed files, so a converted pipeline preserves the exact on-disk encoding it shipped with rather than silently changing file size/behavior on re-run. + +*Material PRs since baseline:* #1606 (added HDF5 compression parameters/behavior), #1544 (moved `executeImpl` logic into the `WriteDREAM3D` algorithm class, no behavior change), #1253 (out-of-core support). + +## Oracle + +*Class:* **1 (Analytical)**, primary. `Compression_LevelsRoundTrip` also carries a **Class 4 (Invariant)** companion check (file size must be non-increasing as gzip level rises) alongside its per-level content round-trip. + +*Applied:* Every test constructs its expected answer directly, without ever running the filter to "produce" the expected value: + +- **Content fidelity:** each test builds a `DataStructure` in C++ (`CreateTestDataStructure()`, or an inline array/geometry), writes it, reads it back, and asserts the read-back content equals what was built — by construction, not by comparison to a previously-captured file. `CheckTestDataStructure()` walks every `DataObject` kind the filter must support (nested `DataGroup`s, `AttributeMatrix`, `NeighborList`, `StringArray`, and all seven geometry types: Vertex/Edge/Triangle/Quad/Tetrahedral/Hexahedral/Image) and asserts exact values against the hand-known fill pattern from `FillDataStore()`. +- **HDF5 physical layout:** the filter's documented compression policy (`docs/WriteDREAM3DFilter.md`) states arrays under 16 KiB always stay contiguous/uncompressed regardless of settings, and any larger array is chunked+deflated at the requested level when compression is enabled. This is a closed-form predicate on `(array byte size, UseCompression, CompressionLevel)` — tests assert it directly via `UnitTest::ProbeHdf5Dataset` rather than trusting the filter's own claim about what it wrote. +- **Pipeline embedding:** when run inside an actual `Pipeline`, the written file's embedded pipeline JSON must reproduce the exact filter sequence and count that was executed — asserted against the hand-built pipeline (`pipeline.size() == 3`, filter names checked by index). + +**Encoded tests:** `src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp` (17 `TEST_CASE`s touching Write, several parameterized via `GENERATE`/`DYNAMIC_SECTION`) — all pass. See Test inventory below for the full list. + +*Second-engineer review:* Outstanding. Recommended focus: the Class 1 boundary-of-scope claim in deviation D2 (StatsDataArray/StructArray) and the 5 uncovered defensive paths (Code path coverage below). + +## Code path coverage + +**14 of 19** paths exercised. The 5 gaps are all defensive guards that require conditions unreachable through the public filter/pipeline API (invalid destination mid-write after preflight already validated it, or a detached `PipelineFilter`) rather than genuine untested behavior. + +Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D.cpp` (82 lines). Preflight guards below live in the sibling `Filters/WriteDREAM3DFilter.cpp` (`preflightImpl`), which the policy still treats as in-scope algorithm surface (parameter validation gates that the Algorithm class depends on). + +| # | Phase | Path | Test case | +|----|--------------------|----------------------------------------------------------------------------------------------|-----------| +| 1 | Preflight | `export_file_path` empty → error `-1` | `"WriteDREAM3DFilter:Invalid Parameters"` § `Empty FilePath` | +| 2 | Preflight | `use_compression=true`, `compression_level < 1` → error `-2` | `"WriteDREAM3DFilter:Invalid Parameters"` § `Bad Compression Level`; `"WriteDREAM3DFilter: Compression_Preflight_RejectsOutOfRangeLevel"` (level=0) | +| 3 | Preflight | `use_compression=true`, `compression_level > 9` → error `-2` | `"WriteDREAM3DFilter: Compression_Preflight_RejectsOutOfRangeLevel"` (level=10) | +| 4 | Preflight | `use_compression=false` → `compression_level` ignored even if out of `[1,9]` | `"WriteDREAM3DFilter: Compression_Preflight_RejectsOutOfRangeLevel"` (level=0, compression off) | +| 5 | Preflight | Valid parameters → success | `"WriteDREAM3DFilter:Valid Parameters"`; implicitly, every passing execute-path test below | +| 6 | Execute — setup | `AtomicFile::Create` fails (unwritable/invalid destination directory) | *Not directly tested.* Preflight only rejects an empty path; a directory-permission failure at execute time would require a filesystem fixture (e.g., a read-only directory) not set up by the current suite. | +| 7 | Execute — setup | `AtomicFile::Create` succeeds | Every passing execute-path test | +| 8 | Execute — pipeline | `PipelineNode != nullptr` and `getPrecedingPipeline()` returns `nullptr` → error `-15` | *Not directly tested.* Only reachable if a `PipelineFilter` is detached from its parent `Pipeline`, which normal `filter.execute()`/`pipeline.execute()` usage never produces. | +| 9 | Execute — pipeline | `PipelineNode != nullptr`, preceding pipeline retrieved successfully → embedded in file | `"DREAM3DFileTest:Import/Export DREAM3D Filter Test"` (`exportPipeline.execute()`); `"DREAM3DFileTest:Import/Export Multi-DREAM3D Filter Test"` (`CreateMultiExportFiles()`) | +| 10 | Execute — pipeline | `PipelineNode == nullptr` → empty pipeline written | `"WriteDREAM3DFilter:Valid Parameters"`, `"DREAM3DFileTest::StringArray"`, all `Compression_*` tests (all call `filter.execute(ds, args)` directly) | +| 11 | Execute — options | `use_compression=true` → `writeOptions.compressionLevel = CompressionLevel` | `"...Compression_On_IsChunkedAndDeflated"`, `"...Compression_SmallArray_Bypasses"`, `"...Compression_LevelsRoundTrip"` | +| 12 | Execute — options | `use_compression=false` → `writeOptions.compressionLevel = 0` | `"...Compression_Off_IsContiguous"`; `"WriteDREAM3DFilter:Valid Parameters"` | +| 13 | Execute — write | `DREAM3D::WriteFile(...)` returns invalid → skip commit, return the error | *Not directly tested* through the full filter/`AtomicFile` path, but concretely reachable (not merely defensive): `HDF5::DataStructureWriter::WriteFile` returns error `-5` ("Could not find IO factory for datatype: …") for any `DataObject` type with no registered HDF5 IO factory — see Known limitations below (a bare `DynamicListArray`, not wrapped as `NeighborList`). No test currently puts such an object in the `DataStructure` before writing. | +| 14 | Execute — write | `DREAM3D::WriteFile(...)` returns valid → proceed to commit | Every passing execute-path test | +| 15 | Execute — commit | `atomicFile.commit()` fails (rename onto final destination fails) | *Not directly tested.* Would require the destination path to become invalid between `AtomicFile::Create` and `commit()` (e.g., concurrent deletion of the parent directory) — a race not exercised by the suite. | +| 16 | Execute — commit | `atomicFile.commit()` succeeds | Every passing execute-path test (the output file is present and re-readable in every round-trip test) | +| 17 | Execute — xdmf | `write_xdmf_file=true` → rename temp `.xdmf` into place, succeeds | `"DREAM3DFileTest:DREAM3D File IO Test"` (writeXdmf=true), `CreateExportPipeline()`/`CreateMultiExportFiles()` (`write_xdmf_file=true`) | +| 18 | Execute — xdmf | `write_xdmf_file=true`, rename fails → `MakeErrorResult` with system error message | *Not directly tested.* Would require the `.xdmf` destination to become unwritable between the HDF5 write succeeding and the rename — not portably reproducible in the current suite. | +| 19 | Execute — xdmf | `write_xdmf_file=false` → skip rename, return `WriteFile`'s result directly | Most `Compression_*` tests, `"DREAM3DFileTest::StringArray"`, `"WriteDREAM3DFilter:Valid Parameters"` | + +## Test inventory + +| Test case | Status | Notes | +|-----------|--------|-------| +| `WriteDREAM3DFilter:Invalid Parameters` (§ Empty FilePath, § Bad Compression Level) | kept | Preflight-only, inline `DataStructure`. Covers Paths 1, 2. | +| `WriteDREAM3DFilter:Valid Parameters` | kept | Preflight + full execute, inline `DataStructure`, compression off. Covers Paths 5, 7, 10, 12, 14, 16, 19. | +| `WriteDREAM3D:Pipeline / WriteXdmf combinations` | kept | Calls `DREAM3D::WriteFile(path, ds, pipeline, writeXdmf)` directly (bypasses `AtomicFile`/the Algorithm class) across `writeXdmf × {real pipeline, empty pipeline}` (4 cases). Exercises the shared write utility the Algorithm depends on, not the Algorithm's own guards. | +| `WriteDREAM3D:Invalid File` | kept | Same free-function call with an empty path (4 cases via `GENERATE`); asserts the underlying `HDF5::FileIO::WriteFile` failure is surfaced, not `AtomicFile`'s. | +| `DREAM3DFileTest:DREAM3D File IO Test` | kept | The primary Class 1 content-fidelity test. Builds every geometry/`DataObject` type (`CreateTestDataStructure`), writes + reads back (`writeXdmf ∈ {true,false}`), and asserts full structural/content equality (`CheckTestDataStructure`) plus pipeline round-trip (`pipeline.size()==3`, filter names by index). | +| `DREAM3DFileTest::StringArray` | kept | Round-trips a `StringArray` through the actual `WriteDREAM3DFilter`/`ReadDREAM3DFilter` classes (not the free function) — covers Path 10 with real filter execution. | +| `DREAM3DFileTest:Import/Export DREAM3D Filter Test` | kept | Executes `WriteDREAM3DFilter` inside a real `Pipeline` (`exportPipeline.execute()`) — the only test exercising Path 9 (non-null `PipelineNode` with a real preceding pipeline). Also checks preflight-imported vs. executed array store types on the read side. | +| `DREAM3DFileTest:Import/Export Multi-DREAM3D Filter Test` | kept | Two independent export pipelines (`CreateMultiExportFiles`) each executing `WriteDREAM3DFilter` in-pipeline, then a single import pipeline consuming both files. Second confirmation of Path 9. | +| `DREAM3DFileTest: Preflight imports geometry connectivity as metadata-only stores` | kept | Read-side only (consumes a pre-existing `geoms.dream3d` asset); does not exercise `WriteDREAM3DFilter`. Listed for completeness since it shares the tag set. | +| `SimplnxCore::WriteDREAM3DFilter: SIMPL Backwards Compatibility` | kept | `DYNAMIC_SECTION` over SIMPL 6.5 (UUID-keyed) and 6.4 (name-keyed) `DataContainerWriter` fixtures. Asserts UUID resolution, empty comments, and `export_file_path`/`write_xdmf_file` value conversion. Not an oracle test — argument-mapping check only. | +| `DREAM3DFileTest: DataArray datasets are chunked+deflated when WriteOptions requests it` | kept | Calls the `WriteOptions`-aware free function directly with a 2 MB array; asserts chunked+deflate layout at level 5 via `ProbeHdf5Dataset`, then round-trips content. Covers the compression-options contract the Algorithm class relies on. | +| `WriteDREAM3DFilter: Compression_Off_IsContiguous` | kept | Full filter execute, `use_compression=false`. Asserts contiguous/no-deflate layout. Covers Path 12. | +| `WriteDREAM3DFilter: Compression_On_IsChunkedAndDeflated` | kept | Full filter execute, `use_compression=true`, level 5, 500K-element array. Asserts chunked+deflate layout and round-trips content. Covers Path 11. | +| `WriteDREAM3DFilter: Compression_SmallArray_Bypasses` | kept | Full filter execute with one <16 KiB and one >16 KiB array in the same `DataStructure`, `use_compression=true`. Asserts the small array stays contiguous/uncompressed while the large one is chunked+deflated — the Class 1 closed-form threshold check. | +| `WriteDREAM3DFilter: Compression_LevelsRoundTrip` | kept | Full filter execute at levels {1,5,9} on the same 1M-element pattern. Round-trips content at each level and asserts non-increasing file size as level rises (Class 4 companion invariant). | +| `WriteDREAM3DFilter: Compression_Preflight_RejectsOutOfRangeLevel` | kept | Three sequential preflight-only checks: level=0 (invalid), level=10 (invalid), level=0 with compression off (valid — ignored). Covers Paths 2, 3, 4. | +| `DREAM3DFileTest: PreflightCache avoids re-reading unchanged files` | kept | Uses `DREAM3D::WriteFile` only to create read-side fixture files; does not exercise `WriteDREAM3DFilter`'s own behavior. Listed for completeness since it shares source-file/tag space. | + +## Exemplar archive + +None. Every test above builds its input `DataStructure` inline in C++ and never loads a downloaded `.tar.gz` exemplar — the Class 1 oracle's "expected output" is the test's own hand-built input, so no cached golden file is required or used. (`Small_IN100_dream3d_v3.tar.gz`, referenced elsewhere in this test file, backs unrelated `ReadDREAM3DFilter`-only test cases and is not consumed by any Write-side test.) + +## Known limitations (current simplnx HDF5 IO layer) + +`DynamicListArray` (the generic variable-length per-tuple list container that `NeighborList` is itself built on) has **no registered HDF5 IO factory**. `HDF5::DataIOManager::addCoreFactories()` (`DataStructure/IO/HDF5/DataIOManager.cpp:33-81`) registers factories per concrete `DataArray` type, per `NeighborList` specialization, geometries, `AttributeMatrix`, `DataGroup`, `StringArray`, and scalar attributes — but nothing for a bare `DynamicListArray`. If one is ever placed directly in a `DataStructure` (outside of a `NeighborList` specialization) and written, `HDF5::DataStructureWriter::WriteFile` hits its "no factory found" guard (`DataStructureWriter.cpp:153-157`) and fails with error `-5` ("Could not find IO factory for datatype: …"), surfacing through `WriteDREAM3DFilter` as Path 13 above. + +This is a gap in the shared HDF5 IO layer, not something specific to `WriteDREAM3DFilter`'s own algorithm — the same gap would affect `ReadDREAM3DFilter` for the same object type. It is not raised as a formal Deviation because there is no confirmed 6.5.171 pipeline behavior being compared against (unlike D2, which names concrete legacy types); it is recorded here as a known, currently-untested capability boundary of what this filter can serialize. No current `WriteDREAM3DFilter` test constructs a bare `DynamicListArray`, so Path 13 remains untested rather than confirmed-safe. + +## Deviations from DREAM3D 6.5.171 + +No byte/dataset-level A/B was run against 6.5.171 `DataContainerWriter` output, and none is warranted: the two writers target deliberately different on-disk contracts (see D1), so such a diff would be 100% noise by design, not signal. `ReadDREAM3DFilter` is the only tool that understands both formats, and cross-format fidelity is its concern, not this filter's. Two scope/format deviations are documented instead: + +- `WriteDREAM3DFilter-D1` — on-disk file format is a deliberate, complete rewrite (v8 `DataStructure` layout vs. legacy v7 `DataContainers` layout; optional gzip compression; atomic write) — see `vv/deviations/WriteDREAM3DFilter.md`. +- `WriteDREAM3DFilter-D2` — `StatsDataArray`/`StructArray` (SIMPL ensemble-statistics types) cannot currently be written because those `DataObject` types do not yet exist in this branch of simplnx — see `vv/deviations/WriteDREAM3DFilter.md`. + +Neither is a bug: D1 is the intended outcome of the Rewrite classification (defended above), and D2 is a scope boundary pending a separate in-progress port of those data types. diff --git a/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md new file mode 100644 index 0000000000..e280424a14 --- /dev/null +++ b/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md @@ -0,0 +1,43 @@ +# Deviations from DREAM3D 6.5.171: WriteDREAM3DFilter + +This file lists every documented behavioral difference between this SIMPLNX filter and its DREAM3D 6.5.171 equivalent (`DataContainerWriter`). + +Entries are referenced by stable ID (`WriteDREAM3DFilter-D`) from the V&V report and from public migration guidance. The ID is stable across renames; the Filter UUID field is the permanent cross-reference anchor. + +--- + +## WriteDREAM3DFilter-D1 + +| Field | Value | +|---|---| +| **Deviation ID** | `WriteDREAM3DFilter-D1` | +| **Filter UUID** | `b3a95784-2ced-41ec-8d3d-0242ac130003` | +| **Status** | active | + +**Symptom:** A `.dream3d` file written by SIMPLNX is not byte-compatible with, and cannot be opened by, a DREAM3D 6.5.171 install expecting the legacy layout — and the reverse is only possible through SIMPLNX's dedicated legacy-import code path (`ReadDREAM3DFilter`'s legacy `DataContainers` reader), not by treating the file as interchangeable. + +**Root cause:** Algorithmic choice. SIMPLNX writes a clean-sheet v8 HDF5 layout: a top-level `DataStructure` group (vs. legacy's `DataContainers` group), a `FileVersion` attribute of `"8.0"` (vs. legacy `"7.0"`), an embedded JSON pipeline representation (vs. legacy's own pipeline serialization), atomic-rename write semantics (`AtomicFile`, so a crash mid-write cannot leave a corrupt file at the destination path), and an optional gzip/deflate compression scheme for `DataArray`/`NeighborList` datasets that has no legacy equivalent. This has been true since the filter's introduction — it was never a translation of legacy `DataContainerWriter`'s C++ (see Algorithm Relationship in the V&V report). + +**Affected users:** Anyone attempting to open a SIMPLNX-written `.dream3d` file directly in a DREAM3D 6.5.171 install, or vice versa, outside of DREAM3D-NX's own dual-format `ReadDREAM3DFilter`. Users staying entirely within DREAM3D-NX (write with `WriteDREAM3DFilter`, read with `ReadDREAM3DFilter`) never observe this — round-trip fidelity within the new format is verified by the Class 1 tests in the V&V report. + +**Why no A/B comparison was attempted:** Because the two writers deliberately target different on-disk contracts, a byte-level or dataset-level A/B diff between a SIMPLNX-written file and a 6.5.171-written file would not measure anything meaningful — every dataset path, group name, and file-version tag differs by construction, so the diff would be 100% noise, not signal. This is why the V&V report's "Legacy comparison" is marked **Not run — and not applicable** rather than "deferred." `ReadDREAM3DFilter` is the only tool in either codebase that understands both the legacy `DataContainers` layout and the current `DataStructure` layout; it is the correct place to verify cross-format fidelity (import old → re-export new → confirm no data loss), not a Write-side A/B. + +**Recommendation:** Trust SIMPLNX. The new format is a deliberate design improvement (atomicity, compression, JSON pipeline embedding) required for capabilities legacy DREAM3D never had (out-of-core datasets, versioned pipeline metadata). It is not "wrong" relative to 6.5.171; it is a different, and newer, on-disk contract. DREAM3D-NX remains able to *read* legacy 6.5.171 files via `ReadDREAM3DFilter`'s legacy import path, so migration is one-directional by design (import old, export new) rather than bidirectional. + +--- + +## WriteDREAM3DFilter-D2 + +| Field | Value | +|---|---| +| **Deviation ID** | `WriteDREAM3DFilter-D2` | +| **Filter UUID** | `b3a95784-2ced-41ec-8d3d-0242ac130003` | +| **Status** | active | + +**Symptom:** A pipeline that would have produced `StatsDataArray` or `StructArray` objects in legacy DREAM3D (e.g., ensemble statistics from a "Generate Ensemble Statistics"-style filter) cannot have those objects written to a `.dream3d` file by `WriteDREAM3DFilter` in the current develop branch — there is nothing in the `DataStructure` for the writer to serialize, because the types themselves do not exist yet. + +**Root cause:** Library (incomplete port, out of scope for this V&V cycle). `StatsDataArray` and `StructArray` are DREAM3D 6.5.171 `DataObject` types whose simplnx equivalents are being implemented on a separate, not-yet-merged branch. `WriteDREAM3DFilter`'s own write path (`Algorithms/WriteDREAM3D.cpp`) has no special-case logic to reject or special-case these types — the gap is entirely upstream, in `DataStructure`/`HDF5::DataStructureWriter` not yet having a type to construct. This V&V pass covers everything `WriteDREAM3DFilter` can currently write; it makes no claim about statistics data because that data cannot currently exist in a simplnx `DataStructure`. + +**Affected users:** Anyone migrating a legacy pipeline that computes per-ensemble statistics. Their exported SIMPLNX `.dream3d` file will simply lack the statistics group entirely (there being no such object to write) until the separate `StatsDataArray`/`StructArray` port lands and this filter is re-verified against it. + +**Recommendation:** Trust 6.5.171 for statistics data until the pending port lands. This is a temporary feature gap, not a correctness defect in `WriteDREAM3DFilter` itself — re-run this V&V cycle's oracle tests once `StatsDataArray`/`StructArray` exist in this branch to confirm the writer handles them correctly. diff --git a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp index 395f14e831..9d4c114b2d 100644 --- a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp +++ b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp @@ -2370,7 +2370,7 @@ Result DREAM3D::ReadFile(const nx::core::HDF5::FileIO& fileRe } auto dataStructure = ImportDataStructureFromFile(fileReader, preflight); - if(pipeline.invalid()) + if(dataStructure.invalid()) { return {{nonstd::make_unexpected(std::move(dataStructure.errors()))}, std::move(dataStructure.warnings())}; }