VV: Create Element Array from Feature Array fully V&V'ed

src/Plugins/SimplnxCore/docs/CopyFeatureArrayToElementArrayFilter.md

@@ -6,12 +6,27 @@ Core (Memory/Management)
 
 ## Description
 
-This **Filter** copies the values associated with a **Feature** to all the **Elements** that belong to that **Feature**.  Xmdf visualization files write only the **Element** attributes, so if the user wants to display a spatial map of a **Feature** level attribute, this **Filter** will transfer that information down to the **Element** level.
+This **Filter** copies the values associated with a **Feature** to all the **Elements** that belong to that **Feature**. For every **Element** `i`, the created array's tuple is the selected **Feature** array's tuple at index `FeatureIds[i]`. Xdmf visualization files write only the **Element** attributes, so if the user wants to display a spatial map of a **Feature** level attribute, this **Filter** will transfer that information down to the **Element** level.
+
+Multiple **Feature** arrays may be selected and copied in a single filter instance; all selected arrays must have the same number of tuples. Each created **Element** array is placed next to the selected *Cell Feature Ids* array and is named by appending the *Created Array Suffix* to the source array's name (e.g., `AvgTemp` with suffix `_Cell` creates `AvgTemp_Cell`). The created arrays keep the source array's data type and component layout.
+
+Note that each created array is **Element**-sized (number of **Elements** × components × bytes per value), so selecting many **Feature** arrays multiplies the additional memory required.
+
+### Input Validation
+
+- All selected **Feature** arrays must have the same number of tuples (error -3020).
+- The *Created Array Suffix* must not contain a `/` character (error -3021).
+- Every value in the *Cell Feature Ids* array must be non-negative. A negative value stops the filter with an error (-5355).
+- The largest **Feature** Id must be a valid tuple index into each selected **Feature** array; otherwise the filter stops with an error (-5351).
+- A **Feature** array with *more* tuples than the largest **Feature** Id requires is accepted — the extra tuples are simply never read. (Legacy DREAM3D 6.5 rejected this case.)
 
 % Auto generated parameter table will be inserted here
 
 ## Example Pipelines
 
+- (02) Image Segmentation (ITKImageProcessing)
+- (04) Porosity Analysis (ITKImageProcessing)
+
 ## License & Copyright
 
 Please see the description file distributed with this **Plugin**

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/CopyFeatureArrayToElementArray.cpp

@@ -1,10 +1,13 @@
 #include "CopyFeatureArrayToElementArray.hpp"
 
 #include "simplnx/DataStructure/DataArray.hpp"
+#include "simplnx/DataStructure/DataStore.hpp"
 #include "simplnx/Utilities/DataArrayUtilities.hpp"
 #include "simplnx/Utilities/ParallelAlgorithmUtilities.hpp"
 #include "simplnx/Utilities/ParallelDataAlgorithm.hpp"
 
+#include <algorithm>
+
 using namespace nx::core;
 
 namespace
@@ -15,17 +18,45 @@ class CopyFeatureArrayToElementArrayImpl
 public:
   using StoreType = AbstractDataStore<T>;
 
-  CopyFeatureArrayToElementArrayImpl(const IDataArray* selectedFeatureArray, const Int32AbstractDataStore& featureIdsStore, IDataArray* createdArray, const std::atomic_bool& shouldCancel)
-  : m_SelectedFeature(selectedFeatureArray->template getIDataStoreRefAs<StoreType>())
+  CopyFeatureArrayToElementArrayImpl(const IDataArray& selectedFeatureArray, const Int32AbstractDataStore& featureIdsStore, IDataArray& createdArray, const std::atomic_bool& shouldCancel)
+  : m_SelectedFeatureStore(selectedFeatureArray.getIDataStoreRefAs<StoreType>())
   , m_FeatureIdsStore(featureIdsStore)
-  , m_CreatedStore(createdArray->template getIDataStoreRefAs<StoreType>())
+  , m_CreatedStore(createdArray.getIDataStoreRefAs<StoreType>())
   , m_ShouldCancel(shouldCancel)
   {
+    // Raw-pointer fast path is only taken when all three stores are concrete in-memory
+    // DataStore<T> objects. Each thread then writes a disjoint index range of a plain
+    // buffer, so there is no shared mutable store state. Any other store type (e.g.
+    // out-of-core) falls back to the virtual AbstractDataStore access path, which runs
+    // serially because IParallelAlgorithm disables parallelization for OOC data.
+    const auto* selectedFeatureStorePtr = dynamic_cast<const DataStore<T>*>(&m_SelectedFeatureStore);
+    const auto* featureIdsStorePtr = dynamic_cast<const DataStore<int32>*>(&m_FeatureIdsStore);
+    auto* createdStorePtr = dynamic_cast<DataStore<T>*>(&m_CreatedStore);
+    if(selectedFeatureStorePtr != nullptr && featureIdsStorePtr != nullptr && createdStorePtr != nullptr)
+    {
+      m_SourcePtr = selectedFeatureStorePtr->data();
+      m_FeatureIdsPtr = featureIdsStorePtr->data();
+      m_DestPtr = createdStorePtr->data();
+    }
   }
 
   void operator()(const Range& range) const
   {
-    const usize totalFeatureArrayComponents = m_SelectedFeature.getNumberOfComponents();
+    const usize numComps = m_SelectedFeatureStore.getNumberOfComponents();
+
+    if(m_SourcePtr != nullptr)
+    {
+      for(usize i = range.min(); i < range.max(); ++i)
+      {
+        if((i & 0xFFFFULL) == 0ULL && m_ShouldCancel)
+        {
+          return;
+        }
+        // Copy the feature tuple indexed by this cell's feature id down to the cell-level array
+        std::copy_n(m_SourcePtr + numComps * static_cast<usize>(m_FeatureIdsPtr[i]), numComps, m_DestPtr + numComps * i);
+      }
+      return;
+    }
 
     for(usize i = range.min(); i < range.max(); ++i)
     {
@@ -34,25 +65,28 @@ class CopyFeatureArrayToElementArrayImpl
         return;
       }
 
-      for(usize faComp = 0; faComp < totalFeatureArrayComponents; faComp++)
+      const auto featureId = static_cast<usize>(m_FeatureIdsStore[i]);
+      for(usize comp = 0; comp < numComps; comp++)
       {
-        // Get the feature identifier (or what ever the user has selected as their "Feature" identifier
-        m_CreatedStore[totalFeatureArrayComponents * i + faComp] = m_SelectedFeature[totalFeatureArrayComponents * m_FeatureIdsStore[i] + faComp];
+        m_CreatedStore[numComps * i + comp] = m_SelectedFeatureStore[numComps * featureId + comp];
       }
     }
   }
 
 private:
-  const StoreType& m_SelectedFeature;
+  const StoreType& m_SelectedFeatureStore;
   const Int32AbstractDataStore& m_FeatureIdsStore;
   StoreType& m_CreatedStore;
   const std::atomic_bool& m_ShouldCancel;
+  const T* m_SourcePtr = nullptr;
+  const int32* m_FeatureIdsPtr = nullptr;
+  T* m_DestPtr = nullptr;
 };
 } // namespace
 
 // -----------------------------------------------------------------------------
 CopyFeatureArrayToElementArray::CopyFeatureArrayToElementArray(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel,
-                                                               CopyFeatureArrayToElementArrayInputValues* inputValues)
+                                                               const CopyFeatureArrayToElementArrayInputValues* inputValues)
 : m_DataStructure(dataStructure)
 , m_InputValues(inputValues)
 , m_ShouldCancel(shouldCancel)
@@ -66,24 +100,38 @@ CopyFeatureArrayToElementArray::~CopyFeatureArrayToElementArray() noexcept = def
 // -----------------------------------------------------------------------------
 Result<> CopyFeatureArrayToElementArray::operator()()
 {
+  if(m_InputValues->SelectedFeatureArrayPaths.empty())
+  {
+    return {};
+  }
+
   const auto& featureIds = m_DataStructure.getDataRefAs<Int32Array>(m_InputValues->FeatureIdsPath);
 
-  for(const auto& selectedFeatureArrayPath : m_InputValues->SelectedFeatureArrayPaths)
+  // Validate the FeatureIds value range once, against the first selected array. Preflight has
+  // already verified (error -3020) that every selected feature array has the same tuple count,
+  // so a single validation covers all of them and avoids re-scanning FeatureIds per array.
+  auto validateNumFeatResult = ValidateFeatureIdsToFeatureAttributeMatrixIndexing(m_DataStructure, m_InputValues->SelectedFeatureArrayPaths[0], featureIds, false, m_MessageHandler);
+  if(validateNumFeatResult.invalid())
   {
-    DataPath createdArrayPath = m_InputValues->FeatureIdsPath.replaceName(selectedFeatureArrayPath.getTargetName() + m_InputValues->CreatedArraySuffix);
-    const auto* selectedFeatureArray = m_DataStructure.getDataAs<IDataArray>(selectedFeatureArrayPath);
+    return validateNumFeatResult;
+  }
 
-    auto validateNumFeatResult = ValidateFeatureIdsToFeatureAttributeMatrixIndexing(m_DataStructure, selectedFeatureArrayPath, featureIds, false, m_MessageHandler);
-    if(validateNumFeatResult.invalid())
+  for(const auto& selectedFeatureArrayPath : m_InputValues->SelectedFeatureArrayPaths)
+  {
+    if(m_ShouldCancel)
     {
-      return validateNumFeatResult;
+      return {};
     }
 
+    DataPath createdArrayPath = m_InputValues->FeatureIdsPath.replaceName(selectedFeatureArrayPath.getTargetName() + m_InputValues->CreatedArraySuffix);
+    const auto& selectedFeatureArray = m_DataStructure.getDataRefAs<IDataArray>(selectedFeatureArrayPath);
+    auto& createdArray = m_DataStructure.getDataRefAs<IDataArray>(createdArrayPath);
+
     m_MessageHandler(IFilter::ProgressMessage{IFilter::ProgressMessage::Type::Info, fmt::format("Copying data into target array '{}'...", createdArrayPath.toString())});
     ParallelDataAlgorithm dataAlg;
     dataAlg.setRange(0, featureIds.getNumberOfTuples());
-    ExecuteParallelFunction<::CopyFeatureArrayToElementArrayImpl>(selectedFeatureArray->getDataType(), dataAlg, selectedFeatureArray, featureIds.getDataStoreRef(),
-                                                                  m_DataStructure.getDataAs<IDataArray>(createdArrayPath), m_ShouldCancel);
+    dataAlg.requireArraysInMemory({&featureIds, &selectedFeatureArray, &createdArray});
+    ExecuteParallelFunction<::CopyFeatureArrayToElementArrayImpl>(selectedFeatureArray.getDataType(), dataAlg, selectedFeatureArray, featureIds.getDataStoreRef(), createdArray, m_ShouldCancel);
   }
 
   return {};

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/CopyFeatureArrayToElementArray.hpp

@@ -21,21 +21,25 @@ struct SIMPLNXCORE_EXPORT CopyFeatureArrayToElementArrayInputValues
 
 /**
  * @class CopyFeatureArrayToElementArray
- * @brief This algorithm implements support code for the CopyFeatureArrayToElementArrayFilter
+ * @brief Copies each selected Feature-level array down to the Element (cell) level: for every
+ * Element i, the created array's tuple is the source array's tuple at index FeatureIds[i].
  */
-
 class SIMPLNXCORE_EXPORT CopyFeatureArrayToElementArray
 {
 public:
   CopyFeatureArrayToElementArray(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel,
-                                 CopyFeatureArrayToElementArrayInputValues* inputValues);
+                                 const CopyFeatureArrayToElementArrayInputValues* inputValues);
   ~CopyFeatureArrayToElementArray() noexcept;
 
   CopyFeatureArrayToElementArray(const CopyFeatureArrayToElementArray&) = delete;
   CopyFeatureArrayToElementArray(CopyFeatureArrayToElementArray&&) noexcept = delete;
   CopyFeatureArrayToElementArray& operator=(const CopyFeatureArrayToElementArray&) = delete;
   CopyFeatureArrayToElementArray& operator=(CopyFeatureArrayToElementArray&&) noexcept = delete;
 
+  /**
+   * @brief Runs the copy for every selected feature array.
+   * @return Invalid Result on FeatureIds range-validation failure (-5355 / -5351).
+   */
   Result<> operator()();
 
 private:

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/CopyFeatureArrayToElementArrayFilter.cpp

@@ -6,6 +6,8 @@
 #include "simplnx/DataStructure/DataPath.hpp"
 #include "simplnx/Filter/Actions/CreateArrayAction.hpp"
 #include "simplnx/Parameters/ArraySelectionParameter.hpp"
+// DataObjectNameParameter.hpp and MultiPathSelectionParameter.hpp provide the SIMPLConversion
+// converters (LinkedPathCreation..., SingleToMultiDataPathSelection...) used in FromSIMPLJson()
 #include "simplnx/Parameters/DataObjectNameParameter.hpp"
 #include "simplnx/Parameters/MultiArraySelectionParameter.hpp"
 #include "simplnx/Parameters/MultiPathSelectionParameter.hpp"
@@ -52,9 +54,9 @@ Parameters CopyFeatureArrayToElementArrayFilter::parameters() const
 
   // Create the parameter descriptors that are needed for this filter
   params.insertSeparator(Parameters::Separator{"Input Feature Data"});
-  params.insert(std::make_unique<MultiArraySelectionParameter>(k_SelectedFeatureArrayPath_Key, "Feature Data to Copy to Cell Data",
+  params.insert(std::make_unique<MultiArraySelectionParameter>(k_SelectedFeatureArrayPaths_Key, "Feature Data to Copy to Cell Data",
                                                                "The DataPath to the feature data that should be copied to the cell level", MultiArraySelectionParameter::ValueType{},
-                                                               MultiArraySelectionParameter::AllowedTypes{IArray::ArrayType::Any}, nx::core::GetAllDataTypes()));
+                                                               MultiArraySelectionParameter::AllowedTypes{IArray::ArrayType::DataArray}, nx::core::GetAllDataTypes()));
 
   params.insertSeparator(Parameters::Separator{"Input Cell Data"});
   params.insert(std::make_unique<ArraySelectionParameter>(k_CellFeatureIdsArrayPath_Key, "Cell Feature Ids", "Specifies to which feature each cell belongs.", DataPath({"Cell Data", "FeatureIds"}),
@@ -81,9 +83,9 @@ IFilter::UniquePointer CopyFeatureArrayToElementArrayFilter::clone() const
 IFilter::PreflightResult CopyFeatureArrayToElementArrayFilter::preflightImpl(const DataStructure& dataStructure, const Arguments& filterArgs, const MessageHandler& messageHandler,
                                                                              const std::atomic_bool& shouldCancel, const ExecutionContext& executionContext) const
 {
-  const auto pSelectedFeatureArrayPathsValue = filterArgs.value<MultiArraySelectionParameter::ValueType>(k_SelectedFeatureArrayPath_Key);
-  const auto pFeatureIdsArrayPathValue = filterArgs.value<DataPath>(k_CellFeatureIdsArrayPath_Key);
-  const auto createdArraySuffix = filterArgs.value<StringParameter::ValueType>(k_CreatedArraySuffix_Key);
+  const auto pSelectedFeatureArrayPathsValue = filterArgs.value<MultiArraySelectionParameter::ValueType>(k_SelectedFeatureArrayPaths_Key);
+  const auto pFeatureIdsArrayPathValue = filterArgs.value<ArraySelectionParameter::ValueType>(k_CellFeatureIdsArrayPath_Key);
+  const auto pCreatedArraySuffixValue = filterArgs.value<StringParameter::ValueType>(k_CreatedArraySuffix_Key);
 
   nx::core::Result<OutputActions> resultOutputActions;
   std::vector<PreflightValue> preflightUpdatedValues;
@@ -93,6 +95,11 @@ IFilter::PreflightResult CopyFeatureArrayToElementArrayFilter::preflightImpl(con
     return MakePreflightErrorResult(nx::core::FilterParameter::Constants::k_Validate_Empty_Value, "You must select at least one feature data array to copy to an element data array.");
   }
 
+  if(pCreatedArraySuffixValue.find('/') != std::string::npos)
+  {
+    return MakePreflightErrorResult(-3021, fmt::format("The Created Array Suffix '{}' must not contain a '/' character.", pCreatedArraySuffixValue));
+  }
+
   auto tupleValidityCheck = dataStructure.validateNumberOfTuples(pSelectedFeatureArrayPathsValue);
   if(!tupleValidityCheck)
   {
@@ -101,14 +108,14 @@ IFilter::PreflightResult CopyFeatureArrayToElementArrayFilter::preflightImpl(con
 
   const auto& featureIdsArray = dataStructure.getDataRefAs<IDataArray>(pFeatureIdsArrayPathValue);
   const IDataStore& featureIdsArrayStore = featureIdsArray.getIDataStoreRef();
-  const std::vector<usize>& tDims = featureIdsArrayStore.getTupleShape();
+  const std::vector<usize>& tupleShape = featureIdsArrayStore.getTupleShape();
 
   for(const auto& selectedFeatureArrayPath : pSelectedFeatureArrayPathsValue)
   {
-    DataPath createdArrayPath = pFeatureIdsArrayPathValue.replaceName(selectedFeatureArrayPath.getTargetName() + createdArraySuffix);
+    DataPath createdArrayPath = pFeatureIdsArrayPathValue.replaceName(selectedFeatureArrayPath.getTargetName() + pCreatedArraySuffixValue);
     const auto& selectedFeatureArray = dataStructure.getDataRefAs<IDataArray>(selectedFeatureArrayPath);
     DataType dataType = selectedFeatureArray.getDataType();
-    auto createArrayAction = std::make_unique<CreateArrayAction>(dataType, tDims, selectedFeatureArray.getComponentShape(), createdArrayPath);
+    auto createArrayAction = std::make_unique<CreateArrayAction>(dataType, tupleShape, selectedFeatureArray.getComponentShape(), createdArrayPath);
     resultOutputActions.value().appendAction(std::move(createArrayAction));
   }
 
@@ -120,7 +127,7 @@ Result<> CopyFeatureArrayToElementArrayFilter::executeImpl(DataStructure& dataSt
                                                            const std::atomic_bool& shouldCancel, const ExecutionContext& executionContext) const
 {
   CopyFeatureArrayToElementArrayInputValues inputValues;
-  inputValues.SelectedFeatureArrayPaths = filterArgs.value<MultiArraySelectionParameter::ValueType>(k_SelectedFeatureArrayPath_Key);
+  inputValues.SelectedFeatureArrayPaths = filterArgs.value<MultiArraySelectionParameter::ValueType>(k_SelectedFeatureArrayPaths_Key);
   inputValues.FeatureIdsPath = filterArgs.value<ArraySelectionParameter::ValueType>(k_CellFeatureIdsArrayPath_Key);
   inputValues.CreatedArraySuffix = filterArgs.value<StringParameter::ValueType>(k_CreatedArraySuffix_Key);
 
@@ -144,7 +151,7 @@ Result<Arguments> CopyFeatureArrayToElementArrayFilter::FromSIMPLJson(const nloh
   std::vector<Result<>> results;
 
   results.push_back(
-      SIMPLConversion::ConvertParameter<SIMPLConversion::SingleToMultiDataPathSelectionFilterParameterConverter>(args, json, SIMPL::k_SelectedFeatureArrayPathKey, k_SelectedFeatureArrayPath_Key));
+      SIMPLConversion::ConvertParameter<SIMPLConversion::SingleToMultiDataPathSelectionFilterParameterConverter>(args, json, SIMPL::k_SelectedFeatureArrayPathKey, k_SelectedFeatureArrayPaths_Key));
   results.push_back(SIMPLConversion::ConvertParameter<SIMPLConversion::DataArraySelectionFilterParameterConverter>(args, json, SIMPL::k_FeatureIdsArrayPathKey, k_CellFeatureIdsArrayPath_Key));
   // Do NOT map the legacy CreatedArrayName onto k_CreatedArraySuffix_Key: the legacy filter converted a
   // single array whose output name was CreatedArrayName, but in SIMPLNX that string would be appended to

src/Plugins/SimplnxCore/src/SimplnxCore/Filters/CopyFeatureArrayToElementArrayFilter.hpp

@@ -10,7 +10,7 @@ namespace nx::core
 /**
  * @class CopyFeatureArrayToElementArrayFilter
  * @brief This Filter copies the values associated with a Feature to all the Elements
- * that belong to that Feature. Xmdf visualization files write only the Element attributes,
+ * that belong to that Feature. Xdmf visualization files write only the Element attributes,
  * so if the user wants to display a spatial map of a Feature level attribute,
  * this Filter will transfer that information down to the Element level.
  */
@@ -27,7 +27,7 @@ class SIMPLNXCORE_EXPORT CopyFeatureArrayToElementArrayFilter : public IFilter
   CopyFeatureArrayToElementArrayFilter& operator=(CopyFeatureArrayToElementArrayFilter&&) noexcept = delete;
 
   // Parameter Keys
-  static constexpr StringLiteral k_SelectedFeatureArrayPath_Key = "selected_feature_array_paths";
+  static constexpr StringLiteral k_SelectedFeatureArrayPaths_Key = "selected_feature_array_paths";
   static constexpr StringLiteral k_CellFeatureIdsArrayPath_Key = "feature_ids_path";
   static constexpr StringLiteral k_CreatedArraySuffix_Key = "created_array_suffix";