Adding DualView examples for constexpr (17) and the copy constructor. WIP on the view with two meshes

Author: JulienLoiseau

CMakeLists.txt

@@ -0,0 +1,15 @@
+cmake_minimum_required(VERSION 3.10)
+project(mesh_prototype)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+find_package(MPI REQUIRED)
+find_package(Kokkos REQUIRED)
+find_package(CUDA REQUIRED)
+
+add_executable(mesh_DV_constexpr mesh_DV_constexpr.cc)
+target_link_libraries(mesh_DV_constexpr PUBLIC Kokkos::kokkoscore)
+
+add_executable(mesh_DV_layer mesh_DV_layer.cc)
+target_link_libraries(mesh_DV_layer PUBLIC Kokkos::kokkoscore)

base_class_example.hpp

@@ -0,0 +1,30 @@
+template<typename Data, typename ExecutionSpace = Kokkos::DefaultExecutionSpace>
+struct CSR;
+
+
+//
+// Note, this API is consistent with amanzi/ecoon/mesh_refactor src/mesh/MeshCache.hh API.
+//
+// If it helped, these could be dual views?
+//
+template<typename ExecutionSpace = Kokkos::DefaultExecutionSpace>
+struct MeshCache {
+
+  using Double_View = Kokkos::View<double*, ExecutionSpace>; // ?????
+  using cDouble_View = Kokkos::View<const double*, ExecutionSpace>; // ?????
+  using Entity_ID_View = Kokkos::View<Entity_ID*, ExecutionSpace>;
+  using cEntity_ID_View = Kokkos::View<const Entity_ID*, ExecutionSpace>;
+
+  MeshCache(Teuchos::RCP<MeshFramework>& mf); // note, feel free to make shared_ptr or raw ptr for now...
+
+  double getCellVolume(const Entity_ID c) const;
+  Double_View cell_volumes; // note, this is public and can be used inside kernels directly
+  // is there a const problem with exposing this directly?  Specifically,
+  // assume you have a const MeshCache.  Can you still change
+  // mc.cell_volumes(3) = 1?  Should this be cDoubleView?
+
+  std::size_t getCellNumFaces(const Entity_ID c) const;
+  cEntity_ID_View getCellFaces(const Entity_ID c) const;
+  Entity_ID getCellFace(Entity_ID c, std::size_t i) const;
+  CRS<Entity_ID, ExecutionSpace> cell_faces;
+}

mesh_kokkos.cc mesh_DV_constexpr.ccmesh_kokkos.cc renamed to mesh_DV_constexpr.cc

@@ -1,81 +1,59 @@
 #include "Kokkos_Core.hpp"
 #include "Kokkos_DualView.hpp"
 
+using Entity_ID = int;
+
+enum memory {host, device};
+
+
 //
 // Note, this API is consistent with amanzi/ecoon/mesh_refactor src/mesh/MeshFramework.hh API
 //
 struct MeshFramework {
   MeshFramework(const int size): mesh_size_(size) {}
 
   // In MeshFramework this function "compute" the result on Host
-  int getCellVolume(const int i) const {
+  double getCellVolume(const Entity_ID i) const {
     return i;
   }
 
   // In MeshFramework this function "compute" the result on Host
-  void getCellFaces(const int id, Kokkos::View<const int*,Kokkos::HostSpace>& ret) const {
+  Kokkos::View<const Entity_ID*,Kokkos::HostSpace> getCellFaces(const Entity_ID id) const {
     // 2 Elements per entries
-    Kokkos::View<int*, Kokkos::HostSpace> tmp("",2);
+    Kokkos::View<Entity_ID*, Kokkos::HostSpace> tmp("",2);
     for(int i = 0 ; i < 2; ++i){
       tmp(i) = value*i;
     }
-    ret = tmp;
+    return tmp;
   }
 
-  int meshSize() const { return mesh_size_; }
+  Entity_ID getCellFace(Entity_ID c, std::size_t i) const {
+    return value*i; 
+  }
+
+  Entity_ID meshSize() const { return mesh_size_; }
 
 private:
-  const int value = 10;
+  const Entity_ID value = 10;
   int mesh_size_;
 };
 
-
-// NOTE: need this...
-template<typename Data, typename ExecutionSpace = Kokkos::DefaultExecutionSpace>
-struct CSR;
-
-
-using Entity_ID = int;
-
-//
-// Note, this API is consistent with amanzi/ecoon/mesh_refactor src/mesh/MeshCache.hh API.
-//
-// If it helped, these could be dual views?
-//
+// MeshCache templated on the memory type
+// Defaulted to the device memory space
 template<typename ExecutionSpace = Kokkos::DefaultExecutionSpace>
 struct MeshCache {
+
   using Double_View = Kokkos::View<double*, ExecutionSpace>; // ?????
   using cDouble_View = Kokkos::View<const double*, ExecutionSpace>; // ?????
   using Entity_ID_View = Kokkos::View<Entity_ID*, ExecutionSpace>;
   using cEntity_ID_View = Kokkos::View<const Entity_ID*, ExecutionSpace>;
 
-  MeshCache(Teuchos::RCP<MeshFramework>& mf); // note, feel free to make shared_ptr or raw ptr for now...
-
-  double getCellVolume(const Entity_ID c) const;
-  Double_View cell_volumes; // note, this is public and can be used inside kernels directly
-  // is there a const problem with exposing this directly?  Specifically,
-  // assume you have a const MeshCache.  Can you still change
-  // mc.cell_volumes(3) = 1?  Should this be cDoubleView?
-
-  std::size_t getCellNumFaces(const Entity_ID c) const;
-  cEntity_ID_View getCellFaces(const Entity_ID c) const;
-  Entity_ID getCellFace(Entity_ID c, std::size_t i) const;
-  CRS<Entity_ID, ExecutionSpace> cell_faces;
-}
-
-
-
-
-// MeshCache templated on the memory type
-// Defaulted to the device memory space
-template<typename ExecutionSpace = Kokkos::DefaultExecutionSpace>
-struct MeshCache {
 
   // Type to get CRS on DualViews
-  template<typename T, typename ExecutionSpace>
+  template<typename T, typename ES>
   struct DualCrs{
-    Kokkos::DualView<int*, ExecutionSpace> row_map;
-    Kokkos::DualView<T, ExecutionSpace> entries;
+    Kokkos::DualView<int*, ES> row_map;
+    Kokkos::DualView<T, ES> entries;
   };
 
   // Building a Mesh Cache on ExecutionSpace memory space.
@@ -91,7 +69,7 @@ struct MeshCache {
     for(int i = 0 ; i < mesh_size_ ; ++i){
       value_2d_.row_map.view_host()(i) = i*2;
       Kokkos::View<const int*, Kokkos::HostSpace> tmp;
-      mf->getCellFaces(i,tmp);
+      tmp = mf->getCellFaces(i);
       for(int j = 0 ; j < 2; ++j){
         value_2d_.entries.view_host()(i*2+j) = tmp(j);
       }
@@ -104,51 +82,50 @@ struct MeshCache {
   }
 
   // This function can just be overloaded with the Kokkos::View type.
-  KOKKOS_INLINE_FUNCTION void getCellFaces(int id, Kokkos::View<const int*,ExecutionSpace>& ret) const {
-      ret = Kokkos::subview(value_2d_.entries.view_device(),
+  template<memory MEM = device>
+  KOKKOS_INLINE_FUNCTION auto getCellFaces(Entity_ID id) const {
+    if constexpr (MEM == device){
+      Kokkos::View<const int*,ExecutionSpace> ret = Kokkos::subview(value_2d_.entries.view_device(),
         Kokkos::make_pair(value_2d_.row_map.view_device()(id),
         value_2d_.row_map.view_device()(id + 1)));
-  }
-  void getCellFaces(int id, Kokkos::View<const int*,Kokkos::HostSpace>& ret) const {
-      Kokkos::subview(value_2d_.entries.view_host(),
+      return ret; 
+    } else if (MEM == host) { 
+      Kokkos::View<const Entity_ID*,Kokkos::HostSpace> ret = Kokkos::subview(value_2d_.entries.view_host(),
         Kokkos::make_pair(value_2d_.row_map.view_host()(id),
         value_2d_.row_map.view_host()(id + 1)));
+      return ret; 
+    }
   }
 
-  // The following function cannot be overloaded with parameters
-  // Default: only use Device function
-  KOKKOS_INLINE_FUNCTION int getCellVolume(int i) const {
-    return value_1d_.view_device()(i);
-  }
-
-  // Duplicate function with host decoration
-  int getCellVolume_host(int i) const {
-    return value_1d_.view_host()(i);
+  template<memory MEM = device>
+  KOKKOS_INLINE_FUNCTION Entity_ID getCellFace(Entity_ID c, int i) const {
+    if constexpr (MEM == device){
+      return value_2d_.entries.view_device()(value_2d_.row_map.view_device()(c)+i);
+    } else if (MEM == host){
+      return value_2d_.entries.view_host()(value_2d_.row_map.view_host()(c)+i); 
+    }
   }
 
-  // Using C++ 17 with constexpr
-  #if V2
-  template<typename FExecutionSpace = Kokkos::HostSpace>
-  int getCellVolume(int i) const {
-    if constexpr(std::is_same<Kokkos::HostSpace,FExecutionSpace>::value) {
-      return value_1d_.view_host()(i);
-    } else if (std::is_same<Kokkos::CudaSpace,FExecutionSpace>::value) {
+  // The following function cannot be overloaded with parameters
+  // Default: only use Device function
+  template<memory MEM = device>
+  KOKKOS_INLINE_FUNCTION double getCellVolume(Entity_ID i) const {
+    if constexpr (MEM == device){
       return value_1d_.view_device()(i);
+    } else if (MEM == host) {
+      return value_1d_.view_host()(i);
     }
   }
-  #endif
 
 private:
-  Kokkos::DualView<int*,ExecutionSpace> value_1d_;
-  DualCrs<int*,ExecutionSpace> value_2d_;
+  Kokkos::DualView<double*,ExecutionSpace> value_1d_;
+  DualCrs<Entity_ID*,ExecutionSpace> value_2d_;
   MeshFramework* framework_mesh_;
   int mesh_size_;
 };
 
-
-
-
 void test_mesh()  {
+
     using DeviceSpace = Kokkos::CudaSpace;
     const int ncells = 10;
     MeshFramework m(ncells);
@@ -157,38 +134,52 @@ void test_mesh()  {
     // ------- Mesh Framework -------
     // Only accessible on Host
     for(int i = 0 ; i < ncells ; ++i){
-      // 1d
-      assert(m.getCellVolume(i) == i);
-      // 2d
+      if(m.getCellVolume(i) != i)
+        printf("ERROR %.1f != %d \n",m.getCellVolume(i),i);
       Kokkos::View<const int*, Kokkos::HostSpace> v_h;
-      m.getCellFaces(2, v_h);
+      v_h = m.getCellFaces(2);
+      for(int j = 0 ; j < 2 ; ++j){
+        if(v_h(j) != j*10)
+          printf("ERROR %d != %d \n",v_h(j),j*10);
+      }
       for(int j = 0 ; j < 2 ; ++j){
-        assert(v_h(j) == j*10);
+        if(m.getCellFace(2,j) != j*10)
+          printf("ERROR %d != %d \n",m.getCellFace(2,j),j*10);
       }
     }
 
+    
     // ------- Mesh Cache -------
     // 1. Device Access (Default type of the mesh cache)
     Kokkos::parallel_for(
-      "",
-      ncells,
-      KOKKOS_LAMBDA(const int i){
-        // 1d
-        assert(mc.getCellVolume(i) == i);
-        // 2d
+      ncells, KOKKOS_LAMBDA(const int i){
+        if(mc.getCellVolume(i) != i)
+          printf("ERROR %.1f != %d \n",mc.getCellVolume(i),i);
         Kokkos::View<const int*, DeviceSpace> v_d;
-        mc.getCellFaces(2,v_d);
+        v_d = mc.getCellFaces(2);
         for(int j = 0 ; j < 2 ; ++j){
-          assert(v_d(j) == j*10);
+          if(v_d(j) != j*10)
+            printf("ERROR %d != %d \n",v_d(j),j*10);
+        }
+        for(int j = 0 ; j < 2 ; ++j){
+          if(mc.getCellFace(2,j) != j*10)
+            printf("ERROR %d != %d \n",mc.getCellFace(2,j),j*10);
         }
     });
     // 2. Host Access
     for(int i = 0 ; i < ncells ; ++i){
-      // 1d
-      assert(mc.getCellVolume_host(i) == i);
-      // 2d
+      if(mc.getCellVolume<memory::host>(i) != i)
+        printf("ERROR %.1f != %d \n",mc.getCellVolume<memory::host>(i),i);
       Kokkos::View<const int*, Kokkos::HostSpace> v_h;
-      mc.getCellFaces(2, v_h);
+      v_h = mc.getCellFaces<memory::host>(2);
+      for(int j = 0 ; j < 2 ; ++j){
+        if(v_h(j) != j*10)
+          printf("ERROR %d != %d \n",v_h(j),j*10);
+      }
+      for(int j = 0 ; j < 2 ; ++j){
+        if(mc.getCellFace<memory::host>(2,j) != j*10)
+          printf("ERROR %d != %d \n",mc.getCellFace<memory::host>(2,j),j*10);
+      }
     }
 } // test_mesh