Add global comm

src/pcms/coupler/CMakeLists.txt

@@ -7,6 +7,7 @@ set(PCMS_COUPLER_HEADERS
         field_exchange_planner.h
         partition.h
         overlap_mask.h
+        global_communicator.h
 )
 
 

src/pcms/coupler/coupler.hpp

@@ -11,11 +11,47 @@
 #include "pcms/utility/assert.h"
 #include "pcms/utility/common.h"
 #include "pcms/utility/profile.h"
+#include "pcms/coupler/global_communicator.h"
 #include <memory>
 
 namespace pcms
 {
+template <typename T>
+class GlobalDataInterface
+{
+public:
+  GlobalDataInterface(const std::string& name,
+                      MPI_Comm mpi_comm,
+                      redev::Channel& channel)
+    : mpi_comm_(mpi_comm),
+      type_info_(typeid(T)),
+      comm_(name, mpi_comm_, channel)
+  {
+    PCMS_FUNCTION_TIMER;
+  }
+
+  void SendData(T* msg,
+            std::string VarName,
+            size_t msg_size,
+            Mode mode = Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    comm_.Send(msg, std::move(VarName), msg_size, mode);
+  }
+
+  std::vector<T> ReceiveData(std::string VarName,
+                         size_t msg_size,
+                         Mode mode = Mode::Synchronous)
+  {
+    PCMS_FUNCTION_TIMER;
+    return comm_.Receive(std::move(VarName), msg_size, mode);
+  }
 
+private:
+  MPI_Comm mpi_comm_;
+  const std::type_info& type_info_;
+  GlobalCommunicator<T> comm_;
+};
 class Application;
 
 template <typename T>
@@ -70,6 +106,8 @@ class Application
   FieldHandle<T> AddField(std::string name, Field<T>&& field,
                           std::unique_ptr<FieldSerializer<T>> serializer,
                           bool participates = true);
+  template <typename T>
+  std::unique_ptr<GlobalDataInterface<T>> AddData(std::string name, MPI_Comm mpi_comm);
 
   void SendField(const std::string& name,
                  redev::Mode mode = redev::Mode::Synchronous)
@@ -162,6 +200,13 @@ class Application
   std::map<std::string, std::unique_ptr<OverlapMask>> layout_overlap_masks_;
 };
 
+template <typename T>
+std::unique_ptr<GlobalDataInterface<T>>
+Application::AddData(std::string name, MPI_Comm mpi_comm)
+{
+  PCMS_FUNCTION_TIMER;
+  return std::make_unique<GlobalDataInterface<T>>(name, mpi_comm, channel_);
+}
 class Coupler
 {
 private:

src/pcms/coupler/global_communicator.h

@@ -0,0 +1,48 @@
+#ifndef PCMS_GLOBAL_COMMUNICATOR_H
+#define PCMS_GLOBAL_COMMUNICATOR_H
+#include <redev.h>
+#include <pcms/utility/profile.h>
+namespace pcms
+{
+  using redev::Mode;
+  template <typename T>
+  struct GlobalCommunicator
+  {
+    using value_type = T;
+    public:
+    GlobalCommunicator(std::string name, MPI_Comm mpi_comm, redev::Channel& channel)
+      : mpi_comm(mpi_comm),
+        channel_(channel),
+        name_(std::move(name))
+    {
+      PCMS_FUNCTION_TIMER;
+      comm_ = channel_.CreateComm<T>(name_, mpi_comm, redev::CommType::Global );
+    }
+    GlobalCommunicator(const GlobalCommunicator&) = delete;
+    GlobalCommunicator& operator=(const GlobalCommunicator&) = delete;
+    GlobalCommunicator(GlobalCommunicator&&)= default;
+    GlobalCommunicator& operator=(GlobalCommunicator&&) = default;
+
+    void Send(T* msg, std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous)
+    {
+      PCMS_FUNCTION_TIMER;
+      PCMS_ALWAYS_ASSERT(channel_.InSendCommunicationPhase());
+      comm_.SetCommParams( VarName, msg_size);
+      comm_.Send(msg, mode);
+    }
+    std::vector<T> Receive(std::string VarName, size_t msg_size, Mode mode = Mode::Synchronous)
+    {
+      PCMS_FUNCTION_TIMER;
+      PCMS_ALWAYS_ASSERT(channel_.InReceiveCommunicationPhase());
+      comm_.SetCommParams(VarName, msg_size);
+      auto data = comm_.Recv(mode);
+      return data;
+    }
+  private:
+    MPI_Comm mpi_comm;
+    redev::Channel& channel_;
+    std::string name_;
+    redev::BidirectionalComm<T> comm_;
+  };
+}
+#endif // PCMS_GLOBAL_COMMUNICATOR_H

test/CMakeLists.txt

@@ -96,6 +96,37 @@ if(PCMS_ENABLE_OMEGA_H)
       ${d3d16p}
       ignored)
   endif()
+  add_exe(test_GDI)
+  tri_mpi_test(
+          TESTNAME
+          test_GDI
+          TIMEOUT
+          20
+          NAME1
+          app
+          EXE1
+          ./test_GDI
+          PROCS1
+          1
+          ARGS1
+          1
+          NAME2
+          rdv
+          EXE2
+          ./test_GDI
+          PROCS2
+          1
+          ARGS2
+          -1
+          NAME3
+          app
+          EXE3
+          ./test_GDI
+          PROCS3
+          1
+          ARGS3
+          0
+  )
 
   set(d3d8p ${PCMS_TEST_DATA_DIR}/d3d/d3d-full_9k_sfc_p8.osh/)
   add_exe(test_twoClientOverlap)
@@ -380,7 +411,6 @@ if(Catch2_FOUND)
       APPEND
       PCMS_UNIT_TEST_SOURCES
       test_error_handling.cpp
-      test_eqdsk.cpp
       test_uniform_grid.cpp
       test_field_evaluation.cpp
       test_field_interpolation.cpp
@@ -401,7 +431,6 @@ if(Catch2_FOUND)
       test_omega_h_lagrange_field.cpp
       test_point_evaluator.cpp)
   endif()
-
   if(PCMS_ENABLE_MESHFIELDS)
     list(APPEND PCMS_UNIT_TEST_SOURCES
       test_load_vector.cpp)
@@ -426,13 +455,6 @@ if(Catch2_FOUND)
     target_link_libraries(unit_tests PRIVATE PETSc::PETSc)
   endif()
 
-  target_link_libraries(unit_tests PUBLIC 
-    Catch2::Catch2 
-    pcms::core
-    pcms_transfer
-    pcms_transfer
-  )
-
  target_include_directories(unit_tests PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
   add_executable(test_interpolation_on_ltx_mesh test_interpolation_on_ltx_mesh.cpp)

test/test_GDI.cpp

@@ -0,0 +1,128 @@
+
+#include <Omega_h_mesh.hpp>
+#include <Omega_h_file.hpp>
+#include "test_support.h"
+#include "pcms/coupler/coupler.hpp"
+#include <pcms/utility/types.h>
+static constexpr bool done = true;
+static constexpr int COMM_ROUNDS = 1;
+
+void xgc_delta_f(MPI_Comm comm)
+{
+  pcms::Coupler coupler("proxy_couple", comm, false, {});
+  pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_delta_f");
+
+  const auto GDI = app->AddData<pcms::GO>("global_comm", comm);
+  auto mean = std::vector<long>(1);
+  mean[0] = 16;
+  do {
+    for (int i = 0; i < COMM_ROUNDS; ++i) {
+      app->BeginSendPhase();
+      GDI->SendData(mean.data(), "mean", mean.size());
+      app->EndSendPhase();
+      printf("delta Sent mean:%d\n", mean[0]);
+      app->BeginReceivePhase();
+      mean = GDI->ReceiveData("mean", mean.size());
+      app->EndReceivePhase();
+      mean[0] = mean[0]/2;
+    }
+  } while (!done);
+  printf("final Mean = %d\n", mean[0]);
+  assert(std::fabs(mean[0] - 1.0) < 1e-12);
+   printf("GDI test successful.\n");
+}
+void xgc_total_f(MPI_Comm comm)
+{
+  pcms::Coupler coupler("proxy_couple", comm, false, {});
+  pcms::Application* app = coupler.AddApplication("proxy_couple_xgc_total_f");
+
+  auto GDI = app->AddData<pcms::GO>("global_comm", comm);
+  auto mean = std::vector<pcms::GO>(1);
+  do {
+    for (int i = 0; i < COMM_ROUNDS; ++i) {
+      app->BeginReceivePhase();
+      mean = GDI->ReceiveData("mean", mean.size());
+      app->EndReceivePhase();
+      printf("total Recieved mean:%d\n", mean[0]);
+      mean[0] = mean[0]/2;
+      app->BeginSendPhase();
+      GDI->SendData(mean.data(), "mean", mean.size());
+      app->EndSendPhase();
+      printf("total Sent mean:%d\n", mean[0]);
+    }
+  } while (!done);
+}
+void xgc_coupler(MPI_Comm comm)
+{
+  // Define Partition
+  redev::LO dim = 3;
+  redev::LOs ranks(1);
+  std::iota(ranks.begin(), ranks.end(), 0);
+  redev::Reals cuts = {0};
+  auto partition = redev::Partition{redev::RCBPtn{dim, ranks, cuts}};
+
+  pcms::Coupler cpl("proxy_couple", comm, true,
+                   partition);
+  auto* total_f = cpl.AddApplication("proxy_couple_xgc_total_f");
+  auto* delta_f = cpl.AddApplication("proxy_couple_xgc_delta_f");
+
+  auto GDI_total = total_f->AddData<pcms::GO>("global_comm", comm);
+  auto GDI_delta = delta_f->AddData<pcms::GO>("global_comm", comm);
+  auto mean = std::vector<pcms::GO>(1);
+  do {
+    for (int i = 0; i < COMM_ROUNDS; ++i) {
+      delta_f->BeginReceivePhase();
+      mean = GDI_delta->ReceiveData("mean", 1);
+      delta_f->EndReceivePhase();
+      printf("delta Received mean:%d\n", mean[0]);
+      mean[0] = mean[0]/2;
+      const auto  msg_size = mean.size();
+      total_f->BeginSendPhase();
+      GDI_total->SendData(mean.data(), "mean", msg_size);
+      total_f->EndSendPhase();
+      printf("total sent mean:%d\n", mean[0]);
+      total_f->BeginReceivePhase();
+      mean = GDI_total->ReceiveData("mean", msg_size);
+      total_f->EndReceivePhase();
+      printf("delta Received mean:%d\n", mean[0]);
+      mean[0] = mean[0]/2;
+      delta_f->BeginSendPhase();
+      GDI_delta->SendData(mean.data(), "mean", msg_size);
+      delta_f->EndSendPhase();
+      printf("detla sent mean:%d\n", mean[0]);
+    }
+  } while (!done);
+}
+
+int main(int argc, char** argv)
+{
+  MPI_Init(&argc, &argv);
+
+  OMEGA_H_CHECK(argc == 2);
+
+  const auto clientId = std::atoi(argv[1]);
+  REDEV_ALWAYS_ASSERT(clientId >= -1 && clientId <= 1);
+
+  MPI_Comm comm = MPI_COMM_WORLD;
+
+  switch (clientId) {
+    case -1:
+      xgc_coupler(comm);
+      break;
+
+    case 0:
+      xgc_delta_f(comm);
+      break;
+
+    case 1:
+      xgc_total_f(comm);
+      break;
+
+    default:
+      std::cerr << "Unhandled client id; expected -1, 0, or 1\n";
+      MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
+  }
+
+  MPI_Finalize();
+  return 0;
+}