diff --git a/src/pcms/coupler/coupler.hpp b/src/pcms/coupler/coupler.hpp index eef0f564..168865a5 100644 --- a/src/pcms/coupler/coupler.hpp +++ b/src/pcms/coupler/coupler.hpp @@ -11,12 +11,16 @@ #include "pcms/utility/assert.h" #include "pcms/utility/common.h" #include "pcms/utility/profile.h" +#include "pcms/transfer/transfer_operator.hpp" +#include #include +#include namespace pcms { class Application; +class Coupler; template class FieldHandle @@ -40,7 +44,7 @@ class FieldHandle // FunctionHandle is a FieldHandle that additionally carries the function space, // so it can be evaluated / used to build transfer operators. Only AddFunction -// produces one; CreateTransfer accepts FunctionHandle (not FieldHandle), which +// produces one; AddTransfer accepts FunctionHandle (not FieldHandle), which // makes passing a comm-only field to a transfer a compile error. template class FunctionHandle : public FieldHandle @@ -66,6 +70,62 @@ class FunctionHandle : public FieldHandle std::shared_ptr space_; }; +// Transfer is the non-templated base for a runnable transfer, owned by the +// Coupler. Run() takes no typed arguments (the operator's source/target fields +// are bound in the concrete BoundTransfer), so it erases the value type T and +// lets the coupler hold a heterogeneous set of transfers uniformly. +class Transfer +{ +public: + virtual void Run() const = 0; + virtual ~Transfer() noexcept = default; +}; + +// BoundTransfer binds a built transfer operator to the source/target functions +// it connects. Owned by the Coupler (as a Transfer) and referenced through a +// TransferHandle; the expensive build happened in AddTransfer, Run() is the +// cheap per-step Apply. +template +class BoundTransfer : public Transfer +{ +public: + BoundTransfer(FunctionHandle source, FunctionHandle target, + std::unique_ptr> op) + : source_(std::move(source)), + target_(std::move(target)), + operator_(std::move(op)) + { + } + + void Run() const override + { + PCMS_FUNCTION_TIMER; + operator_->Apply(source_.GetField(), target_.GetField()); + } + +private: + FunctionHandle source_; + FunctionHandle target_; + std::unique_ptr> operator_; +}; + +// Lightweight, copyable reference to a coupler-owned transfer (mirrors +// FieldHandle). Run() dispatches to the owning Coupler. +class TransferHandle +{ +public: + void Run() const; + +private: + TransferHandle(Coupler* coupler, std::size_t id) : coupler_(coupler), id_(id) + { + } + friend class Coupler; + + Coupler* coupler_; + std::size_t id_; +}; + class Application { public: @@ -249,12 +309,22 @@ class Coupler return redev_.GetPartition(); } + // Build and register a transfer from source to target using the given method + // recipe (see pcms/transfer/transfer_method.hpp). The coupler owns the built + // operator; the returned handle's Run() applies it in the coupling loop. + template + TransferHandle AddTransfer(FunctionHandle source, FunctionHandle target, + const Method& method); + + void RunTransfer(std::size_t id) { transfers_.at(id)->Run(); } + private: std::string name_; MPI_Comm mpi_comm_; redev::Redev redev_; // gather and scatter operations have reference to internal fields std::map applications_; + std::vector> transfers_; }; } // namespace pcms @@ -385,4 +455,24 @@ pcms::FunctionHandle pcms::Application::AddFunction( return FunctionHandle{this, std::move(name), std::move(space)}; } +inline void pcms::TransferHandle::Run() const +{ + PCMS_ALWAYS_ASSERT(coupler_ != nullptr); + coupler_->RunTransfer(id_); +} + +template +pcms::TransferHandle pcms::Coupler::AddTransfer(FunctionHandle source, + FunctionHandle target, + const Method& method) +{ + PCMS_FUNCTION_TIMER; + std::unique_ptr> op = + method.Build(source.GetSpace(), target.GetSpace()); + const std::size_t id = transfers_.size(); + transfers_.push_back(std::make_unique>( + std::move(source), std::move(target), std::move(op))); + return TransferHandle{this, id}; +} + #endif // COUPLER2_H_ diff --git a/src/pcms/transfer/transfer_method.hpp b/src/pcms/transfer/transfer_method.hpp new file mode 100644 index 00000000..c63706b7 --- /dev/null +++ b/src/pcms/transfer/transfer_method.hpp @@ -0,0 +1,97 @@ +#ifndef PCMS_TRANSFER_METHOD_H +#define PCMS_TRANSFER_METHOD_H + +#include "pcms/transfer/transfer_operator.hpp" +#include "pcms/transfer/copy.h" +#include "pcms/transfer/interpolator.h" +#include "pcms/transfer/omega_h_conservative_projection.hpp" +#include "pcms/transfer/omega_h_control_variate_projection.hpp" +#include "pcms/transfer/monte_carlo_sampling.hpp" +#include "pcms/field/out_of_bounds_policy.h" +#include "pcms/utility/types.h" +#include +#include + +namespace pcms +{ + +class FunctionSpace; + +// A "transfer method" is a parameter-carrying recipe: it owns the +// method-specific parameters, and its +// +// std::unique_ptr> Build(const FunctionSpace& source, +// const FunctionSpace& target) +// const +// +// member turns a space pair into a transfer operator. Selecting or comparing +// methods is then a one-line change at the AddTransfer call site, e.g. +// AddTransfer(src, tgt, +// method::ConservativeMonteCarlo{.samples_per_element = 64}); +// +// Methods are aggregates (no polymorphic base) so their parameters can be set +// with designated initializers; they are matched by AddTransfer as a +// duck-typed concept rather than a base class. The value type T is enforced +// through Build's return type: a Real-only method will not compile against +// FunctionHandle, and templated methods work for any supported T. +// +// The recipes live in `pcms::method` so their short names (e.g. `Copy`) do not +// collide with the transfer-operator types they wrap. +namespace method +{ + +// Identity copy; source and target must share a layout. Any supported T. +template +struct Copy +{ + std::unique_ptr> Build(const FunctionSpace& source, + const FunctionSpace& target) const + { + return std::make_unique>(source, target); + } +}; + +// Point interpolation: evaluate the source field at the target DOF sites. Any +// supported T. +template +struct Interpolation +{ + OutOfBoundsPolicy policy{}; + + std::unique_ptr> Build(const FunctionSpace& source, + const FunctionSpace& target) const + { + return std::make_unique>(source, target, policy); + } +}; + +// Conservative Galerkin (L2) projection via mesh intersection. Real only. +struct ConservativeIntersection +{ + std::unique_ptr> Build( + const FunctionSpace& source, const FunctionSpace& target) const + { + return std::make_unique(source, target); + } +}; + +// Conservative projection with a Monte-Carlo / control-variate RHS. Real only. +struct ConservativeMonteCarlo +{ + int samples_per_element = 32; + MonteCarloSampling sampling = MonteCarloSampling::UniformRandom; + std::uint64_t seed = 8675309; + + std::unique_ptr> Build( + const FunctionSpace& source, const FunctionSpace& target) const + { + return std::make_unique( + source, target, samples_per_element, sampling, seed); + } +}; + +} // namespace method + +} // namespace pcms + +#endif // PCMS_TRANSFER_METHOD_H diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5efdc81e..0d1ae784 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -214,6 +214,47 @@ if(PCMS_ENABLE_OMEGA_H) ${d3d1p} ignored) endif() + if(PCMS_ENABLE_PETSC AND PCMS_ENABLE_MESHFIELDS) + add_executable(coupler_transfer test_coupler_transfer.cpp) + target_link_libraries(coupler_transfer PUBLIC pcms::core test_support) + if(HOST_NPROC GREATER_EQUAL 4) + tri_mpi_test( + TESTNAME + test_coupler_transfer_4p + TIMEOUT + 60 + NAME1 + rdv + EXE1 + ./coupler_transfer + PROCS1 + 2 + ARGS1 + -1 + ${d3d1p} + ${d3d2p_cpn} + NAME2 + source + EXE2 + ./coupler_transfer + PROCS2 + 1 + ARGS2 + 0 + ${d3d1p} + ignored + NAME3 + target + EXE3 + ./coupler_transfer + PROCS3 + 1 + ARGS3 + 1 + ${d3d1p} + ignored) + endif() + endif() if(HOST_NPROC GREATER_EQUAL 26) tri_mpi_test( TESTNAME diff --git a/test/test_coupler_transfer.cpp b/test/test_coupler_transfer.cpp new file mode 100644 index 00000000..50fb5291 --- /dev/null +++ b/test/test_coupler_transfer.cpp @@ -0,0 +1,185 @@ +#include +#include +#include +#include +#include +#include +#include +#include "test_support.h" +#include "pcms/coupler/coupler.hpp" +#include "pcms/transfer/transfer_method.hpp" +#include "pcms/field/function_space/lagrange.h" + +using pcms::Real; +namespace ts = test_support; + +static constexpr bool done = true; +static constexpr int COMM_ROUNDS = 4; + +namespace +{ + +std::shared_ptr MakeSpace(Omega_h::Mesh& mesh) +{ + // Both ends name the layout "field" so their GID exchanges connect. + return pcms::LagrangeFunctionSpace::FromMesh( + mesh, 1, 1, pcms::CoordinateSystem::Cartesian, "global", + pcms::LagrangeFunctionSpace::DefaultBackend, "field"); +} + +void SetFieldToGids(const pcms::FieldLayout& layout, + pcms::FieldData* field) +{ + auto gids = layout.GetGidsHost(); + const auto n = layout.GetNumOwnedDofHolder(); + Omega_h::HostWrite values(n); + Kokkos::parallel_for( + "set_gids", + Kokkos::RangePolicy(0, n), + [=](int i) { values[i] = gids[i]; }); + field->SetDOFHolderDataHost( + pcms::Rank2View(values.data(), n, 1)); +} + +bool FieldEqualsGids(const pcms::FieldLayout& layout, + pcms::FieldData* field, int rank) +{ + auto gids = layout.GetGidsHost(); + auto owned = layout.GetOwnedHost(); + auto got = pcms::FlattenToRank1View(field->GetDOFHolderDataHost()); + const auto n = layout.GetNumOwnedDofHolder(); + + int expected = 0, matched = 0; + Kokkos::parallel_reduce( + "count_owned", + Kokkos::RangePolicy(0, + owned.size()), + KOKKOS_LAMBDA(int i, int& s) { s += owned[i] != 0; }, expected); + Kokkos::parallel_reduce( + "count_match", + Kokkos::RangePolicy(0, n), + KOKKOS_LAMBDA(int i, int& s) { + // Identity interpolation of an arbitrary per-vertex GID field: in exact + // arithmetic each target vertex recovers its source value, but the + // point-in-element search leaves a tiny (~1e-7) barycentric perturbation + // that, scaled by the large unordered GID differences, can nudge a value + // just below its integer. Round rather than truncate to compare. + if (owned[i]) + s += std::llround(got[i]) == (long long)gids[i]; + }, + matched); + + const bool ok = matched == expected; + std::cerr << "Rank " << rank << " - target field validation " + << (ok ? "PASSED" : "FAILED") << " (" << matched << "/" << expected + << ")\n"; + return ok; +} + +} // namespace + +// -------------------------------------------------------------------------- +// The coupling server: receives a field from the source app, transfers it onto +// the target app's space, and sends it on. This is the whole coupler-integrated +// transfer story. +// -------------------------------------------------------------------------- +void transfer_server(MPI_Comm comm, Omega_h::Mesh& mesh, + std::string_view cpn_file) +{ + pcms::Coupler cpl("coupler_transfer", comm, /*isServer=*/true, + redev::Partition{ts::setupServerPartition(mesh, cpn_file)}); + auto* source_app = cpl.AddApplication("coupler_transfer_source"); + auto* target_app = cpl.AddApplication("coupler_transfer_target"); + + auto source_space = MakeSpace(mesh); + auto target_space = MakeSpace(mesh); + + auto source = + source_app->AddFunction(source_space->CreateFunction("field")); + auto target = + target_app->AddFunction(target_space->CreateFunction("field")); + + auto transfer = + cpl.AddTransfer(source, target, pcms::method::Interpolation{}); + + do { + for (int i = 0; i < COMM_ROUNDS; ++i) { + source_app->ReceivePhase([&] { source.Receive(); }); + transfer.Run(); // source field -> target field + target_app->SendPhase([&] { target.Send(); }); + } + } while (!done); +} + +void transfer_source_client(MPI_Comm comm, Omega_h::Mesh& mesh) +{ + pcms::Coupler coupler("coupler_transfer", comm, false, {}); + auto* app = coupler.AddApplication("coupler_transfer_source"); + auto factory = MakeSpace(mesh); + + auto field = factory->CreateFunction("field"); + auto* field_ptr = &field.GetData(); + SetFieldToGids(*factory->GetLayout(), field_ptr); + app->AddField(std::move(field)); + + do { + for (int i = 0; i < COMM_ROUNDS; ++i) { + app->SendPhase([&] { app->SendField("field"); }); + } + } while (!done); +} + +void transfer_target_client(MPI_Comm comm, Omega_h::Mesh& mesh) +{ + int rank; + MPI_Comm_rank(comm, &rank); + pcms::Coupler coupler("coupler_transfer", comm, false, {}); + auto* app = coupler.AddApplication("coupler_transfer_target"); + auto factory = MakeSpace(mesh); + + auto field = factory->CreateFunction("field"); + auto* field_ptr = &field.GetData(); + app->AddField(std::move(field)); + + do { + for (int i = 0; i < COMM_ROUNDS; ++i) { + app->ReceivePhase([&] { app->ReceiveField("field"); }); + // Identity interpolation on the same mesh: the target must receive the + // exact field the source sent. + if (!FieldEqualsGids(*factory->GetLayout(), field_ptr, rank)) { + exit(EXIT_FAILURE); + } + } + } while (!done); +} + +int main(int argc, char** argv) +{ + try { + auto lib = Omega_h::Library(&argc, &argv); + const int rank = lib.world()->rank(); + if (argc != 4) { + if (!rank) { + std::cerr + << "Usage: " << argv[0] + << " /path/to/mesh /path/to/partition.cpn\n"; + } + exit(EXIT_FAILURE); + } + const auto clientId = atoi(argv[1]); + REDEV_ALWAYS_ASSERT(clientId >= -1 && clientId <= 1); + Omega_h::Mesh mesh(&lib); + Omega_h::binary::read(argv[2], lib.world(), &mesh); + MPI_Comm mpi_comm = lib.world()->get_impl(); + switch (clientId) { + case -1: transfer_server(mpi_comm, mesh, argv[3]); break; + case 0: transfer_source_client(mpi_comm, mesh); break; + case 1: transfer_target_client(mpi_comm, mesh); break; + default: break; // unreachable: clientId asserted to [-1, 1] above + } + return 0; + } catch (const std::exception& e) { + std::cerr << "Exception caught in main: " << e.what() << std::endl; + return 1; + } +} diff --git a/test/test_mesh_intersection_field_transfer.cpp b/test/test_mesh_intersection_field_transfer.cpp index fbb34ddb..4afa0f43 100644 --- a/test/test_mesh_intersection_field_transfer.cpp +++ b/test/test_mesh_intersection_field_transfer.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include "field_test_utils.h" @@ -424,3 +425,63 @@ TEST_CASE("OmegaHConservativeProjection P1 source to P0 target", Catch::Approx(IntegrateP1Field(source_mesh, source)).margin(1e-9)); } } + +// Exercises the pcms::method::* recipe factory: each recipe's Build() must +// produce a working transfer operator equivalent to constructing it directly, +// and parameter-carrying recipes must forward their parameters. +TEST_CASE("TransferMethod recipes build working operators", + "[transfer][method]") +{ + Omega_h::Library lib; + Omega_h::Mesh source_mesh = + BuildUnitSquare(lib, Omega_h::LOs({0, 1, 2, 0, 2, 3})); + Omega_h::Mesh target_mesh = + BuildUnitSquare(lib, Omega_h::LOs({0, 1, 3, 1, 2, 3})); + + auto source_space = MakeP1Space(source_mesh); + auto target_space = MakeP1Space(target_mesh); + + auto source = source_space->CreateFunction(); + auto target = target_space->CreateFunction(); + pcms::test::SetField( + source, OMEGA_H_LAMBDA(pcms::Real x, pcms::Real y) { return x + y; }); + + const auto tgt_coords_h = + Omega_h::HostRead(target_mesh.coords()); + auto check_reproduces_linear = [&]() { + const auto values = pcms::FlattenToRank1View(target.GetDOFHolderDataHost()); + for (Omega_h::LO i = 0; i < target_mesh.nverts(); ++i) { + const double expected = tgt_coords_h[2 * i + 0] + tgt_coords_h[2 * i + 1]; + REQUIRE(values[i] == Catch::Approx(expected).margin(1e-9)); + } + }; + + SECTION("ConservativeIntersection reproduces a linear field") + { + auto op = pcms::method::ConservativeIntersection{}.Build(*source_space, + *target_space); + op->Apply(source, target); + check_reproduces_linear(); + } + + SECTION("Interpolation reproduces a linear field") + { + auto op = pcms::method::Interpolation{}.Build(*source_space, + *target_space); + op->Apply(source, target); + check_reproduces_linear(); + } + + SECTION("ConservativeMonteCarlo forwards params and conserves the integral") + { + // A linear field lives in the target P1 space, so the control-variate + // residual is zero and the projection is exact for any sample count. + auto op = + pcms::method::ConservativeMonteCarlo{.samples_per_element = 16}.Build( + *source_space, *target_space); + op->Apply(source, target); + check_reproduces_linear(); + REQUIRE(IntegrateP1Field(target_mesh, target) == + Catch::Approx(IntegrateP1Field(source_mesh, source)).margin(1e-9)); + } +}