Skip to content

Add CUDA-aware MPI halo exchange for MPMesh #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Shahrear2000 wants to merge 5 commits into
base: dn/Velocity_solver_improvement
Choose a base branch
from
Open

Add CUDA-aware MPI halo exchange for MPMesh #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b98b69a
First commit to port
dhyan1272 Jun 29, 2026
12417d3
Conditional for HALO exchange of nProcsTot=1 in polyMPO
dhyan1272 Jun 30, 2026
229c8cb
SetMPArea and modify in strain rate
dhyan1272 Jul 13, 2026
37dc4b8
Add CUDA-aware MPI halo exchange
Jun 29, 2026
fdb365e
Add CUDA-aware communication path to MPMesh
Jul 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,4 +50,4 @@ if(IS_TESTING)
add_subdirectory (test)
endif()

bob_end_package()
bob_end_package()
19 changes: 19 additions & 0 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,25 @@ set(SOURCES
)

add_library(polyMPO-core ${SOURCES})

if(DEFINED ENV{PE_MPICH_GTL_DIR_nvidia80}
AND DEFINED ENV{PE_MPICH_GTL_LIBS_nvidia80})

target_link_options(polyMPO-core PUBLIC
"$ENV{PE_MPICH_GTL_DIR_nvidia80}"
"$ENV{PE_MPICH_GTL_LIBS_nvidia80}"
)

message(STATUS
"polyMPO: enabling Cray MPICH CUDA GTL linkage")
else()
message(FATAL_ERROR
"Cray MPICH CUDA GTL environment variables are unavailable. "
"Load cudatoolkit and craype-accel-nvidia80, or set "
"CRAY_ACCEL_TARGET=nvidia80 before configuring.")
endif()

target_compile_definitions(polyMPO-core PUBLIC CUDA_AWARE_MPI)

@cwsmith cwsmith Jun 30, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Naming this 'GPU_AWARE_MPI' would be more portable (i.e., to AMD and Intel GPUs).

set_property(TARGET polyMPO-core PROPERTY CXX_STANDARD "17")
set_property(TARGET polyMPO-core PROPERTY CXX_STANDARD_REQUIRED ON)
set_property(TARGET polyMPO-core PROPERTY CXX_EXTENSIONS OFF)
Expand Down
13 changes: 10 additions & 3 deletions src/pmpo_MPMesh.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@ void MPMesh::calculateStrain(){
auto MPsBasisGrads = p_MPs->getData<MPF_Basis_Grad_Vals>();
auto MPsAppID = p_MPs->getData<MPF_MP_APP_ID>();
auto MPsStrainRate = p_MPs->getData<MPF_Strain_Rate>();
auto MPsArea = p_MPs->getData<polyMPO::MPF_Area>();
//Mesh Fields
auto tanLatVertexRotatedOverRadius = p_mesh->getMeshField<MeshF_TanLatVertexRotatedOverRadius>();
auto elm2VtxConn = p_mesh->getElm2VtxConn();
auto velField = p_mesh->getMeshField<MeshF_Vel>();
auto solveStress = p_mesh->getMeshField<polyMPO::MeshF_SolveStress>();
auto elasticTimeStep = p_mesh->getElasticTimeStep();

auto setMPStrainRate = PS_LAMBDA(const int& elm, const int& mp, const int& mask){
if(mask){
Expand All @@ -29,7 +31,7 @@ void MPMesh::calculateStrain(){
MPsStrainRate(mp, 2) = 0.0;
return;
}

int numVtx = elm2VtxConn(elm,0);

double v11 = 0.0;
Expand All @@ -52,6 +54,8 @@ void MPMesh::calculateStrain(){
MPsStrainRate(mp, 0) = v11 - vTanOverR;
MPsStrainRate(mp, 1) = v22;
MPsStrainRate(mp, 2) = 0.5*(v12 + v21 + uTanOverR);

MPsArea(mp, 0) = MPsArea(mp, 0) * exp((v11+v22-vTanOverR)*elasticTimeStep);
}
};
p_MPs->parallel_for(setMPStrainRate, "setMPStrainRate");
Expand All @@ -75,13 +79,16 @@ void MPMesh::calculateStress(const int constitutive_relation){
if(mask){
Vec3d strain_rate (MPsStrainRate(mp, 0), MPsStrainRate(mp, 1), MPsStrainRate(mp, 2));
Vec3d stress(MPsStress(mp, 0), MPsStress(mp, 1), MPsStress(mp, 2));
double rep_pressure=MPsRepPressure(mp,0);

if (constitutive_relation == 1)
constitutive_evp(strain_rate, stress, MPsIcePressure(mp,0), MPsRepPressure(mp,0), MPsArea(mp,0), elasticTimeStep, dampingTimescale);
constitutive_evp(strain_rate, stress, MPsIcePressure(mp,0), rep_pressure, MPsArea(mp,0), elasticTimeStep, dampingTimescale);
else if(constitutive_relation == 3)
constitutive_linear(strain_rate, stress);

for (int m=0 ; m<3; m++)
MPsStress(mp, m) = stress[m]*solveStress(elm);
MPsRepPressure(mp,0)=rep_pressure;
}
};
p_MPs->parallel_for(setMPStress, "setMPStress");
Expand Down Expand Up @@ -171,7 +178,7 @@ void MPMesh::calculateStressDivergence(){
timer.reset();
if(numProcsTot>1){
//Takes contribution of halo vertices and adds it in owner procs
communicate_and_take_halo_contributions1(stress_divUV, numVertices, 2, 0, 0);
communicate_and_take_halo_contributions1_improved(stress_divUV, numVertices, 2, 0, 0);
//Transfer the correct values at owned vertices to halo vertices
//communicate_and_take_halo_contributions(stress_divUV, numVertices, 2, 1, 1);
}
Expand Down
Loading