add cholesky MPI test

.buildkite/pipeline.yml

@@ -90,6 +90,22 @@ steps:
       - JuliaCI/julia-coverage#v1:
           codecov: true
 
+  - label: Julia 1.11 (MPI)
+    timeout_in_minutes: 60
+    <<: *test
+    command: |
+      julia --project=test -e 'using Pkg; Pkg.instantiate(); using MPIPreferences; MPIPreferences.use_jll_binary("MPICH_jll"; export_prefs=true)'
+      julia --project -e 'using Pkg; Pkg.develop(;path="lib/TimespanLogging"); Pkg.instantiate(); Pkg.test(; coverage=true, julia_args=["--threads=2"])'
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.11"
+      - JuliaCI/julia-test#v1:
+          julia_args: "--threads=1"
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+    env:
+      CI_USE_MPI: "1"
+
   - label: Julia 1.11 (CUDA)
     timeout_in_minutes: 60
     <<: *gputest

test/Project.toml

@@ -12,6 +12,8 @@ Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
+MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 MemPool = "f9f48841-c794-520a-933b-121f7ba6ed94"
 OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

test/mpitest.jl

@@ -0,0 +1,22 @@
+using MPI, Dagger, Test
+
+Dagger.accelerate!(:mpi)
+
+comm = MPI.COMM_WORLD
+rank = MPI.Comm_rank(comm)
+sz = MPI.Comm_size(comm)
+
+@testset "MPI Dagger P2P" begin
+    if sz == 2
+        tag = 42
+        if rank == 0
+            A = rand(2, 2)
+            Dagger.send_yield(A, comm, 1, tag)
+        elseif rank == 1
+            B = Dagger.recv_yield(comm, 0, tag)
+            @test size(B) == (2, 2)
+        end
+    end
+end
+
+MPI.Finalize()

test/runtests.jl

@@ -5,6 +5,7 @@ USE_ROCM = parse(Bool, get(ENV, "CI_USE_ROCM", "0"))
 USE_ONEAPI = parse(Bool, get(ENV, "CI_USE_ONEAPI", "0"))
 USE_METAL = parse(Bool, get(ENV, "CI_USE_METAL", "0"))
 USE_OPENCL = parse(Bool, get(ENV, "CI_USE_OPENCL", "0"))
+USE_MPI = parse(Bool, get(ENV, "CI_USE_MPI", "0"))
 USE_GPU = USE_CUDA || USE_ROCM || USE_ONEAPI || USE_METAL || USE_OPENCL
 
 tests = [
@@ -46,6 +47,8 @@ tests = [
     ("Reusable Data Structures", "reuse.jl"),
     ("External Languages - Python", "extlang/python.jl"),
     ("Preferences", "preferences.jl"),
+    ("MPI_test", "mpitest.jl"),
+    #("MPI", "mpi.jl")
     #("Fault Tolerance", "fault-tolerance.jl"),
 ]
 if USE_GPU
@@ -55,6 +58,15 @@ if USE_GPU
         ("Array - Stencils", "array/stencil.jl"),
     ]
 end
+
+if USE_MPI
+    #Only run MPI tests
+    tests = [
+        #("MPI", "mpi.jl"),
+        ("MPI_test", "mpitest.jl"),
+    ]
+end
+
 all_test_names = map(test -> replace(last(test), ".jl"=>""), tests)
 
 additional_workers::Int = 3
@@ -65,6 +77,9 @@ if PROGRAM_FILE != "" && realpath(PROGRAM_FILE) == @__FILE__
     using Pkg
     Pkg.activate(@__DIR__)
     try
+        # If I not use Pkg.develop it returns the error "Package Dagger not found in current path.
+        #                                                Run `import Pkg; Pkg.add("Dagger")` to install the Dagger package."
+        Pkg.develop(path=joinpath(@__DIR__, ".."))
         Pkg.instantiate()
     catch
     end
@@ -162,11 +177,25 @@ else
     @info "Running all tests"
 end
 
+if USE_MPI
+    include("setup_mpi.jl")
+    @info "Running MPI tests via mpiexecjl"
+    mpiexecjl_path = joinpath(DEPOT_PATH[1], "bin", "mpiexecjl")
+    cmd = `$mpiexecjl_path -n 2 $(Base.julia_cmd()) --project=$(Base.active_project()) $(joinpath(@__DIR__, "mpitest.jl"))`
+    @info "Executing: $cmd"
+    run(cmd)
+    exit(0)
+end
+
 using Distributed
 if additional_workers > 0
     # We put this inside a branch because addprocs() takes a minimum of 1s to
     # complete even if doing nothing, which is annoying.
     addprocs(additional_workers; exeflags="--project=$(joinpath(@__DIR__, ".."))")
+    @everywhere begin
+        using Pkg
+        Pkg.instantiate()
+    end
 end
 
 include("imports.jl")

test/setup_mpi.jl

@@ -0,0 +1,4 @@
+if USE_MPI
+    using MPI
+    MPI.install_mpiexecjl(; destdir=joinpath(DEPOT_PATH[1], "bin"), force=true)
+end