From 2e5441fbd8b262aa8d8ddb561fa760893592f723 Mon Sep 17 00:00:00 2001 From: JPRichings Date: Wed, 15 Apr 2026 17:13:52 +0100 Subject: [PATCH 1/3] initial commit enabling GPU direct communictation for Cray-MPICH --- quest/src/comm/comm_config.cpp | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 854a12bd..3089bb8e 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -18,6 +18,8 @@ #include "quest/src/comm/comm_config.hpp" #include "quest/src/core/errors.hpp" +#include + #if COMPILE_MPI #include #endif @@ -67,15 +69,29 @@ bool comm_isMpiGpuAware() { /// non-OpenMPI MPI compilers are always dismissed as /// not being CUDA-aware. Check e.g. MPICH method! - // definitely not GPU-aware if compiler declares it is not - #if defined(MPIX_CUDA_AWARE_SUPPORT) && ! MPIX_CUDA_AWARE_SUPPORT - return false; - #endif + + #ifdef OPEN_MPI + // definitely not GPU-aware if compiler declares it is not + #if defined(MPIX_CUDA_AWARE_SUPPORT) && ! MPIX_CUDA_AWARE_SUPPORT + return false; + #endif + + // check CUDA-awareness at run-time if we know it's principally supported + #if defined(MPIX_CUDA_AWARE_SUPPORT) + return (bool) MPIX_Query_cuda_support(); + #endif + + #ifdef CRAY_MPICH // need cononical way to set this variable + // Check MPI awareness for Cray-MPICH + #if defined(MPICH_GPU_SUPPORT_ENABLED) && ! MPICH_GPU_SUPPORT_ENABLED + return false; + #endif + + #if defined(MPICH_GPU_SUPPORT_ENABLED) + const char* var = std::getenv("MPICH_GPU_SUPPORT_ENABLED"); + return (bool) var; + #endif - // check CUDA-awareness at run-time if we know it's principally supported - #if defined(MPIX_CUDA_AWARE_SUPPORT) - return (bool) MPIX_Query_cuda_support(); - #endif // if we can't ascertain CUDA-awareness, just assume no to avoid seg-fault return false; From 09805cdeeaa1f1ce9e97e4c77c7eba6081877200 Mon Sep 17 00:00:00 2001 From: JPRichings Date: Fri, 17 Apr 2026 00:10:31 +0100 Subject: [PATCH 2/3] Update to GPU aware MPI checking logic --- quest/include/environment.h | 1 + quest/src/api/environment.cpp | 1 + quest/src/comm/comm_config.cpp | 49 +++++++++++++++++++++++++++++++--- quest/src/comm/comm_config.hpp | 5 +++- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/quest/include/environment.h b/quest/include/environment.h index 04f24bfe..4cca96a4 100644 --- a/quest/include/environment.h +++ b/quest/include/environment.h @@ -36,6 +36,7 @@ typedef struct { int isMultithreaded; int isGpuAccelerated; int isDistributed; + int isMPIGPUAware; // deployment modes which cannot be directly changed after compilation int isCuQuantumEnabled; diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 54149189..7aedcd24 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -147,6 +147,7 @@ void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMulti globalEnvPtr->isDistributed = useDistrib; globalEnvPtr->isCuQuantumEnabled = useCuQuantum; globalEnvPtr->isGpuSharingEnabled = permitGpuSharing; + globalEnvPtr->isMPIGPUAware = comm_set_isMpiGpuAware(); // bind distributed info globalEnvPtr->rank = (useDistrib)? comm_getRank() : 0; diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 3089bb8e..f4d63892 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -12,6 +12,7 @@ * @author Tyson Jones */ +#include "quest/include/environment.h" #include "quest/include/config.h" #include "quest/include/types.h" @@ -19,6 +20,7 @@ #include "quest/src/core/errors.hpp" #include +#include #if COMPILE_MPI #include @@ -62,15 +64,53 @@ bool comm_isMpiCompiled() { return (bool) COMPILE_MPI; } +enum Mpi_version {NONE, OPENMPI, CRAYMPICH}; + +int comm_whichMpi() { + + char version_string[] = ""; + int resultlen[] = {0}; + + //MPI_Get_library_version(version_string, resultlen); + + enum Mpi_version version = NONE; + + // Check if Openmpi used + #ifdef OPEN_MPI + version = OPENMPI; + #endif + + // Check if Cray MPI used + + const char* cray_string = "CRAY MPICH"; + + std::string v_string = version_string; + + if (v_string.find(cray_string) != string::npos) { + version = CRAYMPICH; + } + + return version; + +} + + bool comm_isMpiGpuAware() { + return getQuESTEnv().isMPIGPUAware; +} + +bool comm_set_isMpiGpuAware() { + /// @todo these checks may be OpenMPI specific, so that /// non-OpenMPI MPI compilers are always dismissed as /// not being CUDA-aware. Check e.g. MPICH method! + + int mpi_lib = comm_whichMpi(); - #ifdef OPEN_MPI + if(OPENMPI==mpi_lib) { // definitely not GPU-aware if compiler declares it is not #if defined(MPIX_CUDA_AWARE_SUPPORT) && ! MPIX_CUDA_AWARE_SUPPORT return false; @@ -79,9 +119,10 @@ bool comm_isMpiGpuAware() { // check CUDA-awareness at run-time if we know it's principally supported #if defined(MPIX_CUDA_AWARE_SUPPORT) return (bool) MPIX_Query_cuda_support(); - #endif + #endif + } - #ifdef CRAY_MPICH // need cononical way to set this variable + if(CRAYMPICH==mpi_lib) { // Check MPI awareness for Cray-MPICH #if defined(MPICH_GPU_SUPPORT_ENABLED) && ! MPICH_GPU_SUPPORT_ENABLED return false; @@ -91,7 +132,7 @@ bool comm_isMpiGpuAware() { const char* var = std::getenv("MPICH_GPU_SUPPORT_ENABLED"); return (bool) var; #endif - + } // if we can't ascertain CUDA-awareness, just assume no to avoid seg-fault return false; diff --git a/quest/src/comm/comm_config.hpp b/quest/src/comm/comm_config.hpp index 444d1dbf..76f424a8 100644 --- a/quest/src/comm/comm_config.hpp +++ b/quest/src/comm/comm_config.hpp @@ -13,8 +13,11 @@ constexpr int ROOT_RANK = 0; +int comm_whichMpi(); + bool comm_isMpiCompiled(); bool comm_isMpiGpuAware(); +bool comm_set_isMpiGpuAware(); void comm_init(); void comm_end(); @@ -28,4 +31,4 @@ bool comm_isRootNode(); bool comm_isRootNode(int rank); -#endif // COMM_CONFIG_HPP \ No newline at end of file +#endif // COMM_CONFIG_HPP From 55fff7c754a812d8da376b045edfa02614a58c68 Mon Sep 17 00:00:00 2001 From: JPRichings Date: Sat, 18 Apr 2026 00:05:29 +0100 Subject: [PATCH 3/3] update gpu aware setting logic --- quest/src/comm/comm_config.cpp | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index f4d63892..3a1a636d 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -21,6 +21,7 @@ #include #include +#include #if COMPILE_MPI #include @@ -68,10 +69,10 @@ enum Mpi_version {NONE, OPENMPI, CRAYMPICH}; int comm_whichMpi() { - char version_string[] = ""; + char version_string[1000]; int resultlen[] = {0}; - //MPI_Get_library_version(version_string, resultlen); + MPI_Get_library_version(version_string, resultlen); enum Mpi_version version = NONE; @@ -90,6 +91,9 @@ int comm_whichMpi() { version = CRAYMPICH; } + + std::cout << "MPI version: " << version << std::endl; + return version; } @@ -98,6 +102,7 @@ int comm_whichMpi() { bool comm_isMpiGpuAware() { + std::cout << "MPI GPU aware check: " << getQuESTEnv().isMPIGPUAware << std::endl; return getQuESTEnv().isMPIGPUAware; } @@ -107,6 +112,7 @@ bool comm_set_isMpiGpuAware() { /// non-OpenMPI MPI compilers are always dismissed as /// not being CUDA-aware. Check e.g. MPICH method! + std::cout << "Start comm_set_isMpiGpuAware()" << std::endl; int mpi_lib = comm_whichMpi(); @@ -123,16 +129,15 @@ bool comm_set_isMpiGpuAware() { } if(CRAYMPICH==mpi_lib) { - // Check MPI awareness for Cray-MPICH - #if defined(MPICH_GPU_SUPPORT_ENABLED) && ! MPICH_GPU_SUPPORT_ENABLED - return false; - #endif - - #if defined(MPICH_GPU_SUPPORT_ENABLED) - const char* var = std::getenv("MPICH_GPU_SUPPORT_ENABLED"); - return (bool) var; - #endif - } + std::cout << "CRAYMPICH comm_set_isMpiGpuAware()" << std::endl; + + const char* var = std::getenv("MPICH_GPU_SUPPORT_ENABLED"); + std::cout << "MPICH_GPU_SUPPORT_ENABLED: " << var << std::endl; + + return (bool) var; + } + + std::cout << "End comm_set_isMpiGpuAware()" << std::endl; // if we can't ascertain CUDA-awareness, just assume no to avoid seg-fault return false;