diff --git a/quest/include/environment.h b/quest/include/environment.h index 04f24bfe..4cca96a4 100644 --- a/quest/include/environment.h +++ b/quest/include/environment.h @@ -36,6 +36,7 @@ typedef struct { int isMultithreaded; int isGpuAccelerated; int isDistributed; + int isMPIGPUAware; // deployment modes which cannot be directly changed after compilation int isCuQuantumEnabled; diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 54149189..7aedcd24 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -147,6 +147,7 @@ void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMulti globalEnvPtr->isDistributed = useDistrib; globalEnvPtr->isCuQuantumEnabled = useCuQuantum; globalEnvPtr->isGpuSharingEnabled = permitGpuSharing; + globalEnvPtr->isMPIGPUAware = comm_set_isMpiGpuAware(); // bind distributed info globalEnvPtr->rank = (useDistrib)? comm_getRank() : 0; diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 854a12bd..3a1a636d 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -12,12 +12,17 @@ * @author Tyson Jones */ +#include "quest/include/environment.h" #include "quest/include/config.h" #include "quest/include/types.h" #include "quest/src/comm/comm_config.hpp" #include "quest/src/core/errors.hpp" +#include +#include +#include + #if COMPILE_MPI #include #endif @@ -60,22 +65,79 @@ bool comm_isMpiCompiled() { return (bool) COMPILE_MPI; } +enum Mpi_version {NONE, OPENMPI, CRAYMPICH}; + +int comm_whichMpi() { + + char version_string[1000]; + int resultlen[] = {0}; + + MPI_Get_library_version(version_string, resultlen); + + enum Mpi_version version = NONE; + + // Check if Openmpi used + #ifdef OPEN_MPI + version = OPENMPI; + #endif + + // Check if Cray MPI used + + const char* cray_string = "CRAY MPICH"; + + std::string v_string = version_string; + + if (v_string.find(cray_string) != string::npos) { + version = CRAYMPICH; + } + + + std::cout << "MPI version: " << version << std::endl; + + return version; + +} + + bool comm_isMpiGpuAware() { + std::cout << "MPI GPU aware check: " << getQuESTEnv().isMPIGPUAware << std::endl; + return getQuESTEnv().isMPIGPUAware; +} + +bool comm_set_isMpiGpuAware() { + /// @todo these checks may be OpenMPI specific, so that /// non-OpenMPI MPI compilers are always dismissed as /// not being CUDA-aware. Check e.g. MPICH method! - // definitely not GPU-aware if compiler declares it is not - #if defined(MPIX_CUDA_AWARE_SUPPORT) && ! MPIX_CUDA_AWARE_SUPPORT - return false; - #endif + std::cout << "Start comm_set_isMpiGpuAware()" << std::endl; - // check CUDA-awareness at run-time if we know it's principally supported - #if defined(MPIX_CUDA_AWARE_SUPPORT) - return (bool) MPIX_Query_cuda_support(); - #endif + int mpi_lib = comm_whichMpi(); + + if(OPENMPI==mpi_lib) { + // definitely not GPU-aware if compiler declares it is not + #if defined(MPIX_CUDA_AWARE_SUPPORT) && ! MPIX_CUDA_AWARE_SUPPORT + return false; + #endif + + // check CUDA-awareness at run-time if we know it's principally supported + #if defined(MPIX_CUDA_AWARE_SUPPORT) + return (bool) MPIX_Query_cuda_support(); + #endif + } + + if(CRAYMPICH==mpi_lib) { + std::cout << "CRAYMPICH comm_set_isMpiGpuAware()" << std::endl; + + const char* var = std::getenv("MPICH_GPU_SUPPORT_ENABLED"); + std::cout << "MPICH_GPU_SUPPORT_ENABLED: " << var << std::endl; + + return (bool) var; + } + + std::cout << "End comm_set_isMpiGpuAware()" << std::endl; // if we can't ascertain CUDA-awareness, just assume no to avoid seg-fault return false; diff --git a/quest/src/comm/comm_config.hpp b/quest/src/comm/comm_config.hpp index 444d1dbf..76f424a8 100644 --- a/quest/src/comm/comm_config.hpp +++ b/quest/src/comm/comm_config.hpp @@ -13,8 +13,11 @@ constexpr int ROOT_RANK = 0; +int comm_whichMpi(); + bool comm_isMpiCompiled(); bool comm_isMpiGpuAware(); +bool comm_set_isMpiGpuAware(); void comm_init(); void comm_end(); @@ -28,4 +31,4 @@ bool comm_isRootNode(); bool comm_isRootNode(int rank); -#endif // COMM_CONFIG_HPP \ No newline at end of file +#endif // COMM_CONFIG_HPP