Loading src/KOKKOS/kokkos.cpp +56 −28 Original line number Diff line number Diff line Loading @@ -30,32 +30,38 @@ // for detecting CUDA-aware MPI support: // the variable int have_cuda_aware // - is -1 if CUDA-aware MPI support is unknown // - is 0 if no CUDA-aware MPI support available // - is 1 if CUDA-aware MPI support is available // - is 0 if CUDA-aware MPI support is unavailable // - is -1 if CUDA-aware MPI support is unknown #define CUDA_AWARE_UNKNOWN static int have_cuda_aware = -1; // OpenMPI supports detecting CUDA-aware MPI as of version 2.0.0 #if OPEN_MPI // IBM Spectrum MPI looks like OpenMPI but defines MPIX_CUDA_AWARE_SUPPORT=0 // even when CUDA-aware MPI is available, and also has a runtime option // to turn CUDA-aware MPI on/off, so support is unknown #if (defined OPEN_MPI) && !(defined SPECTRUM_MPI) #if (OMPI_MAJOR_VERSION >= 2) #include <mpi-ext.h> #if defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT static int have_cuda_aware = 1; #elif defined(MPIX_CUDA_AWARE_SUPPORT) && !MPIX_CUDA_AWARE_SUPPORT static int have_cuda_aware = 0; #else CUDA_AWARE_UNKNOWN #endif #endif // defined(MPIX_CUDA_AWARE_SUPPORT) #else // old OpenMPI CUDA_AWARE_UNKNOWN #endif #endif // (OMPI_MAJOR_VERSION >=2) #else // unknown MPI library CUDA_AWARE_UNKNOWN #endif #endif // OPEN_MPI #endif // KOKKOS_ENABLE_CUDA Loading Loading @@ -147,29 +153,10 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) #ifdef KOKKOS_ENABLE_CUDA if (ngpus <= 0) error->all(FLERR,"Kokkos has been compiled for CUDA but no GPUs are requested"); // check and warn about CUDA-aware MPI availability when using multiple MPI tasks int nmpi = 0; MPI_Comm_size(world,&nmpi); if ((nmpi > 1) && (me == 0)) { if ( 1 == have_cuda_aware ) { ; // all good, nothing to warn about } else if ( -1 == have_cuda_aware ) { error->warning(FLERR,"Kokkos with CUDA assumes CUDA-aware MPI is available," " but cannot determine if this is the case\n try" " '-pk kokkos cuda/aware off' when getting segmentation faults"); } else if ( 0 == have_cuda_aware ) { error->warning(FLERR,"CUDA-aware MPI is NOT available, " "using '-pk kokkos cuda/aware off' by default"); } else { ; // should never get here } } #endif #ifndef KOKKOS_ENABLE_SERIAL if (nthreads == 1) if (nthreads == 1 && me == 0) error->warning(FLERR,"When using a single thread, the Kokkos Serial backend " "(i.e. Makefile.kokkos_mpi_only) gives better performance " "than the OpenMP backend"); Loading @@ -185,7 +172,11 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) // default settings for package kokkos command binsize = 0.0; #if KOKKOS_USE_CUDA cuda_aware_flag = 1; #else cuda_aware_flag = 0; #endif neigh_thread = 0; neigh_thread_set = 0; neighflag_qeq_set = 0; Loading @@ -209,9 +200,46 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) } #ifdef KOKKOS_ENABLE_CUDA // check and warn about CUDA-aware MPI availability when using multiple MPI tasks // change default only if we can safely detect that CUDA-aware MPI is not available if (0 == have_cuda_aware) cuda_aware_flag = 0; int nmpi = 0; MPI_Comm_size(world,&nmpi); if (nmpi > 0) { if (have_cuda_aware == 0) { if (me == 0) error->warning(FLERR,"CUDA-aware MPI is NOT available, " "using '-pk kokkos cuda/aware off'"); cuda_aware_flag = 0; } else if (have_cuda_aware == -1) { // maybe we are dealing with MPICH, MVAPICH2 or some derivative? // MVAPICH2 #if (defined MPICH) && (defined MVAPICH2_VERSION) char* str; if (!(str = getenv("MV2_ENABLE_CUDA") && (!(strcmp(str,"1") == 0))) { if (me == 0) error->warning(FLERR,"MVAPICH2 'MV2_ENABLE_CUDA' environment variable is not set. Disabling CUDA-aware MPI"); cuda_aware_flag = 0; } // pure MPICH or some unsupported MPICH derivative #elif (defined MPICH) && !(defined MVAPICH2_VERSION) if (me == 0) error->warning(FLERR,"Detected MPICH. Disabling CUDA-aware MPI"); cuda_aware_flag = 0; #elif (defined SPECTRUM_MPI) if (me == 0) error->warning(FLERR,"Must use the '-gpu' flag with Spectrum MPI to enable " "CUDA-aware MPI support"); #else if (me == 0) error->warning(FLERR,"Kokkos with CUDA assumes CUDA-aware MPI is available," " but cannot determine if this is the case\n try" " '-pk kokkos cuda/aware off' if getting segmentation faults"); #endif } // if (-1 == have_cuda_aware) } // nmpi > 0 #endif // KOKKOS_USE_CUDA #ifdef KILL_KOKKOS_ON_SIGSEGV signal(SIGSEGV, my_signal_handler); Loading src/MAKE/MACHINES/Makefile.lassen 0 → 100644 +125 −0 Original line number Diff line number Diff line # lassen_kokkos = KOKKOS/CUDA, V100 GPU and Power9, IBM Spectrum MPI, nvcc compiler with gcc 7.3.1 SHELL = /bin/sh # --------------------------------------------------------------------- # compiler/linker settings # specify flags and libraries needed for your compiler KOKKOS_ABSOLUTE_PATH = $(shell cd $(KOKKOS_PATH); pwd) export MPICH_CXX = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper export OMPI_CXX = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper CC = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper CCFLAGS = -g -O3 SHFLAGS = -fPIC DEPFLAGS = -M LINK = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper LINKFLAGS = -g -O3 LIB = SIZE = size ARCHIVE = ar ARFLAGS = -rc SHLIBFLAGS = -shared KOKKOS_DEVICES = Cuda KOKKOS_ARCH = Power9,Volta70 # --------------------------------------------------------------------- # LAMMPS-specific settings, all OPTIONAL # specify settings for LAMMPS features you will use # if you change any -D setting, do full re-compile after "make clean" # LAMMPS ifdef settings # see possible settings in Section 2.2 (step 4) of manual LMP_INC = -DLAMMPS_GZIP # MPI library # see discussion in Section 2.2 (step 5) of manual # MPI wrapper compiler/linker can provide this info # can point to dummy MPI library in src/STUBS as in Makefile.serial # use -D MPICH and OMPI settings in INC to avoid C++ lib conflicts # INC = path for mpi.h, MPI compiler settings # PATH = path for MPI library # LIB = name of MPI library MPI_INC = -DSPECTRUM_MPI -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1 -I/usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-7.3.1/include MPI_PATH = MPI_LIB = -L/usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-7.3.1/lib -lmpi_ibm # FFT library # see discussion in Section 2.2 (step 6) of manaul # can be left blank to use provided KISS FFT library # INC = -DFFT setting, e.g. -DFFT_FFTW, FFT compiler settings # PATH = path for FFT library # LIB = name of FFT library FFT_INC = FFT_PATH = FFT_LIB = # JPEG and/or PNG library # see discussion in Section 2.2 (step 7) of manual # only needed if -DLAMMPS_JPEG or -DLAMMPS_PNG listed with LMP_INC # INC = path(s) for jpeglib.h and/or png.h # PATH = path(s) for JPEG library and/or PNG library # LIB = name(s) of JPEG library and/or PNG library JPG_INC = JPG_PATH = JPG_LIB = # --------------------------------------------------------------------- # build rules and dependencies # do not edit this section include Makefile.package.settings include Makefile.package EXTRA_INC = $(LMP_INC) $(PKG_INC) $(MPI_INC) $(FFT_INC) $(JPG_INC) $(PKG_SYSINC) EXTRA_PATH = $(PKG_PATH) $(MPI_PATH) $(FFT_PATH) $(JPG_PATH) $(PKG_SYSPATH) EXTRA_LIB = $(PKG_LIB) $(MPI_LIB) $(FFT_LIB) $(JPG_LIB) $(PKG_SYSLIB) EXTRA_CPP_DEPENDS = $(PKG_CPP_DEPENDS) EXTRA_LINK_DEPENDS = $(PKG_LINK_DEPENDS) # Path to src files vpath %.cpp .. vpath %.h .. # Link target $(EXE): $(OBJ) $(EXTRA_LINK_DEPENDS) $(LINK) $(LINKFLAGS) $(EXTRA_PATH) $(OBJ) $(EXTRA_LIB) $(LIB) -o $(EXE) $(SIZE) $(EXE) # Library targets lib: $(OBJ) $(EXTRA_LINK_DEPENDS) $(ARCHIVE) $(ARFLAGS) $(EXE) $(OBJ) shlib: $(OBJ) $(EXTRA_LINK_DEPENDS) $(CC) $(CCFLAGS) $(SHFLAGS) $(SHLIBFLAGS) $(EXTRA_PATH) -o $(EXE) \ $(OBJ) $(EXTRA_LIB) $(LIB) # Compilation rules %.o:%.cpp $(EXTRA_CPP_DEPENDS) $(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $< %.d:%.cpp $(EXTRA_CPP_DEPENDS) $(CC) $(CCFLAGS) $(EXTRA_INC) $(DEPFLAGS) $< > $@ %.o:%.cu $(EXTRA_CPP_DEPENDS) $(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $< # Individual dependencies depend : fastdep.exe $(SRC) @./fastdep.exe $(EXTRA_INC) -- $^ > .depend || exit 1 fastdep.exe: ../DEPEND/fastdep.c cc -O -o $@ $< sinclude .depend Loading
src/KOKKOS/kokkos.cpp +56 −28 Original line number Diff line number Diff line Loading @@ -30,32 +30,38 @@ // for detecting CUDA-aware MPI support: // the variable int have_cuda_aware // - is -1 if CUDA-aware MPI support is unknown // - is 0 if no CUDA-aware MPI support available // - is 1 if CUDA-aware MPI support is available // - is 0 if CUDA-aware MPI support is unavailable // - is -1 if CUDA-aware MPI support is unknown #define CUDA_AWARE_UNKNOWN static int have_cuda_aware = -1; // OpenMPI supports detecting CUDA-aware MPI as of version 2.0.0 #if OPEN_MPI // IBM Spectrum MPI looks like OpenMPI but defines MPIX_CUDA_AWARE_SUPPORT=0 // even when CUDA-aware MPI is available, and also has a runtime option // to turn CUDA-aware MPI on/off, so support is unknown #if (defined OPEN_MPI) && !(defined SPECTRUM_MPI) #if (OMPI_MAJOR_VERSION >= 2) #include <mpi-ext.h> #if defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT static int have_cuda_aware = 1; #elif defined(MPIX_CUDA_AWARE_SUPPORT) && !MPIX_CUDA_AWARE_SUPPORT static int have_cuda_aware = 0; #else CUDA_AWARE_UNKNOWN #endif #endif // defined(MPIX_CUDA_AWARE_SUPPORT) #else // old OpenMPI CUDA_AWARE_UNKNOWN #endif #endif // (OMPI_MAJOR_VERSION >=2) #else // unknown MPI library CUDA_AWARE_UNKNOWN #endif #endif // OPEN_MPI #endif // KOKKOS_ENABLE_CUDA Loading Loading @@ -147,29 +153,10 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) #ifdef KOKKOS_ENABLE_CUDA if (ngpus <= 0) error->all(FLERR,"Kokkos has been compiled for CUDA but no GPUs are requested"); // check and warn about CUDA-aware MPI availability when using multiple MPI tasks int nmpi = 0; MPI_Comm_size(world,&nmpi); if ((nmpi > 1) && (me == 0)) { if ( 1 == have_cuda_aware ) { ; // all good, nothing to warn about } else if ( -1 == have_cuda_aware ) { error->warning(FLERR,"Kokkos with CUDA assumes CUDA-aware MPI is available," " but cannot determine if this is the case\n try" " '-pk kokkos cuda/aware off' when getting segmentation faults"); } else if ( 0 == have_cuda_aware ) { error->warning(FLERR,"CUDA-aware MPI is NOT available, " "using '-pk kokkos cuda/aware off' by default"); } else { ; // should never get here } } #endif #ifndef KOKKOS_ENABLE_SERIAL if (nthreads == 1) if (nthreads == 1 && me == 0) error->warning(FLERR,"When using a single thread, the Kokkos Serial backend " "(i.e. Makefile.kokkos_mpi_only) gives better performance " "than the OpenMP backend"); Loading @@ -185,7 +172,11 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) // default settings for package kokkos command binsize = 0.0; #if KOKKOS_USE_CUDA cuda_aware_flag = 1; #else cuda_aware_flag = 0; #endif neigh_thread = 0; neigh_thread_set = 0; neighflag_qeq_set = 0; Loading @@ -209,9 +200,46 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) } #ifdef KOKKOS_ENABLE_CUDA // check and warn about CUDA-aware MPI availability when using multiple MPI tasks // change default only if we can safely detect that CUDA-aware MPI is not available if (0 == have_cuda_aware) cuda_aware_flag = 0; int nmpi = 0; MPI_Comm_size(world,&nmpi); if (nmpi > 0) { if (have_cuda_aware == 0) { if (me == 0) error->warning(FLERR,"CUDA-aware MPI is NOT available, " "using '-pk kokkos cuda/aware off'"); cuda_aware_flag = 0; } else if (have_cuda_aware == -1) { // maybe we are dealing with MPICH, MVAPICH2 or some derivative? // MVAPICH2 #if (defined MPICH) && (defined MVAPICH2_VERSION) char* str; if (!(str = getenv("MV2_ENABLE_CUDA") && (!(strcmp(str,"1") == 0))) { if (me == 0) error->warning(FLERR,"MVAPICH2 'MV2_ENABLE_CUDA' environment variable is not set. Disabling CUDA-aware MPI"); cuda_aware_flag = 0; } // pure MPICH or some unsupported MPICH derivative #elif (defined MPICH) && !(defined MVAPICH2_VERSION) if (me == 0) error->warning(FLERR,"Detected MPICH. Disabling CUDA-aware MPI"); cuda_aware_flag = 0; #elif (defined SPECTRUM_MPI) if (me == 0) error->warning(FLERR,"Must use the '-gpu' flag with Spectrum MPI to enable " "CUDA-aware MPI support"); #else if (me == 0) error->warning(FLERR,"Kokkos with CUDA assumes CUDA-aware MPI is available," " but cannot determine if this is the case\n try" " '-pk kokkos cuda/aware off' if getting segmentation faults"); #endif } // if (-1 == have_cuda_aware) } // nmpi > 0 #endif // KOKKOS_USE_CUDA #ifdef KILL_KOKKOS_ON_SIGSEGV signal(SIGSEGV, my_signal_handler); Loading
src/MAKE/MACHINES/Makefile.lassen 0 → 100644 +125 −0 Original line number Diff line number Diff line # lassen_kokkos = KOKKOS/CUDA, V100 GPU and Power9, IBM Spectrum MPI, nvcc compiler with gcc 7.3.1 SHELL = /bin/sh # --------------------------------------------------------------------- # compiler/linker settings # specify flags and libraries needed for your compiler KOKKOS_ABSOLUTE_PATH = $(shell cd $(KOKKOS_PATH); pwd) export MPICH_CXX = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper export OMPI_CXX = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper CC = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper CCFLAGS = -g -O3 SHFLAGS = -fPIC DEPFLAGS = -M LINK = $(KOKKOS_ABSOLUTE_PATH)/bin/nvcc_wrapper LINKFLAGS = -g -O3 LIB = SIZE = size ARCHIVE = ar ARFLAGS = -rc SHLIBFLAGS = -shared KOKKOS_DEVICES = Cuda KOKKOS_ARCH = Power9,Volta70 # --------------------------------------------------------------------- # LAMMPS-specific settings, all OPTIONAL # specify settings for LAMMPS features you will use # if you change any -D setting, do full re-compile after "make clean" # LAMMPS ifdef settings # see possible settings in Section 2.2 (step 4) of manual LMP_INC = -DLAMMPS_GZIP # MPI library # see discussion in Section 2.2 (step 5) of manual # MPI wrapper compiler/linker can provide this info # can point to dummy MPI library in src/STUBS as in Makefile.serial # use -D MPICH and OMPI settings in INC to avoid C++ lib conflicts # INC = path for mpi.h, MPI compiler settings # PATH = path for MPI library # LIB = name of MPI library MPI_INC = -DSPECTRUM_MPI -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1 -I/usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-7.3.1/include MPI_PATH = MPI_LIB = -L/usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-7.3.1/lib -lmpi_ibm # FFT library # see discussion in Section 2.2 (step 6) of manaul # can be left blank to use provided KISS FFT library # INC = -DFFT setting, e.g. -DFFT_FFTW, FFT compiler settings # PATH = path for FFT library # LIB = name of FFT library FFT_INC = FFT_PATH = FFT_LIB = # JPEG and/or PNG library # see discussion in Section 2.2 (step 7) of manual # only needed if -DLAMMPS_JPEG or -DLAMMPS_PNG listed with LMP_INC # INC = path(s) for jpeglib.h and/or png.h # PATH = path(s) for JPEG library and/or PNG library # LIB = name(s) of JPEG library and/or PNG library JPG_INC = JPG_PATH = JPG_LIB = # --------------------------------------------------------------------- # build rules and dependencies # do not edit this section include Makefile.package.settings include Makefile.package EXTRA_INC = $(LMP_INC) $(PKG_INC) $(MPI_INC) $(FFT_INC) $(JPG_INC) $(PKG_SYSINC) EXTRA_PATH = $(PKG_PATH) $(MPI_PATH) $(FFT_PATH) $(JPG_PATH) $(PKG_SYSPATH) EXTRA_LIB = $(PKG_LIB) $(MPI_LIB) $(FFT_LIB) $(JPG_LIB) $(PKG_SYSLIB) EXTRA_CPP_DEPENDS = $(PKG_CPP_DEPENDS) EXTRA_LINK_DEPENDS = $(PKG_LINK_DEPENDS) # Path to src files vpath %.cpp .. vpath %.h .. # Link target $(EXE): $(OBJ) $(EXTRA_LINK_DEPENDS) $(LINK) $(LINKFLAGS) $(EXTRA_PATH) $(OBJ) $(EXTRA_LIB) $(LIB) -o $(EXE) $(SIZE) $(EXE) # Library targets lib: $(OBJ) $(EXTRA_LINK_DEPENDS) $(ARCHIVE) $(ARFLAGS) $(EXE) $(OBJ) shlib: $(OBJ) $(EXTRA_LINK_DEPENDS) $(CC) $(CCFLAGS) $(SHFLAGS) $(SHLIBFLAGS) $(EXTRA_PATH) -o $(EXE) \ $(OBJ) $(EXTRA_LIB) $(LIB) # Compilation rules %.o:%.cpp $(EXTRA_CPP_DEPENDS) $(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $< %.d:%.cpp $(EXTRA_CPP_DEPENDS) $(CC) $(CCFLAGS) $(EXTRA_INC) $(DEPFLAGS) $< > $@ %.o:%.cu $(EXTRA_CPP_DEPENDS) $(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $< # Individual dependencies depend : fastdep.exe $(SRC) @./fastdep.exe $(EXTRA_INC) -- $^ > .depend || exit 1 fastdep.exe: ../DEPEND/fastdep.c cc -O -o $@ $< sinclude .depend
