Commit f509f133 authored by Steve Plimpton's avatar Steve Plimpton
Browse files

patch 13Dec16: neighbor refactor, Stan pppm/disp bug fix, M Brown INTEL package updates

parent 624c57e9

doc/src/Manual.txt

+2 −2
Original line number Diff line number Diff line 
<!-- HTML_ONLY -->

<HEAD>

<TITLE>LAMMPS Users Manual</TITLE>

<META NAME="docnumber" CONTENT="30 Nov 2016 version">

<META NAME="docnumber" CONTENT="13 Dec 2016 version">

<META NAME="author" CONTENT="http://lammps.sandia.gov - Sandia National Laboratories">

<META NAME="copyright" CONTENT="Copyright (2003) Sandia Corporation.  This software and manual is distributed under the GNU General Public License.">

</HEAD>
@@ -21,7 +21,7 @@ 
<H1></H1>



LAMMPS Documentation :c,h3

30 Nov 2016 version :c,h4

13 Dec 2016 version :c,h4



Version info: :h4

src/MAKE/MACHINES/Makefile.cori2

0 → 100755
+127 −0
Original line number Diff line number Diff line 
# cori2 = NERSC Cori II KNL, static build, FFTW (single precision)



# ---------------------------------------------------------------------

#   module swap craype-haswell craype-mic-knl

#   module load fftw

#   module load craype-hugepages2M

#   Recommend using #SBATCH -S 2 for core specialization

# ---------------------------------------------------------------------



SHELL = /bin/sh



# ---------------------------------------------------------------------

# compiler/linker settings

# specify flags and libraries needed for your compiler



CC =		CC

OPTFLAGS =      -xMIC-AVX512 -O2 -fp-model fast=2 -no-prec-div -qoverride-limits

CCFLAGS =	-g -qopenmp -DLAMMPS_MEMALIGN=64 -qno-offload \

                -fno-alias -ansi-alias -restrict $(OPTFLAGS) -DLMP_INTEL_NO_TBB

SHFLAGS =	-fPIC

DEPFLAGS =	-M



LINK =		CC

LINKFLAGS =	-g -qopenmp $(OPTFLAGS)

LIB =           

SIZE =		size



ARCHIVE =	ar

ARFLAGS =	-rc

SHLIBFLAGS =	-shared



# ---------------------------------------------------------------------

# LAMMPS-specific settings, all OPTIONAL

# specify settings for LAMMPS features you will use

# if you change any -D setting, do full re-compile after "make clean"



# LAMMPS ifdef settings

# see possible settings in Section 2.2 (step 4) of manual



LMP_INC =	#-DLAMMPS_GZIP -DLAMMPS_JPEG



# MPI library

# see discussion in Section 2.2 (step 5) of manual

# MPI wrapper compiler/linker can provide this info

# can point to dummy MPI library in src/STUBS as in Makefile.serial

# use -D MPICH and OMPI settings in INC to avoid C++ lib conflicts

# INC = path for mpi.h, MPI compiler settings

# PATH = path for MPI library

# LIB = name of MPI library



MPI_INC =       -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1

MPI_PATH = 

MPI_LIB =



# FFT library

# see discussion in Section 2.2 (step 6) of manaul

# can be left blank to use provided KISS FFT library

# INC = -DFFT setting, e.g. -DFFT_FFTW, FFT compiler settings

# PATH = path for FFT library

# LIB = name of FFT library



FFT_INC =       -DFFT_FFTW3 -DFFT_SINGLE

FFT_PATH = 

FFT_LIB =       -lfftw3f



# JPEG and/or PNG library

# see discussion in Section 2.2 (step 7) of manual

# only needed if -DLAMMPS_JPEG or -DLAMMPS_PNG listed with LMP_INC

# INC = path(s) for jpeglib.h and/or png.h

# PATH = path(s) for JPEG library and/or PNG library

# LIB = name(s) of JPEG library and/or PNG library



JPG_INC =       

JPG_PATH = 	

JPG_LIB =	#-ljpeg



# ---------------------------------------------------------------------

# build rules and dependencies

# do not edit this section



include	Makefile.package.settings

include	Makefile.package



EXTRA_INC = $(LMP_INC) $(PKG_INC) $(MPI_INC) $(FFT_INC) $(JPG_INC) $(PKG_SYSINC)

EXTRA_PATH = $(PKG_PATH) $(MPI_PATH) $(FFT_PATH) $(JPG_PATH) $(PKG_SYSPATH)

EXTRA_LIB = $(PKG_LIB) $(MPI_LIB) $(FFT_LIB) $(JPG_LIB) $(PKG_SYSLIB)



# Path to src files



vpath %.cpp ..

vpath %.h ..



# Link target



$(EXE):	$(OBJ)

	$(LINK) $(LINKFLAGS) $(EXTRA_PATH) $(OBJ) $(EXTRA_LIB) $(LIB) -o $(EXE)

	$(SIZE) $(EXE)



# Library targets



lib:	$(OBJ)

	$(ARCHIVE) $(ARFLAGS) $(EXE) $(OBJ)



shlib:	$(OBJ)

	$(CC) $(CCFLAGS) $(SHFLAGS) $(SHLIBFLAGS) $(EXTRA_PATH) -o $(EXE) \

        $(OBJ) $(EXTRA_LIB) $(LIB)



# Compilation rules



%.o:%.cpp

	$(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $<



%.d:%.cpp

	$(CC) $(CCFLAGS) $(EXTRA_INC) $(DEPFLAGS) $< > $@



%.o:%.cu

	$(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $<



# Individual dependencies



depend : fastdep.exe $(SRC)

	@./fastdep.exe $(EXTRA_INC) -- $^ > .depend || exit 1



fastdep.exe: ../DEPEND/fastdep.c

	cc -O -o $@ $<



sinclude .depend

src/MAKE/OPTIONS/Makefile.intel_cpu

0 → 100755
+123 −0
Original line number Diff line number Diff line 
# intel_cpu_intelmpi = USER-INTEL package, Intel MPI, MKL FFT



SHELL = /bin/sh



# ---------------------------------------------------------------------

# compiler/linker settings

# specify flags and libraries needed for your compiler



CC =		mpiicpc 

OPTFLAGS =      -xHost -O2 -fp-model fast=2 -no-prec-div -qoverride-limits

CCFLAGS =	-g -qopenmp -DLAMMPS_MEMALIGN=64 -no-offload \

                -fno-alias -ansi-alias -restrict $(OPTFLAGS)

SHFLAGS =	-fPIC

DEPFLAGS =	-M



LINK =		mpiicpc

LINKFLAGS =	-g -qopenmp $(OPTFLAGS)

LIB =           -ltbbmalloc -ltbbmalloc_proxy

SIZE =		size



ARCHIVE =	ar

ARFLAGS =	-rc

SHLIBFLAGS =	-shared



# ---------------------------------------------------------------------

# LAMMPS-specific settings, all OPTIONAL

# specify settings for LAMMPS features you will use

# if you change any -D setting, do full re-compile after "make clean"



# LAMMPS ifdef settings

# see possible settings in Section 2.2 (step 4) of manual



LMP_INC =	-DLAMMPS_GZIP -DLAMMPS_JPEG



# MPI library

# see discussion in Section 2.2 (step 5) of manual

# MPI wrapper compiler/linker can provide this info

# can point to dummy MPI library in src/STUBS as in Makefile.serial

# use -D MPICH and OMPI settings in INC to avoid C++ lib conflicts

# INC = path for mpi.h, MPI compiler settings

# PATH = path for MPI library

# LIB = name of MPI library



MPI_INC =       -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1

MPI_PATH = 

MPI_LIB =



# FFT library

# see discussion in Section 2.2 (step 6) of manaul

# can be left blank to use provided KISS FFT library

# INC = -DFFT setting, e.g. -DFFT_FFTW, FFT compiler settings

# PATH = path for FFT library

# LIB = name of FFT library



FFT_INC =       -DFFT_MKL -DFFT_SINGLE

FFT_PATH = 

FFT_LIB =       -L$MKLROOT/lib/intel64/ -lmkl_intel_ilp64 \

                -lmkl_sequential -lmkl_core	



# JPEG and/or PNG library

# see discussion in Section 2.2 (step 7) of manual

# only needed if -DLAMMPS_JPEG or -DLAMMPS_PNG listed with LMP_INC

# INC = path(s) for jpeglib.h and/or png.h

# PATH = path(s) for JPEG library and/or PNG library

# LIB = name(s) of JPEG library and/or PNG library



JPG_INC =       

JPG_PATH = 	

JPG_LIB =	-ljpeg



# ---------------------------------------------------------------------

# build rules and dependencies

# do not edit this section



include	Makefile.package.settings

include	Makefile.package



EXTRA_INC = $(LMP_INC) $(PKG_INC) $(MPI_INC) $(FFT_INC) $(JPG_INC) $(PKG_SYSINC)

EXTRA_PATH = $(PKG_PATH) $(MPI_PATH) $(FFT_PATH) $(JPG_PATH) $(PKG_SYSPATH)

EXTRA_LIB = $(PKG_LIB) $(MPI_LIB) $(FFT_LIB) $(JPG_LIB) $(PKG_SYSLIB)

EXTRA_CPP_DEPENDS = $(PKG_CPP_DEPENDS)

EXTRA_LINK_DEPENDS = $(PKG_LINK_DEPENDS)



# Path to src files



vpath %.cpp ..

vpath %.h ..



# Link target



$(EXE):	$(OBJ) $(EXTRA_LINK_DEPENDS)

	$(LINK) $(LINKFLAGS) $(EXTRA_PATH) $(OBJ) $(EXTRA_LIB) $(LIB) -o $(EXE)

	$(SIZE) $(EXE)



# Library targets



lib:	$(OBJ) $(EXTRA_LINK_DEPENDS)

	$(ARCHIVE) $(ARFLAGS) $(EXE) $(OBJ)



shlib:	$(OBJ) $(EXTRA_LINK_DEPENDS)

	$(CC) $(CCFLAGS) $(SHFLAGS) $(SHLIBFLAGS) $(EXTRA_PATH) -o $(EXE) \

        $(OBJ) $(EXTRA_LIB) $(LIB)



# Compilation rules



%.o:%.cpp $(EXTRA_CPP_DEPENDS)

	$(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $<



%.d:%.cpp $(EXTRA_CPP_DEPENDS)

	$(CC) $(CCFLAGS) $(EXTRA_INC) $(DEPFLAGS) $< > $@



%.o:%.cu $(EXTRA_CPP_DEPENDS)

	$(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $<



# Individual dependencies



depend : fastdep.exe $(SRC)

	@./fastdep.exe $(EXTRA_INC) -- $^ > .depend || exit 1



fastdep.exe: ../DEPEND/fastdep.c

	cc -O -o $@ $<



sinclude .depend

src/MAKE/OPTIONS/Makefile.intel_knl_coprocessor

0 → 100644
+123 −0
Original line number Diff line number Diff line 
# intel_phi = USER-INTEL with Phi x200 (KNL) offload support,Intel MPI,MKL FFT



SHELL = /bin/sh



# ---------------------------------------------------------------------

# compiler/linker settings

# specify flags and libraries needed for your compiler



CC =		mpiicpc 

MIC_OPT =       -qoffload-arch=mic-avx512 -fp-model fast=2

CCFLAGS =	-g -O3 -qopenmp -DLMP_INTEL_OFFLOAD -DLAMMPS_MEMALIGN=64 \

                -xHost -fno-alias -ansi-alias -restrict \

                -qoverride-limits $(MIC_OPT)

SHFLAGS =	-fPIC

DEPFLAGS =	-M



LINK =		mpiicpc

LINKFLAGS =	-g -O3 -xHost -qopenmp -qoffload $(MIC_OPT)

LIB =           -ltbbmalloc

SIZE =		size



ARCHIVE =	ar

ARFLAGS =	-rc

SHLIBFLAGS =	-shared



# ---------------------------------------------------------------------

# LAMMPS-specific settings, all OPTIONAL

# specify settings for LAMMPS features you will use

# if you change any -D setting, do full re-compile after "make clean"



# LAMMPS ifdef settings

# see possible settings in Section 2.2 (step 4) of manual



LMP_INC =	-DLAMMPS_GZIP -DLAMMPS_JPEG



# MPI library

# see discussion in Section 2.2 (step 5) of manual

# MPI wrapper compiler/linker can provide this info

# can point to dummy MPI library in src/STUBS as in Makefile.serial

# use -D MPICH and OMPI settings in INC to avoid C++ lib conflicts

# INC = path for mpi.h, MPI compiler settings

# PATH = path for MPI library

# LIB = name of MPI library



MPI_INC =       -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1

MPI_PATH = 

MPI_LIB =



# FFT library

# see discussion in Section 2.2 (step 6) of manaul

# can be left blank to use provided KISS FFT library

# INC = -DFFT setting, e.g. -DFFT_FFTW, FFT compiler settings

# PATH = path for FFT library

# LIB = name of FFT library



FFT_INC =      -DFFT_MKL -DFFT_SINGLE

FFT_PATH = 

FFT_LIB =	-L$(MKLROOT)/lib/intel64/ -lmkl_intel_ilp64 -lmkl_sequential -lmkl_core



# JPEG and/or PNG library

# see discussion in Section 2.2 (step 7) of manual

# only needed if -DLAMMPS_JPEG or -DLAMMPS_PNG listed with LMP_INC

# INC = path(s) for jpeglib.h and/or png.h

# PATH = path(s) for JPEG library and/or PNG library

# LIB = name(s) of JPEG library and/or PNG library



JPG_INC =       

JPG_PATH = 	

JPG_LIB =	-ljpeg



# ---------------------------------------------------------------------

# build rules and dependencies

# do not edit this section



include	Makefile.package.settings

include	Makefile.package



EXTRA_INC = $(LMP_INC) $(PKG_INC) $(MPI_INC) $(FFT_INC) $(JPG_INC) $(PKG_SYSINC)

EXTRA_PATH = $(PKG_PATH) $(MPI_PATH) $(FFT_PATH) $(JPG_PATH) $(PKG_SYSPATH)

EXTRA_LIB = $(PKG_LIB) $(MPI_LIB) $(FFT_LIB) $(JPG_LIB) $(PKG_SYSLIB)

EXTRA_CPP_DEPENDS = $(PKG_CPP_DEPENDS)

EXTRA_LINK_DEPENDS = $(PKG_LINK_DEPENDS)



# Path to src files



vpath %.cpp ..

vpath %.h ..



# Link target



$(EXE):	$(OBJ) $(EXTRA_LINK_DEPENDS)

	$(LINK) $(LINKFLAGS) $(EXTRA_PATH) $(OBJ) $(EXTRA_LIB) $(LIB) -o $(EXE)

	$(SIZE) $(EXE)



# Library targets



lib:	$(OBJ) $(EXTRA_LINK_DEPENDS)

	$(ARCHIVE) $(ARFLAGS) $(EXE) $(OBJ)



shlib:	$(OBJ) $(EXTRA_LINK_DEPENDS)

	$(CC) $(CCFLAGS) $(SHFLAGS) $(SHLIBFLAGS) $(EXTRA_PATH) -o $(EXE) \

        $(OBJ) $(EXTRA_LIB) $(LIB)



# Compilation rules



%.o:%.cpp $(EXTRA_CPP_DEPENDS)

	$(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $<



%.d:%.cpp $(EXTRA_CPP_DEPENDS)

	$(CC) $(CCFLAGS) $(EXTRA_INC) $(DEPFLAGS) $< > $@



%.o:%.cu $(EXTRA_CPP_DEPENDS)

	$(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $<



# Individual dependencies



depend : fastdep.exe $(SRC)

	@./fastdep.exe $(EXTRA_INC) -- $^ > .depend || exit 1



fastdep.exe: ../DEPEND/fastdep.c

	cc -O -o $@ $<



sinclude .depend

src/USER-INTEL/fix_intel.cpp

+24 −8
Original line number Diff line number Diff line
@@ -63,7 +63,7 @@ FixIntel::FixIntel(LAMMPS *lmp, int narg, char **arg) : Fix(lmp, narg, arg) 
  _nbor_pack_width = 1;



  _precision_mode = PREC_MODE_MIXED;

  _offload_balance = 1.0;

  _offload_balance = -1.0;

  _overflow_flag[LMP_OVERFLOW] = 0;

  _off_overflow_flag[LMP_OVERFLOW] = 0;
@@ -189,10 +189,18 @@ FixIntel::FixIntel(LAMMPS *lmp, int narg, char **arg) : Fix(lmp, narg, arg) 
      offload_cores = omp_get_num_procs();

      omp_set_num_threads(offload_cores);

      max_offload_threads = omp_get_max_threads();

      #ifdef __AVX512F__

      if ( (offload_cores / 4) % 2 == 1) {

        offload_cores += 4;

        max_offload_threads += 4;

      }

      #endif

    }

    _max_offload_threads = max_offload_threads;

    _offload_cores = offload_cores;

    if (_offload_threads == 0) _offload_threads = offload_cores;

    if (_offload_cores > 244 && _offload_tpc > 2)

      _offload_tpc = 2;

  }

  #endif
@@ -317,6 +325,8 @@ void FixIntel::init() 
    error->all(FLERR,

	       "Currently, cannot use more than one intel style with hybrid.");



  neighbor->fix_intel = (void *)this;



  check_neighbor_intel();

  if (_precision_mode == PREC_MODE_SINGLE)

    _single_buffers->zero_ev();
@@ -1004,8 +1014,10 @@ void FixIntel::set_offload_affinity() 
  int offload_threads = _offload_threads;

  int offload_tpc = _offload_tpc;

  int offload_affinity_balanced = _offload_affinity_balanced;

  int offload_cores = _offload_cores;

  #pragma offload target(mic:_cop) mandatory \

    in(node_rank,offload_threads,offload_tpc,offload_affinity_balanced)

    in(node_rank,offload_threads,offload_tpc,offload_affinity_balanced, \

       offload_cores)

  {

    omp_set_num_threads(offload_threads);

    #pragma omp parallel
@@ -1013,20 +1025,24 @@ void FixIntel::set_offload_affinity() 
      int tnum = omp_get_thread_num();

      kmp_affinity_mask_t mask;

      kmp_create_affinity_mask(&mask);

      int proc;

      if (offload_affinity_balanced) {

	proc = offload_threads * node_rank + tnum;

      int proc = offload_threads * node_rank + tnum;

      #ifdef __AVX512F__

      proc = (proc / offload_tpc) + (proc % offload_tpc) * 

	     ((offload_cores) / 4);

      proc += 68;

      #else

      if (offload_affinity_balanced)

	proc = proc * 4 - (proc / 60) * 240 + proc / 60 + 1;

      } else {

	proc = offload_threads * node_rank + tnum;

      else

	proc += (proc / 4) * (4 - offload_tpc) + 1;

      }

      #endif

      kmp_set_affinity_mask_proc(proc, &mask);

      if (kmp_set_affinity(&mask) != 0)

	printf("Could not set affinity on rank %d thread %d to %d\n",

	       node_rank, tnum, proc);

    }

  }



  if (_precision_mode == PREC_MODE_SINGLE)

    _single_buffers->set_off_params(offload_threads, _cop, _separate_buffers);

  else if (_precision_mode == PREC_MODE_MIXED)

CRA Git | Maintained and supported by SUSTech CRA and CCSE