Commit 95218144 authored by Axel Kohlmeyer's avatar Axel Kohlmeyer
Browse files

rename Kokkos::gpu_direct to Kokkos::gpu_direct_flag 

This is for consistency with other parts of LAMMPS where
such variables have "flag" in their name. Also reduces
confusion with have_gpu_direct() function.

When we can safely detect, that GPU-direct is not available,
change the default setting of Kokkos::gpu_direct_flag from 1 to 0
parent dd9fed11

doc/src/package.txt

+8 −3
Original line number Diff line number Diff line
@@ -512,9 +512,12 @@ this keyword is set to {on}, buffers in GPU memory are passed directly 
through MPI send/receive calls. This reduces overhead of first copying

the data to the host CPU. However GPU-direct is not supported on all

systems, which can lead to segmentation faults and would require

using a value of {off}. When the {gpu/direct} keyword is set to {off}

while any of the {comm} keywords are set to {device}, the value for the

{comm} keywords will be automatically changed to {host}.

using a value of {off}. If LAMMPS can safely detect that GPU-direct is

not available (currently only possible with OpenMPI v2.0.0 or later),

then the {gpu/direct} keyword is automatically set to {off} by default.

When the {gpu/direct} keyword is set to {off} while any of the {comm}

keywords are set to {device}, the value for these {comm} keywords will

be automatically changed to {host}.



:line
@@ -624,6 +627,8 @@ switch"_Section_start.html#start_6. 


For the KOKKOS package, the option defaults neigh = full, neigh/qeq =

full, newton = off, binsize = 0.0, and comm = device, gpu/direct = on.

When LAMMPS can safely detect, that GPU-direct is not available, the

default value of gpu/direct becomes "off".

These settings are made automatically by the required "-k on"

"command-line switch"_Section_start.html#start_6. You can change them by

using the package kokkos command in your input script or via the "-pk

src/KOKKOS/comm_kokkos.cpp

+2 −2
Original line number Diff line number Diff line
@@ -406,7 +406,7 @@ void CommKokkos::forward_comm_pair_device(Pair *pair) 
    if (sendproc[iswap] != me) {

      double* buf_send_pair;

      double* buf_recv_pair;

      if (lmp->kokkos->gpu_direct) {

      if (lmp->kokkos->gpu_direct_flag) {

        buf_send_pair = k_buf_send_pair.view<DeviceType>().data();

        buf_recv_pair = k_buf_recv_pair.view<DeviceType>().data();

      } else {
@@ -424,7 +424,7 @@ void CommKokkos::forward_comm_pair_device(Pair *pair) 
        MPI_Send(buf_send_pair,n,MPI_DOUBLE,sendproc[iswap],0,world);

      if (recvnum[iswap]) MPI_Wait(&request,MPI_STATUS_IGNORE);



      if (!lmp->kokkos->gpu_direct) {

      if (!lmp->kokkos->gpu_direct_flag) {

        k_buf_recv_pair.modify<LMPHostType>();

        k_buf_recv_pair.sync<DeviceType>();

      }

src/KOKKOS/gridcomm_kokkos.cpp

+4 −4
Original line number Diff line number Diff line
@@ -529,7 +529,7 @@ void GridCommKokkos<DeviceType>::forward_comm(KSpace *kspace, int which) 
    if (swap[m].sendproc != me) {

      FFT_SCALAR* buf1;

      FFT_SCALAR* buf2;

      if (lmp->kokkos->gpu_direct) {

      if (lmp->kokkos->gpu_direct_flag) {

        buf1 = k_buf1.view<DeviceType>().data();

        buf2 = k_buf2.view<DeviceType>().data();

      } else {
@@ -545,7 +545,7 @@ void GridCommKokkos<DeviceType>::forward_comm(KSpace *kspace, int which) 
               swap[m].sendproc,0,gridcomm);

      MPI_Wait(&request,MPI_STATUS_IGNORE);



      if (!lmp->kokkos->gpu_direct) {

      if (!lmp->kokkos->gpu_direct_flag) {

        k_buf2.modify<LMPHostType>();

        k_buf2.sync<DeviceType>();

      }
@@ -579,7 +579,7 @@ void GridCommKokkos<DeviceType>::reverse_comm(KSpace *kspace, int which) 
    if (swap[m].recvproc != me) {

      FFT_SCALAR* buf1;

      FFT_SCALAR* buf2;

      if (lmp->kokkos->gpu_direct) {

      if (lmp->kokkos->gpu_direct_flag) {

        buf1 = k_buf1.view<DeviceType>().data();

        buf2 = k_buf2.view<DeviceType>().data();

      } else {
@@ -595,7 +595,7 @@ void GridCommKokkos<DeviceType>::reverse_comm(KSpace *kspace, int which) 
               swap[m].recvproc,0,gridcomm);

      MPI_Wait(&request,MPI_STATUS_IGNORE);



      if (!lmp->kokkos->gpu_direct) {

      if (!lmp->kokkos->gpu_direct_flag) {

        k_buf2.modify<LMPHostType>();

        k_buf2.sync<DeviceType>();

      }

src/KOKKOS/kokkos.cpp

+12 −9
Original line number Diff line number Diff line
@@ -158,14 +158,12 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) 
                     " but cannot determine if this is the case\n         try"

                     " '-pk kokkos gpu/direct off' when getting segmentation faults");

    } else if ( 0 == have_gpu_direct() ) {

      error->warning(FLERR,"GPU-direct is NOT available, but some parts of "

                     "Kokkos with CUDA require it by default\n         try"

                     " '-pk kokkos gpu/direct off' when getting segmentation faults");

      error->warning(FLERR,"GPU-direct is NOT available, "

                     "using '-pk kokkos gpu/direct off' by default");

    } else {

      ; // should never get here

    }

  }



#endif



  Kokkos::InitArguments args;
@@ -186,7 +184,12 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) 
  exchange_comm_on_host = 0;

  forward_comm_on_host = 0;

  reverse_comm_on_host = 0;

  gpu_direct = 1;

  gpu_direct_flag = 1;



#if KOKKOS_USE_CUDA

  // only if we can safely detect, that GPU-direct is not available, change default

  if (0 == have_gpu_direct()) gpu_direct_flag = 0;

#endif



#ifdef KILL_KOKKOS_ON_SIGSEGV

  signal(SIGSEGV, my_signal_handler);
@@ -217,7 +220,7 @@ void KokkosLMP::accelerator(int narg, char **arg) 
  double binsize = 0.0;

  exchange_comm_classic = forward_comm_classic = reverse_comm_classic = 0;

  exchange_comm_on_host = forward_comm_on_host = reverse_comm_on_host = 0;

  gpu_direct = 1;

  gpu_direct_flag = 1;



  int iarg = 0;

  while (iarg < narg) {
@@ -303,8 +306,8 @@ void KokkosLMP::accelerator(int narg, char **arg) 
      iarg += 2;

    } else if (strcmp(arg[iarg],"gpu/direct") == 0) {

      if (iarg+2 > narg) error->all(FLERR,"Illegal package kokkos command");

      if (strcmp(arg[iarg+1],"off") == 0) gpu_direct = 0;

      else if (strcmp(arg[iarg+1],"on") == 0) gpu_direct = 1;

      if (strcmp(arg[iarg+1],"off") == 0) gpu_direct_flag = 0;

      else if (strcmp(arg[iarg+1],"on") == 0) gpu_direct_flag = 1;

      else error->all(FLERR,"Illegal package kokkos command");

      iarg += 2;

    } else error->all(FLERR,"Illegal package kokkos command");
@@ -312,7 +315,7 @@ void KokkosLMP::accelerator(int narg, char **arg) 


  // if "gpu/direct off" and "comm device", change to "comm host"



  if (!gpu_direct) {

  if (!gpu_direct_flag) {

   if (exchange_comm_classic == 0 && exchange_comm_on_host == 0)

     exchange_comm_on_host = 1;

   if (forward_comm_classic == 0 && forward_comm_on_host == 0)

src/KOKKOS/kokkos.h

+1 −1
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ class KokkosLMP : protected Pointers { 
  int num_threads,ngpu;

  int numa;

  int auto_sync;

  int gpu_direct;

  int gpu_direct_flag;



  KokkosLMP(class LAMMPS *, int, char **);

  ~KokkosLMP();

CRA Git | Maintained and supported by SUSTech CRA and CCSE