Unverified Commit 6bd6e627 authored by Steve Plimpton's avatar Steve Plimpton Committed by GitHub
Browse files

Merge pull request #763 from stanmoore1/kk_neighlist

Enhance Kokkos Neighbor lists
parents 57dd6c78 a5998179
Loading
Loading
Loading
Loading
+4 −0
Original line number Diff line number Diff line
@@ -151,6 +151,10 @@ action neighbor_kokkos.cpp
action neighbor_kokkos.h
action npair_copy_kokkos.cpp
action npair_copy_kokkos.h
action npair_halffull_kokkos.cpp
action npair_halffull_kokkos.h
action npair_skip_kokkos.cpp
action npair_skip_kokkos.h
action npair_kokkos.cpp
action npair_kokkos.h
action npair_ssa_kokkos.cpp npair_half_bin_newton_ssa.cpp
+128 −0
Original line number Diff line number Diff line
/* ----------------------------------------------------------------------
   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
   http://lammps.sandia.gov, Sandia National Laboratories
   Steve Plimpton, sjplimp@sandia.gov

   Copyright (2003) Sandia Corporation.  Under the terms of Contract
   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
   certain rights in this software.  This software is distributed under
   the GNU General Public License.

   See the README file in the top-level LAMMPS directory.
------------------------------------------------------------------------- */

#include "npair_halffull_kokkos.h"
#include "neighbor.h"
#include "neigh_list_kokkos.h"
#include "atom_kokkos.h"
#include "atom_vec.h"
#include "molecule.h"
#include "domain.h"
#include "my_page.h"
#include "error.h"
#include "atom_masks.h"

using namespace LAMMPS_NS;

/* ---------------------------------------------------------------------- */

template<class DeviceType, int NEWTON>
NPairHalffullKokkos<DeviceType,NEWTON>::NPairHalffullKokkos(LAMMPS *lmp) : NPair(lmp) {
  atomKK = (AtomKokkos *) atom;
  execution_space = ExecutionSpaceFromDevice<DeviceType>::space;
}

/* ----------------------------------------------------------------------
   build half list from full list
   pair stored once if i,j are both owned and i < j
   pair stored by me if j is ghost (also stored by proc owning j)
   works if full list is a skip list
   works for owned (non-ghost) list, also for ghost list
   if ghost, also store neighbors of ghost atoms & set inum,gnum correctly
------------------------------------------------------------------------- */

template<class DeviceType, int NEWTON>
void NPairHalffullKokkos<DeviceType,NEWTON>::build(NeighList *list)
{
  if (NEWTON) {
    x = atomKK->k_x.view<DeviceType>();
    atomKK->sync(execution_space,X_MASK);
  }
  nlocal = atom->nlocal;

  NeighListKokkos<DeviceType>* k_list_full = static_cast<NeighListKokkos<DeviceType>*>(list->listfull);
  d_ilist_full = k_list_full->d_ilist;
  d_numneigh_full = k_list_full->d_numneigh;
  d_neighbors_full = k_list_full->d_neighbors;
  int inum_full = list->listfull->inum;
  if (list->ghost) inum_full += list->listfull->gnum;

  NeighListKokkos<DeviceType>* k_list = static_cast<NeighListKokkos<DeviceType>*>(list);
  k_list->maxneighs = k_list_full->maxneighs; // simple, but could be made more memory efficient
  k_list->grow(atom->nmax);
  d_ilist = k_list->d_ilist;
  d_numneigh = k_list->d_numneigh;
  d_neighbors = k_list->d_neighbors;

  // loop over parent full list

  copymode = 1;
  Kokkos::parallel_for(Kokkos::RangePolicy<DeviceType, TagNPairHalffullCompute>(0,inum_full),*this);

  list->inum = k_list_full->inum;
  list->gnum = k_list_full->gnum;
  k_list->k_ilist.template modify<DeviceType>();

  copymode = 0;
}

template<class DeviceType, int NEWTON>
KOKKOS_INLINE_FUNCTION
void NPairHalffullKokkos<DeviceType,NEWTON>::operator()(TagNPairHalffullCompute, const int &ii) const {
  int n = 0;

  const int i = d_ilist_full(ii);
  F_FLOAT xtmp,ytmp,ztmp;
  if (NEWTON) {
    xtmp = x(i,0);
    ytmp = x(i,1);
    ztmp = x(i,2);
  }

  // loop over full neighbor list

  const int jnum = d_numneigh_full(i);
  const AtomNeighbors neighbors_i = AtomNeighbors(&d_neighbors(i,0),d_numneigh(i),
                                                  &d_neighbors(i,1)-&d_neighbors(i,0));

  for (int jj = 0; jj < jnum; jj++) {
    const int joriginal = d_neighbors_full(i,jj);
    const int j = joriginal & NEIGHMASK;
    if (NEWTON) {
      if (j < nlocal) {
        if (i > j) continue;
      } else {
        if (x(j,2) < ztmp) continue;
        if (x(j,2) == ztmp) {
          if (x(j,1) < ytmp) continue;
          if (x(j,1) == ytmp && x(j,0) < xtmp) continue;
        }
      }
      neighbors_i(n++) = joriginal;
    } else {
      if (j > i) neighbors_i(n++) = joriginal;
    }
  }

  d_numneigh(i) = n;
  d_ilist(ii) = i;
}

namespace LAMMPS_NS {
template class NPairHalffullKokkos<LMPDeviceType,0>;
template class NPairHalffullKokkos<LMPDeviceType,1>;
#ifdef KOKKOS_HAVE_CUDA
template class NPairHalffullKokkos<LMPHostType,0>;
template class NPairHalffullKokkos<LMPHostType,1>;
#endif
}
+168 −0
Original line number Diff line number Diff line
/* -*- c++ -*- ----------------------------------------------------------
   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
   http://lammps.sandia.gov, Sandia National Laboratories
   Steve Plimpton, sjplimp@sandia.gov

   Copyright (2003) Sandia Corporation.  Under the terms of Contract
   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
   certain rights in this software.  This software is distributed under
   the GNU General Public License.

   See the README file in the top-level LAMMPS directory.
------------------------------------------------------------------------- */

#ifdef NPAIR_CLASS

// Newton

typedef NPairHalffullKokkos<LMPDeviceType,1> NPairKokkosHalffullNewtonDevice;
NPairStyle(halffull/newton/kk/device,
           NPairKokkosHalffullNewtonDevice,
           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,1> NPairKokkosHalffullNewtonHost;
NPairStyle(halffull/newton/kk/host,
           NPairKokkosHalffullNewtonHost,
           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_KOKKOS_HOST)

typedef NPairHalffullKokkos<LMPDeviceType,1> NPairKokkosHalffullNewtonDevice;
NPairStyle(halffull/newton/skip/kk/device,
           NPairKokkosHalffullNewtonDevice,
           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_SKIP | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,1> NPairKokkosHalffullNewtonHost;
NPairStyle(halffull/newton/skip/kk/host,
           NPairKokkosHalffullNewtonHost,
           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_SKIP | NP_KOKKOS_HOST)

// Newtoff

typedef NPairHalffullKokkos<LMPDeviceType,0> NPairKokkosHalffullNewtoffDevice;
NPairStyle(halffull/newtoff/kk/device,
           NPairKokkosHalffullNewtoffDevice,
           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,0> NPairKokkosHalffullNewtoffHost;
NPairStyle(halffull/newtoff/kk/host,
           NPairKokkosHalffullNewtoffHost,
           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_KOKKOS_HOST)

typedef NPairHalffullKokkos<LMPDeviceType,0> NPairKokkosHalffullNewtoffDevice;
NPairStyle(halffull/newtoff/skip/kk/device,
           NPairKokkosHalffullNewtoffDevice,
           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_SKIP | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,0> NPairKokkosHalffullNewtoffHost;
NPairStyle(halffull/newtoff/skip/kk/host,
           NPairKokkosHalffullNewtoffHost,
           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_SKIP | NP_KOKKOS_HOST)

//************ Ghost **************

// Newton

typedef NPairHalffullKokkos<LMPDeviceType,1> NPairKokkosHalffullNewtonGhostDevice;
NPairStyle(halffull/newton/ghost/kk/device,
           NPairKokkosHalffullNewtonGhostDevice,
           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,1> NPairKokkosHalffullNewtonHost;
NPairStyle(halffull/newton/ghost/kk/host,
           NPairKokkosHalffullNewtonHost,
           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_HOST)

typedef NPairHalffullKokkos<LMPDeviceType,1> NPairKokkosHalffullNewtonGhostDevice;
NPairStyle(halffull/newton/skip/ghost/kk/device,
           NPairKokkosHalffullNewtonGhostDevice,
           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_GHOST | NP_SKIP | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,1> NPairKokkosHalffullNewtonHost;
NPairStyle(halffull/newton/skip/ghost/kk/host,
           NPairKokkosHalffullNewtonHost,
           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_GHOST | NP_SKIP | NP_KOKKOS_HOST)

// Newtoff

typedef NPairHalffullKokkos<LMPDeviceType,0> NPairKokkosHalffullNewtoffGhostDevice;
NPairStyle(halffull/newtoff/ghost/kk/device,
           NPairKokkosHalffullNewtoffGhostDevice,
           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,0> NPairKokkosHalffullNewtoffHost;
NPairStyle(halffull/newtoff/ghost/kk/host,
           NPairKokkosHalffullNewtoffHost,
           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_HOST)

typedef NPairHalffullKokkos<LMPDeviceType,0> NPairKokkosHalffullNewtoffGhostDevice;
NPairStyle(halffull/newtoff/skip/ghost/kk/device,
           NPairKokkosHalffullNewtoffGhostDevice,
           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_GHOST | NP_SKIP | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,0> NPairKokkosHalffullNewtoffHost;
NPairStyle(halffull/newtoff/skip/ghost/kk/host,
           NPairKokkosHalffullNewtoffHost,
           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |
           NP_ORTHO | NP_TRI | NP_GHOST | NP_SKIP | NP_KOKKOS_HOST)

#else

#ifndef LMP_NPAIR_HALFFULL_KOKKOS_H
#define LMP_NPAIR_HALFFULL_KOKKOS_H

#include "npair.h"
#include "kokkos_type.h"

namespace LAMMPS_NS {

struct TagNPairHalffullCompute{};

template<class DeviceType, int NEWTON>
class NPairHalffullKokkos : public NPair {
 public:
  typedef DeviceType device_type;
  typedef ArrayTypes<DeviceType> AT;

  NPairHalffullKokkos(class LAMMPS *);
  ~NPairHalffullKokkos() {}
  void build(class NeighList *);

  KOKKOS_INLINE_FUNCTION
  void operator()(TagNPairHalffullCompute, const int&) const;

 private:
  int nlocal;

  typename AT::t_x_array_randomread x;

  typename AT::t_neighbors_2d_const d_neighbors_full;
  typename AT::t_int_1d_const d_ilist_full;
  typename AT::t_int_1d_const d_numneigh_full;

  typename AT::t_neighbors_2d d_neighbors;
  typename AT::t_int_1d d_ilist;
  typename AT::t_int_1d d_numneigh;
};

}

#endif
#endif

/* ERROR/WARNING messages:

*/
+157 −0
Original line number Diff line number Diff line
/* ----------------------------------------------------------------------
   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
   http://lammps.sandia.gov, Sandia National Laboratories
   Steve Plimpton, sjplimp@sandia.gov

   Copyright (2003) Sandia Corporation.  Under the terms of Contract
   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
   certain rights in this software.  This software is distributed under
   the GNU General Public License.

   See the README file in the top-level LAMMPS directory.
------------------------------------------------------------------------- */

#include "npair_skip_kokkos.h"
#include "neighbor.h"
#include "neigh_list_kokkos.h"
#include "atom_kokkos.h"
#include "atom_vec.h"
#include "molecule.h"
#include "domain.h"
#include "my_page.h"
#include "error.h"
#include "atom_masks.h"

using namespace LAMMPS_NS;

/* ---------------------------------------------------------------------- */

template<class DeviceType>
NPairSkipKokkos<DeviceType>::NPairSkipKokkos(LAMMPS *lmp) : NPair(lmp) {
  atomKK = (AtomKokkos *) atom;
  execution_space = ExecutionSpaceFromDevice<DeviceType>::space;
  d_inum = typename AT::t_int_scalar("npair_skip:inum");
}

/* ----------------------------------------------------------------------
   build skip list for subset of types from parent list
   works for half and full lists
   works for owned (non-ghost) list, also for ghost list
   iskip and ijskip flag which atom types and type pairs to skip
   if ghost, also store neighbors of ghost atoms & set inum,gnum correctly
------------------------------------------------------------------------- */

template<class DeviceType>
void NPairSkipKokkos<DeviceType>::build(NeighList *list)
{
  atomKK->sync(execution_space,TYPE_MASK);
  type = atomKK->k_type.view<DeviceType>();
  nlocal = atom->nlocal;


  NeighListKokkos<DeviceType>* k_list_skip = static_cast<NeighListKokkos<DeviceType>*>(list->listskip);
  d_ilist_skip = k_list_skip->d_ilist;
  d_numneigh_skip = k_list_skip->d_numneigh;
  d_neighbors_skip = k_list_skip->d_neighbors;

  num_skip = list->listskip->inum;
  if (list->ghost) num_skip += list->listskip->gnum;

  NeighListKokkos<DeviceType>* k_list = static_cast<NeighListKokkos<DeviceType>*>(list);
  k_list->maxneighs = k_list_skip->maxneighs; // simple, but could be made more memory efficient
  k_list->grow(atom->nmax);
  d_ilist = k_list->d_ilist;
  d_numneigh = k_list->d_numneigh;
  d_neighbors = k_list->d_neighbors;

  int ntypes = atom->ntypes;

  k_iskip = DAT::tdual_int_1d("npair_skip:iskip",ntypes+1);
  k_ijskip = DAT::tdual_int_2d("npair_skip:ijskip",ntypes+1,ntypes+1);
  d_iskip = k_iskip.view<DeviceType>();
  d_ijskip = k_ijskip.view<DeviceType>();

  for (int itype = 1; itype <= ntypes; itype++) {
    k_iskip.h_view(itype) = list->iskip[itype];
    for (int jtype = 1; jtype <= ntypes; jtype++) {
      k_ijskip.h_view(itype,jtype) = list->ijskip[itype][jtype];
    }
  }
  k_iskip.modify<LMPHostType>();
  k_ijskip.modify<LMPHostType>();

  k_iskip.sync<DeviceType>();
  k_ijskip.sync<DeviceType>();

  // loop over atoms in other list
  // skip I atom entirely if iskip is set for type[I]
  // skip I,J pair if ijskip is set for type[I],type[J]

  copymode = 1;
  Kokkos::parallel_scan(Kokkos::RangePolicy<DeviceType, TagNPairSkipCompute>(0,num_skip),*this);

  auto h_inum = Kokkos::create_mirror_view(d_inum);
  Kokkos::deep_copy(h_inum,d_inum);
  const int inum = h_inum();
  list->inum = inum;
  if (list->ghost) {
    int num = 0;
    Kokkos::parallel_reduce(Kokkos::RangePolicy<DeviceType, TagNPairSkipCountLocal>(0,inum),*this,num);
    list->inum = num;
    list->gnum = inum - num;
  }
  copymode = 0;
}

template<class DeviceType>
KOKKOS_INLINE_FUNCTION
void NPairSkipKokkos<DeviceType>::operator()(TagNPairSkipCompute, const int &ii, int &inum, const bool &final) const {

  const int i = d_ilist_skip(ii);
  const int itype = type(i);

  if (!d_iskip(itype)) {

    if (final) {

      int n = 0;

      // loop over parent non-skip list

      const int jnum = d_numneigh_skip(i);
      const AtomNeighbors neighbors_i = AtomNeighbors(&d_neighbors(i,0),d_numneigh(i),
                                                    &d_neighbors(i,1)-&d_neighbors(i,0));

      for (int jj = 0; jj < jnum; jj++) {
        const int joriginal = d_neighbors_skip(i,jj);
        int j = joriginal & NEIGHMASK;
        if (d_ijskip(itype,type(j))) continue;
        neighbors_i(n++) = joriginal;
      }

      d_numneigh(i) = n;
      d_ilist(inum) = i;
    }

    inum++;
  }

  if (final) {
    if (ii == num_skip-1)
      d_inum() = inum;
  }
}

template<class DeviceType>
KOKKOS_INLINE_FUNCTION
void NPairSkipKokkos<DeviceType>::operator()(TagNPairSkipCountLocal, const int &i, int &num) const {
  if (d_ilist[i] < nlocal) num++;
}


namespace LAMMPS_NS {
template class NPairSkipKokkos<LMPDeviceType>;
#ifdef KOKKOS_HAVE_CUDA
template class NPairSkipKokkos<LMPHostType>;
#endif
}
+102 −0
Original line number Diff line number Diff line
/* -*- c++ -*- ----------------------------------------------------------
   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
   http://lammps.sandia.gov, Sandia National Laboratories
   Steve Plimpton, sjplimp@sandia.gov

   Copyright (2003) Sandia Corporation.  Under the terms of Contract
   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
   certain rights in this software.  This software is distributed under
   the GNU General Public License.

   See the README file in the top-level LAMMPS directory.
------------------------------------------------------------------------- */

#ifdef NPAIR_CLASS

typedef NPairSkipKokkos<LMPDeviceType> NPairKokkosSkipDevice;
NPairStyle(skip/kk/device,
           NPairKokkosSkipDevice,
           NP_SKIP | NP_HALF | NP_FULL |
           NP_NSQ | NP_BIN | NP_MULTI |
           NP_NEWTON | NP_NEWTOFF | NP_ORTHO | NP_TRI | NP_KOKKOS_DEVICE)

typedef NPairSkipKokkos<LMPDeviceType> NPairKokkosSkipGhostDevice;
NPairStyle(skip/ghost/kk/device,
           NPairKokkosSkipGhostDevice,
           NP_SKIP | NP_HALF | NP_FULL |
           NP_NSQ | NP_BIN | NP_MULTI |
           NP_NEWTON | NP_NEWTOFF | NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_DEVICE)

typedef NPairSkipKokkos<LMPHostType> NPairKokkosSkipHost;
NPairStyle(skip/kk/host,
           NPairKokkosSkipHost,
           NP_SKIP | NP_HALF | NP_FULL |
           NP_NSQ | NP_BIN | NP_MULTI |
           NP_NEWTON | NP_NEWTOFF | NP_ORTHO | NP_TRI | NP_KOKKOS_HOST)

typedef NPairSkipKokkos<LMPHostType> NPairKokkosSkipGhostHost;
NPairStyle(skip/ghost/kk/host,
           NPairKokkosSkipGhostHost,
           NP_SKIP | NP_HALF | NP_FULL |
           NP_NSQ | NP_BIN | NP_MULTI |
           NP_NEWTON | NP_NEWTOFF | NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_HOST)

#else

#ifndef LMP_NPAIR_SKIP_KOKKOS_H
#define LMP_NPAIR_SKIP_KOKKOS_H

#include "npair.h"
#include "kokkos_type.h"

namespace LAMMPS_NS {

struct TagNPairSkipCompute{};
struct TagNPairSkipCountLocal{};

template<class DeviceType>
class NPairSkipKokkos : public NPair {
 public:
  typedef DeviceType device_type;
  typedef int value_type;
  typedef ArrayTypes<DeviceType> AT;

  NPairSkipKokkos(class LAMMPS *);
  ~NPairSkipKokkos() {}
  void build(class NeighList *);

  KOKKOS_INLINE_FUNCTION
  void operator()(TagNPairSkipCompute, const int&, int&, const bool&) const;

  KOKKOS_INLINE_FUNCTION
  void operator()(TagNPairSkipCountLocal, const int&, int&) const;

 private:
  int nlocal,num_skip;

  typename AT::t_int_1d_randomread type;

  typename AT::t_int_scalar d_inum;

  typename AT::t_neighbors_2d_const d_neighbors_skip;
  typename AT::t_int_1d_const d_ilist_skip;
  typename AT::t_int_1d_const d_numneigh_skip;

  typename AT::t_neighbors_2d d_neighbors;
  typename AT::t_int_1d d_ilist;
  typename AT::t_int_1d d_numneigh;

  DAT::tdual_int_1d k_iskip;
  DAT::tdual_int_2d k_ijskip;
  typename AT::t_int_1d d_iskip;
  typename AT::t_int_2d d_ijskip;
};

}

#endif
#endif

/* ERROR/WARNING messages:

*/
Loading