Merge pull request #763 from stanmoore1/kk_neighlist

action neighbor_kokkos.h

action npair_copy_kokkos.cpp

action npair_copy_kokkos.h

action npair_halffull_kokkos.cpp

action npair_halffull_kokkos.h

action npair_skip_kokkos.cpp

action npair_skip_kokkos.h

action npair_kokkos.cpp

action npair_kokkos.h

action npair_ssa_kokkos.cpp npair_half_bin_newton_ssa.cpp

/* ----------------------------------------------------------------------

   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator

   http://lammps.sandia.gov, Sandia National Laboratories

   Steve Plimpton, sjplimp@sandia.gov

   Copyright (2003) Sandia Corporation.  Under the terms of Contract

   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains

   certain rights in this software.  This software is distributed under

   the GNU General Public License.

   See the README file in the top-level LAMMPS directory.

------------------------------------------------------------------------- */

#include "npair_halffull_kokkos.h"

#include "neighbor.h"

#include "neigh_list_kokkos.h"

#include "atom_kokkos.h"

#include "atom_vec.h"

#include "molecule.h"

#include "domain.h"

#include "my_page.h"

#include "error.h"

#include "atom_masks.h"

using namespace LAMMPS_NS;

/* ---------------------------------------------------------------------- */

template<class DeviceType, int NEWTON>

NPairHalffullKokkos<DeviceType,NEWTON>::NPairHalffullKokkos(LAMMPS *lmp) : NPair(lmp) {

  atomKK = (AtomKokkos *) atom;

  execution_space = ExecutionSpaceFromDevice<DeviceType>::space;

}

/* ----------------------------------------------------------------------

   build half list from full list

   pair stored once if i,j are both owned and i < j

   pair stored by me if j is ghost (also stored by proc owning j)

   works if full list is a skip list

   works for owned (non-ghost) list, also for ghost list

   if ghost, also store neighbors of ghost atoms & set inum,gnum correctly

------------------------------------------------------------------------- */

template<class DeviceType, int NEWTON>

void NPairHalffullKokkos<DeviceType,NEWTON>::build(NeighList *list)

{

  if (NEWTON) {

    x = atomKK->k_x.view<DeviceType>();

    atomKK->sync(execution_space,X_MASK);

  }

  nlocal = atom->nlocal;

  NeighListKokkos<DeviceType>* k_list_full = static_cast<NeighListKokkos<DeviceType>*>(list->listfull);

  d_ilist_full = k_list_full->d_ilist;

  d_numneigh_full = k_list_full->d_numneigh;

  d_neighbors_full = k_list_full->d_neighbors;

  int inum_full = list->listfull->inum;

  if (list->ghost) inum_full += list->listfull->gnum;

  NeighListKokkos<DeviceType>* k_list = static_cast<NeighListKokkos<DeviceType>*>(list);

  k_list->maxneighs = k_list_full->maxneighs; // simple, but could be made more memory efficient

  k_list->grow(atom->nmax);

  d_ilist = k_list->d_ilist;

  d_numneigh = k_list->d_numneigh;

  d_neighbors = k_list->d_neighbors;

  // loop over parent full list

  copymode = 1;

  Kokkos::parallel_for(Kokkos::RangePolicy<DeviceType, TagNPairHalffullCompute>(0,inum_full),*this);

  list->inum = k_list_full->inum;

  list->gnum = k_list_full->gnum;

  k_list->k_ilist.template modify<DeviceType>();

  copymode = 0;

}

template<class DeviceType, int NEWTON>

KOKKOS_INLINE_FUNCTION

void NPairHalffullKokkos<DeviceType,NEWTON>::operator()(TagNPairHalffullCompute, const int &ii) const {

  int n = 0;

  const int i = d_ilist_full(ii);

  F_FLOAT xtmp,ytmp,ztmp;

  if (NEWTON) {

    xtmp = x(i,0);

    ytmp = x(i,1);

    ztmp = x(i,2);

  }

  // loop over full neighbor list

  const int jnum = d_numneigh_full(i);

  const AtomNeighbors neighbors_i = AtomNeighbors(&d_neighbors(i,0),d_numneigh(i),

                                                  &d_neighbors(i,1)-&d_neighbors(i,0));

  for (int jj = 0; jj < jnum; jj++) {

    const int joriginal = d_neighbors_full(i,jj);

    const int j = joriginal & NEIGHMASK;

    if (NEWTON) {

      if (j < nlocal) {

        if (i > j) continue;

      } else {

        if (x(j,2) < ztmp) continue;

        if (x(j,2) == ztmp) {

          if (x(j,1) < ytmp) continue;

          if (x(j,1) == ytmp && x(j,0) < xtmp) continue;

        }

      }

      neighbors_i(n++) = joriginal;

    } else {

      if (j > i) neighbors_i(n++) = joriginal;

    }

  }

  d_numneigh(i) = n;

  d_ilist(ii) = i;

}

namespace LAMMPS_NS {

template class NPairHalffullKokkos<LMPDeviceType,0>;

template class NPairHalffullKokkos<LMPDeviceType,1>;

#ifdef KOKKOS_HAVE_CUDA

template class NPairHalffullKokkos<LMPHostType,0>;

template class NPairHalffullKokkos<LMPHostType,1>;

#endif

}

/* -*- c++ -*- ----------------------------------------------------------

   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator

   http://lammps.sandia.gov, Sandia National Laboratories

   Steve Plimpton, sjplimp@sandia.gov

   Copyright (2003) Sandia Corporation.  Under the terms of Contract

   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains

   certain rights in this software.  This software is distributed under

   the GNU General Public License.

   See the README file in the top-level LAMMPS directory.

------------------------------------------------------------------------- */

#ifdef NPAIR_CLASS

// Newton

typedef NPairHalffullKokkos<LMPDeviceType,1> NPairKokkosHalffullNewtonDevice;

NPairStyle(halffull/newton/kk/device,

           NPairKokkosHalffullNewtonDevice,

           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,1> NPairKokkosHalffullNewtonHost;

NPairStyle(halffull/newton/kk/host,

           NPairKokkosHalffullNewtonHost,

           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_KOKKOS_HOST)

typedef NPairHalffullKokkos<LMPDeviceType,1> NPairKokkosHalffullNewtonDevice;

NPairStyle(halffull/newton/skip/kk/device,

           NPairKokkosHalffullNewtonDevice,

           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_SKIP | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,1> NPairKokkosHalffullNewtonHost;

NPairStyle(halffull/newton/skip/kk/host,

           NPairKokkosHalffullNewtonHost,

           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_SKIP | NP_KOKKOS_HOST)

// Newtoff

typedef NPairHalffullKokkos<LMPDeviceType,0> NPairKokkosHalffullNewtoffDevice;

NPairStyle(halffull/newtoff/kk/device,

           NPairKokkosHalffullNewtoffDevice,

           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,0> NPairKokkosHalffullNewtoffHost;

NPairStyle(halffull/newtoff/kk/host,

           NPairKokkosHalffullNewtoffHost,

           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_KOKKOS_HOST)

typedef NPairHalffullKokkos<LMPDeviceType,0> NPairKokkosHalffullNewtoffDevice;

NPairStyle(halffull/newtoff/skip/kk/device,

           NPairKokkosHalffullNewtoffDevice,

           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_SKIP | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,0> NPairKokkosHalffullNewtoffHost;

NPairStyle(halffull/newtoff/skip/kk/host,

           NPairKokkosHalffullNewtoffHost,

           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_SKIP | NP_KOKKOS_HOST)

//************ Ghost **************

// Newton

typedef NPairHalffullKokkos<LMPDeviceType,1> NPairKokkosHalffullNewtonGhostDevice;

NPairStyle(halffull/newton/ghost/kk/device,

           NPairKokkosHalffullNewtonGhostDevice,

           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,1> NPairKokkosHalffullNewtonHost;

NPairStyle(halffull/newton/ghost/kk/host,

           NPairKokkosHalffullNewtonHost,

           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_HOST)

typedef NPairHalffullKokkos<LMPDeviceType,1> NPairKokkosHalffullNewtonGhostDevice;

NPairStyle(halffull/newton/skip/ghost/kk/device,

           NPairKokkosHalffullNewtonGhostDevice,

           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_GHOST | NP_SKIP | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,1> NPairKokkosHalffullNewtonHost;

NPairStyle(halffull/newton/skip/ghost/kk/host,

           NPairKokkosHalffullNewtonHost,

           NP_HALF_FULL | NP_NEWTON | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_GHOST | NP_SKIP | NP_KOKKOS_HOST)

// Newtoff

typedef NPairHalffullKokkos<LMPDeviceType,0> NPairKokkosHalffullNewtoffGhostDevice;

NPairStyle(halffull/newtoff/ghost/kk/device,

           NPairKokkosHalffullNewtoffGhostDevice,

           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,0> NPairKokkosHalffullNewtoffHost;

NPairStyle(halffull/newtoff/ghost/kk/host,

           NPairKokkosHalffullNewtoffHost,

           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_HOST)

typedef NPairHalffullKokkos<LMPDeviceType,0> NPairKokkosHalffullNewtoffGhostDevice;

NPairStyle(halffull/newtoff/skip/ghost/kk/device,

           NPairKokkosHalffullNewtoffGhostDevice,

           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_GHOST | NP_SKIP | NP_KOKKOS_DEVICE)

typedef NPairHalffullKokkos<LMPHostType,0> NPairKokkosHalffullNewtoffHost;

NPairStyle(halffull/newtoff/skip/ghost/kk/host,

           NPairKokkosHalffullNewtoffHost,

           NP_HALF_FULL | NP_NEWTOFF | NP_HALF | NP_NSQ | NP_BIN | NP_MULTI |

           NP_ORTHO | NP_TRI | NP_GHOST | NP_SKIP | NP_KOKKOS_HOST)

#else

#ifndef LMP_NPAIR_HALFFULL_KOKKOS_H

#define LMP_NPAIR_HALFFULL_KOKKOS_H

#include "npair.h"

#include "kokkos_type.h"

namespace LAMMPS_NS {

struct TagNPairHalffullCompute{};

template<class DeviceType, int NEWTON>

class NPairHalffullKokkos : public NPair {

 public:

  typedef DeviceType device_type;

  typedef ArrayTypes<DeviceType> AT;

  NPairHalffullKokkos(class LAMMPS *);

  ~NPairHalffullKokkos() {}

  void build(class NeighList *);

  KOKKOS_INLINE_FUNCTION

  void operator()(TagNPairHalffullCompute, const int&) const;

 private:

  int nlocal;

  typename AT::t_x_array_randomread x;

  typename AT::t_neighbors_2d_const d_neighbors_full;

  typename AT::t_int_1d_const d_ilist_full;

  typename AT::t_int_1d_const d_numneigh_full;

  typename AT::t_neighbors_2d d_neighbors;

  typename AT::t_int_1d d_ilist;

  typename AT::t_int_1d d_numneigh;

};

}

#endif

#endif

/* ERROR/WARNING messages:

*/

/* ----------------------------------------------------------------------

   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator

   http://lammps.sandia.gov, Sandia National Laboratories

   Steve Plimpton, sjplimp@sandia.gov

   Copyright (2003) Sandia Corporation.  Under the terms of Contract

   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains

   certain rights in this software.  This software is distributed under

   the GNU General Public License.

   See the README file in the top-level LAMMPS directory.

------------------------------------------------------------------------- */

#include "npair_skip_kokkos.h"

#include "neighbor.h"

#include "neigh_list_kokkos.h"

#include "atom_kokkos.h"

#include "atom_vec.h"

#include "molecule.h"

#include "domain.h"

#include "my_page.h"

#include "error.h"

#include "atom_masks.h"

using namespace LAMMPS_NS;

/* ---------------------------------------------------------------------- */

template<class DeviceType>

NPairSkipKokkos<DeviceType>::NPairSkipKokkos(LAMMPS *lmp) : NPair(lmp) {

  atomKK = (AtomKokkos *) atom;

  execution_space = ExecutionSpaceFromDevice<DeviceType>::space;

  d_inum = typename AT::t_int_scalar("npair_skip:inum");

}

/* ----------------------------------------------------------------------

   build skip list for subset of types from parent list

   works for half and full lists

   works for owned (non-ghost) list, also for ghost list

   iskip and ijskip flag which atom types and type pairs to skip

   if ghost, also store neighbors of ghost atoms & set inum,gnum correctly

------------------------------------------------------------------------- */

template<class DeviceType>

void NPairSkipKokkos<DeviceType>::build(NeighList *list)

{

  atomKK->sync(execution_space,TYPE_MASK);

  type = atomKK->k_type.view<DeviceType>();

  nlocal = atom->nlocal;

  NeighListKokkos<DeviceType>* k_list_skip = static_cast<NeighListKokkos<DeviceType>*>(list->listskip);

  d_ilist_skip = k_list_skip->d_ilist;

  d_numneigh_skip = k_list_skip->d_numneigh;

  d_neighbors_skip = k_list_skip->d_neighbors;

  num_skip = list->listskip->inum;

  if (list->ghost) num_skip += list->listskip->gnum;

  NeighListKokkos<DeviceType>* k_list = static_cast<NeighListKokkos<DeviceType>*>(list);

  k_list->maxneighs = k_list_skip->maxneighs; // simple, but could be made more memory efficient

  k_list->grow(atom->nmax);

  d_ilist = k_list->d_ilist;

  d_numneigh = k_list->d_numneigh;

  d_neighbors = k_list->d_neighbors;

  int ntypes = atom->ntypes;

  k_iskip = DAT::tdual_int_1d("npair_skip:iskip",ntypes+1);

  k_ijskip = DAT::tdual_int_2d("npair_skip:ijskip",ntypes+1,ntypes+1);

  d_iskip = k_iskip.view<DeviceType>();

  d_ijskip = k_ijskip.view<DeviceType>();

  for (int itype = 1; itype <= ntypes; itype++) {

    k_iskip.h_view(itype) = list->iskip[itype];

    for (int jtype = 1; jtype <= ntypes; jtype++) {

      k_ijskip.h_view(itype,jtype) = list->ijskip[itype][jtype];

    }

  }

  k_iskip.modify<LMPHostType>();

  k_ijskip.modify<LMPHostType>();

  k_iskip.sync<DeviceType>();

  k_ijskip.sync<DeviceType>();

  // loop over atoms in other list

  // skip I atom entirely if iskip is set for type[I]

  // skip I,J pair if ijskip is set for type[I],type[J]

  copymode = 1;

  Kokkos::parallel_scan(Kokkos::RangePolicy<DeviceType, TagNPairSkipCompute>(0,num_skip),*this);

  auto h_inum = Kokkos::create_mirror_view(d_inum);

  Kokkos::deep_copy(h_inum,d_inum);

  const int inum = h_inum();

  list->inum = inum;

  if (list->ghost) {

    int num = 0;

    Kokkos::parallel_reduce(Kokkos::RangePolicy<DeviceType, TagNPairSkipCountLocal>(0,inum),*this,num);

    list->inum = num;

    list->gnum = inum - num;

  }

  copymode = 0;

}

template<class DeviceType>

KOKKOS_INLINE_FUNCTION

void NPairSkipKokkos<DeviceType>::operator()(TagNPairSkipCompute, const int &ii, int &inum, const bool &final) const {

  const int i = d_ilist_skip(ii);

  const int itype = type(i);

  if (!d_iskip(itype)) {

    if (final) {

      int n = 0;

      // loop over parent non-skip list

      const int jnum = d_numneigh_skip(i);

      const AtomNeighbors neighbors_i = AtomNeighbors(&d_neighbors(i,0),d_numneigh(i),

                                                    &d_neighbors(i,1)-&d_neighbors(i,0));

      for (int jj = 0; jj < jnum; jj++) {

        const int joriginal = d_neighbors_skip(i,jj);

        int j = joriginal & NEIGHMASK;

        if (d_ijskip(itype,type(j))) continue;

        neighbors_i(n++) = joriginal;

      }

      d_numneigh(i) = n;

      d_ilist(inum) = i;

    }

    inum++;

  }

  if (final) {

    if (ii == num_skip-1)

      d_inum() = inum;

  }

}

template<class DeviceType>

KOKKOS_INLINE_FUNCTION

void NPairSkipKokkos<DeviceType>::operator()(TagNPairSkipCountLocal, const int &i, int &num) const {

  if (d_ilist[i] < nlocal) num++;

}

namespace LAMMPS_NS {

template class NPairSkipKokkos<LMPDeviceType>;

#ifdef KOKKOS_HAVE_CUDA

template class NPairSkipKokkos<LMPHostType>;

#endif

}

/* -*- c++ -*- ----------------------------------------------------------

   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator

   http://lammps.sandia.gov, Sandia National Laboratories

   Steve Plimpton, sjplimp@sandia.gov

   Copyright (2003) Sandia Corporation.  Under the terms of Contract

   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains

   certain rights in this software.  This software is distributed under

   the GNU General Public License.

   See the README file in the top-level LAMMPS directory.

------------------------------------------------------------------------- */

#ifdef NPAIR_CLASS

typedef NPairSkipKokkos<LMPDeviceType> NPairKokkosSkipDevice;

NPairStyle(skip/kk/device,

           NPairKokkosSkipDevice,

           NP_SKIP | NP_HALF | NP_FULL |

           NP_NSQ | NP_BIN | NP_MULTI |

           NP_NEWTON | NP_NEWTOFF | NP_ORTHO | NP_TRI | NP_KOKKOS_DEVICE)

typedef NPairSkipKokkos<LMPDeviceType> NPairKokkosSkipGhostDevice;

NPairStyle(skip/ghost/kk/device,

           NPairKokkosSkipGhostDevice,

           NP_SKIP | NP_HALF | NP_FULL |

           NP_NSQ | NP_BIN | NP_MULTI |

           NP_NEWTON | NP_NEWTOFF | NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_DEVICE)

typedef NPairSkipKokkos<LMPHostType> NPairKokkosSkipHost;

NPairStyle(skip/kk/host,

           NPairKokkosSkipHost,

           NP_SKIP | NP_HALF | NP_FULL |

           NP_NSQ | NP_BIN | NP_MULTI |

           NP_NEWTON | NP_NEWTOFF | NP_ORTHO | NP_TRI | NP_KOKKOS_HOST)

typedef NPairSkipKokkos<LMPHostType> NPairKokkosSkipGhostHost;

NPairStyle(skip/ghost/kk/host,

           NPairKokkosSkipGhostHost,

           NP_SKIP | NP_HALF | NP_FULL |

           NP_NSQ | NP_BIN | NP_MULTI |

           NP_NEWTON | NP_NEWTOFF | NP_ORTHO | NP_TRI | NP_GHOST | NP_KOKKOS_HOST)

#else

#ifndef LMP_NPAIR_SKIP_KOKKOS_H

#define LMP_NPAIR_SKIP_KOKKOS_H

#include "npair.h"

#include "kokkos_type.h"

namespace LAMMPS_NS {

struct TagNPairSkipCompute{};

struct TagNPairSkipCountLocal{};

template<class DeviceType>

class NPairSkipKokkos : public NPair {

 public:

  typedef DeviceType device_type;

  typedef int value_type;

  typedef ArrayTypes<DeviceType> AT;

  NPairSkipKokkos(class LAMMPS *);

  ~NPairSkipKokkos() {}

  void build(class NeighList *);

  KOKKOS_INLINE_FUNCTION

  void operator()(TagNPairSkipCompute, const int&, int&, const bool&) const;

  KOKKOS_INLINE_FUNCTION

  void operator()(TagNPairSkipCountLocal, const int&, int&) const;

 private:

  int nlocal,num_skip;

  typename AT::t_int_1d_randomread type;

  typename AT::t_int_scalar d_inum;

  typename AT::t_neighbors_2d_const d_neighbors_skip;

  typename AT::t_int_1d_const d_ilist_skip;

  typename AT::t_int_1d_const d_numneigh_skip;

  typename AT::t_neighbors_2d d_neighbors;

  typename AT::t_int_1d d_ilist;

  typename AT::t_int_1d d_numneigh;

  DAT::tdual_int_1d k_iskip;

  DAT::tdual_int_2d k_ijskip;

  typename AT::t_int_1d d_iskip;

  typename AT::t_int_2d d_ijskip;

};

}

#endif

#endif

/* ERROR/WARNING messages:

*/