Commit c6699e19 authored by Axel Kohlmeyer's avatar Axel Kohlmeyer
Browse files

rewrote balancing to use per-atom data stored via fix property/atom 

(cherry picked from commit 1da862b44077c3439b314041d64915efb186af43)
parent 25748911

src/balance.cpp

+62 −26
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include "irregular.h"

#include "domain.h"

#include "force.h"

#include "modify.h"

#include "update.h"

#include "memory.h"

#include "error.h"
@@ -42,6 +43,8 @@ enum{NONE,UNIFORM,USER}; 
enum{X,Y,Z};

enum{LAYOUT_UNIFORM,LAYOUT_NONUNIFORM,LAYOUT_TILED};    // several files



const char * const Balance::bal_id = (const char * const) "BALANCE_WEIGHTS";



/* ---------------------------------------------------------------------- */



Balance::Balance(LAMMPS *lmp) : Pointers(lmp)
@@ -91,6 +94,8 @@ Balance::~Balance() 
  for (int i; i < nimbalance; ++i)

    delete imbalance[i];

  delete [] imbalance;

  int ifix = modify->find_fix(bal_id);

  if (ifix >= 0) modify->delete_fix(bal_id);



  if (fp) fclose(fp);

}
@@ -298,21 +303,39 @@ void Balance::command(int narg, char **arg) 
  if (domain->triclinic) domain->lamda2x(atom->nlocal);



  // compute and apply imbalance weights for local atoms

  double *weight = NULL;

  int iweight = -1;

  if (nimbalance > 0) {

    int i;

    const int nlocal = atom->nlocal;

    weight = new double[nlocal];

    for (i = 0; i < nlocal; ++i)

    int dflag = 0;

    iweight = atom->find_custom(bal_id,dflag);



    // add fix property/atom, for storing weights with atoms, if needed.

    if (iweight < 0 || dflag != 1) {

      char *fixargs[4];

      char *val_id = new char[strlen(bal_id)+3];

      strcat(val_id,"d_");

      strcat(val_id,bal_id);



      fixargs[0] = (char *)bal_id;

      fixargs[1] = (char *)"all";

      fixargs[2] = (char *)"property/atom";

      fixargs[3] = val_id;



      modify->add_fix(4,fixargs);

      iweight = atom->find_custom(bal_id,dflag);

      delete[] val_id;

    }



    double * const weight = atom->dvector[iweight];

    for (int i = 0; i < atom->nlocal; ++i)

      weight[i] = 1.0;

    for (i = 0; i < nimbalance; ++i)

      imbalance[i]->compute(weight);

    for (int n = 0; n < nimbalance; ++n)

      imbalance[n]->compute(weight);

  }



  // imbinit = initial imbalance



  int maxinit;

  double imbinit = imbalance_nlocal(maxinit,weight);

  double imbinit = imbalance_nlocal(maxinit);



  // no load-balance if imbalance doesn't exceed threshhold

  // unless switching from tiled to non tiled layout, then force rebalance
@@ -371,14 +394,14 @@ void Balance::command(int narg, char **arg) 
  if (style == SHIFT) {

    comm->layout = LAYOUT_NONUNIFORM;

    shift_setup_static(bstr);

    niter = shift(weight);

    niter = shift();

  }



  // style BISECTION = recursive coordinate bisectioning



  if (style == BISECTION) {

    comm->layout = LAYOUT_TILED;

    bisection(weight,1);

    bisection(1);

  }



  // reset proc sub-domains
@@ -417,8 +440,8 @@ void Balance::command(int narg, char **arg) 
  if (nimbalance > 0) {

    int i;

    const int nlocal = atom->nlocal;

    delete[] weight;

    weight = new double[nlocal];

    double * const weight = atom->dvector[iweight];



    for (i = 0; i < nlocal; ++i)

      weight[i] = 1.0;

    for (i = 0; i < nimbalance; ++i)
@@ -428,8 +451,7 @@ void Balance::command(int narg, char **arg) 
  // imbfinal = final imbalance based on final (weighted) nlocal



  int maxfinal;

  double imbfinal = imbalance_nlocal(maxfinal,weight);

  delete[] weight;

  double imbfinal = imbalance_nlocal(maxfinal);



  if (me == 0) {

    double stop_time = MPI_Wtime();
@@ -493,15 +515,20 @@ void Balance::command(int narg, char **arg) 
   return imbalance factor = max atom per proc / ave atom per proc

------------------------------------------------------------------------- */



double Balance::imbalance_nlocal(int &maxcost, double *weight)

double Balance::imbalance_nlocal(int &maxcost)

{

  // Compute the cost function of local atoms



  double cost = 0.0;

  if (weight == NULL) {

  int dflag = 0;

  int iweight = atom->find_custom(bal_id,dflag);



  if (iweight < 0 || dflag != 1) {

    cost = atom->nlocal;

  } else {

    for (int i=0; i < atom->nlocal; ++i)

    const double * const weight = atom->dvector[iweight];

    const int nlocal = atom->nlocal;

    for (int i=0; i < nlocal; ++i)

      cost += weight[i];

  }
@@ -526,7 +553,7 @@ double Balance::imbalance_nlocal(int &maxcost, double *weight) 
   return imbalance factor = max atom per proc / ave atom per proc

------------------------------------------------------------------------- */



double Balance::imbalance_splits(int &max, double *weight)

double Balance::imbalance_splits(int &max, const double *weight)

{

  double *xsplit = comm->xsplit;

  double *ysplit = comm->ysplit;
@@ -580,7 +607,7 @@ double Balance::imbalance_splits(int &max, double *weight) 
   sortflag = flag for sorting order of received messages by proc ID

------------------------------------------------------------------------- */



int *Balance::bisection(double *weight, int sortflag)

int *Balance::bisection(int sortflag)

{

  if (!rcb) rcb = new RCB(lmp);
@@ -621,9 +648,13 @@ int *Balance::bisection(double *weight, int sortflag) 


  // invoke RCB

  // then invert() to create list of proc assignements for my atoms

  // Use compute weights for each atom, if available



  rcb->compute(dim,atom->nlocal,atom->x,weight,shrinklo,shrinkhi);

  // Use pre-computed weights for each atom, if available

  int dflag = 0;

  int iweight = atom->find_custom(bal_id,dflag);

  if (iweight < 0 || dflag != 1)

    rcb->compute(dim,atom->nlocal,atom->x,NULL,shrinklo,shrinkhi);

  else rcb->compute(dim,atom->nlocal,atom->x,atom->dvector[iweight],

                    shrinklo,shrinkhi);

  rcb->invert(sortflag);



  // reset RCB lo/hi bounding box to full simulation box as needed
@@ -739,7 +770,7 @@ void Balance::shift_setup(char *str, int nitermax_in, double thresh_in) 
   return niter = iteration count

------------------------------------------------------------------------- */



int Balance::shift(double *weight)

int Balance::shift()

{

  int i,j,k,m,np,max;

  double *split;
@@ -773,11 +804,16 @@ int Balance::shift(double *weight) 


    // intial count and sum



    int dflag = 0;

    int iweight = atom->find_custom(bal_id,dflag);

    const double * const weight =

      (iweight < 0 || dflag != 1) ? NULL : atom->dvector[iweight];

    double cost = 0.0;



    np = procgrid[bdim[idim]];

    tally(bdim[idim],np,split,weight);



    double cost = 0.0;

    if (weight == NULL) {

    if (weight) {

      cost = atom->nlocal;

    } else {

      for (int i=0; i < atom->nlocal; ++i)
@@ -918,7 +954,7 @@ int Balance::shift(double *weight) 
   use binary search to find which slice each atom is in

------------------------------------------------------------------------- */



void Balance::tally(int dim, int n, double *split, double *weight)

void Balance::tally(int dim, int n, double *split, const double *weight)

{

  double *onecost = new double[n];

  for (int i = 0; i < n; i++) onecost[i] = 0.0;

src/balance.h

+23 −21
Original line number Diff line number Diff line
@@ -33,11 +33,13 @@ class Balance : protected Pointers { 
  ~Balance();

  void command(int, char **);

  void shift_setup(char *, int, double);

  int shift(double *);

  int *bisection(double *, int sortflag = 0);

  double imbalance_nlocal(int &, double *);

  int shift();

  int *bisection(int sortflag = 0);

  double imbalance_nlocal(int &);

  void dumpout(bigint, FILE *);



  static const char * const bal_id; // name of custom atom property for weights



 private:

  int me,nprocs;
@@ -72,9 +74,9 @@ class Balance : protected Pointers { 
  FILE *fp;

  int firststep;



  double imbalance_splits(int &, double *);

  double imbalance_splits(int &, const double *);

  void shift_setup_static(char *);

  void tally(int, int, double *, double *);

  void tally(int, int, double *, const double *);

  int adjust(int, double *);

  int binary(double, int, double *);

#ifdef BALANCE_DEBUG

src/fix_balance.cpp

+59 −57
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "atom.h"

#include "comm.h"

#include "domain.h"

#include "modify.h"

#include "neighbor.h"

#include "irregular.h"

#include "force.h"
@@ -157,6 +158,35 @@ FixBalance::FixBalance(LAMMPS *lmp, int narg, char **arg) : 


  irregular = new Irregular(lmp);



  // add per atom weight property, if weighted balancing is requested



  imb_id = NULL;

  if (nimbalance > 0) {

    int dflag = 0;

    int iweight = atom->find_custom(Balance::bal_id,dflag);



    if (iweight < 0 || dflag != 1) {

      char *fixargs[4];

      

      imb_id = new char[strlen(this->id)+strlen(Balance::bal_id)+2];

      char *val_id = new char[strlen(Balance::bal_id)+3];



      strcpy(imb_id,this->id);

      strcat(imb_id,"_");

      strcat(imb_id,Balance::bal_id);

      strcpy(val_id,"d_");

      strcat(val_id,Balance::bal_id);



      fixargs[0] = imb_id;

      fixargs[1] = (char *)"all";

      fixargs[2] = (char *)"property/atom";

      fixargs[3] = val_id;



      modify->add_fix(4,fixargs);

      delete[] val_id;

    }

  }



  // output file



  if (outflag && comm->me == 0) {
@@ -168,24 +198,9 @@ FixBalance::FixBalance(LAMMPS *lmp, int narg, char **arg) : 


  if (nevery) force_reneighbor = 1;



  // compute and apply imbalance weights for local atoms

  double *weight = NULL;

  if (nimbalance > 0) {

    int i;

    const int nlocal = atom->nlocal;

    weight = new double[nlocal];

    for (i = 0; i < nlocal; ++i)

      weight[i] = 1.0;

    for (i = 0; i < nimbalance; ++i)

      imbalance[i]->compute(weight);

  }



  // compute initial outputs



  imbfinal = imbprev = balance->imbalance_nlocal(maxperproc,weight);

  itercount = 0;

  pending = 0;

  delete[] weight;

  imbfinal = imbprev = 0.0;

}



/* ---------------------------------------------------------------------- */
@@ -199,6 +214,7 @@ FixBalance::~FixBalance() 
  for (int i = 0; i < nimbalance; ++i)

    delete imbalance[i];

  delete[] imbalance;

  if (imb_id) modify->delete_fix(imb_id);

}



/* ---------------------------------------------------------------------- */
@@ -244,23 +260,23 @@ void FixBalance::setup_pre_exchange() 
  domain->reset_box();

  if (domain->triclinic) domain->lamda2x(atom->nlocal);



  // perform a rebalance if threshhold exceeded



  // compute and apply imbalance weights for local atoms

  double *weight = NULL;



  int iweight = -1;

  if (nimbalance > 0) {

    int i;

    const int nlocal = atom->nlocal;

    weight = new double[nlocal];

    for (i = 0; i < nlocal; ++i)

    int dflag = 0;

    iweight = atom->find_custom(Balance::bal_id,dflag);

    double * const weight = atom->dvector[iweight];

    for (int i = 0; i < atom->nlocal; ++i)

      weight[i] = 1.0;

    for (i = 0; i < nimbalance; ++i)

      imbalance[i]->compute(weight);

    for (int n = 0; n < nimbalance; ++n)

      imbalance[n]->compute(weight);

  }



  imbnow = balance->imbalance_nlocal(maxperproc,weight);

  if (imbnow > thresh) rebalance(weight);

  delete[] weight;

  // compute initial outputs and perform a rebalance if threshhold exceeded



  imbfinal = imbprev = imbnow = balance->imbalance_nlocal(maxperproc);

  if (imbnow > thresh) rebalance();



  // next_reneighbor = next time to force reneighboring
@@ -286,29 +302,28 @@ void FixBalance::pre_exchange() 
  if (domain->triclinic) domain->lamda2x(atom->nlocal);



  // compute and apply imbalance weights for local atoms

  double *weight = NULL;



  int iweight = -1;

  if (nimbalance > 0) {

    int i;

    const int nlocal = atom->nlocal;

    weight = new double[nlocal];

    for (i = 0; i < nlocal; ++i)

    int dflag = 0;

    iweight = atom->find_custom(Balance::bal_id,dflag);

    double * const weight = atom->dvector[iweight];

    for (int i = 0; i < atom->nlocal; ++i)

      weight[i] = 1.0;

    for (i = 0; i < nimbalance; ++i)

      imbalance[i]->compute(weight);

    for (int n = 0; n < nimbalance; ++n)

      imbalance[n]->compute(weight);

  }



  // return if imbalance < threshhold



  imbnow = balance->imbalance_nlocal(maxperproc,weight);

  imbnow = balance->imbalance_nlocal(maxperproc);



  if (imbnow <= thresh) {

    if (nevery) next_reneighbor = (update->ntimestep/nevery)*nevery + nevery;

    delete[] weight;

    return;

  }



  rebalance(weight);

  delete[] weight;

  rebalance();



  // next timestep to rebalance
@@ -324,28 +339,15 @@ void FixBalance::pre_neighbor() 
{

  if (!pending) return;



  // compute and apply imbalance weights for local atoms

  double *weight = NULL;

  if (nimbalance > 0) {

    int i;

    const int nlocal = atom->nlocal;

    weight = new double[nlocal];

    for (i = 0; i < nlocal; ++i)

      weight[i] = 1.0;

    for (i = 0; i < nimbalance; ++i)

      imbalance[i]->compute(weight);

  }



  imbfinal = balance->imbalance_nlocal(maxperproc,weight);

  imbfinal = balance->imbalance_nlocal(maxperproc);

  pending = 0;

  delete[] weight;

}



/* ----------------------------------------------------------------------

   perform dynamic load balancing

------------------------------------------------------------------------- */



void FixBalance::rebalance(double *weight)

void FixBalance::rebalance()

{

  imbprev = imbnow;
@@ -353,10 +355,10 @@ void FixBalance::rebalance(double *weight) 


  int *sendproc;

  if (lbstyle == SHIFT) {

    itercount = balance->shift(weight);

    itercount = balance->shift();

    comm->layout = LAYOUT_NONUNIFORM;

  } else if (lbstyle == BISECTION) {

    sendproc = balance->bisection(weight);

    sendproc = balance->bisection();

    comm->layout = LAYOUT_TILED;

  }

src/fix_balance.h

+3 −2
Original line number Diff line number Diff line
@@ -55,11 +55,12 @@ class FixBalance : public Fix { 


  int nimbalance;               // number of imbalance weight computes

  class Imbalance **imbalance;  // list of imbalance compute classes

  char *imb_id;                 // id of property/atom fix for storing weights

  

  class Balance *balance;

  class Irregular *irregular;



  void rebalance(double *);

  void rebalance();

};



}

src/fix_controller.h

100755 → 100644
+0 −0

File mode changed from 100755 to 100644.

View file

CRA Git | Maintained and supported by SUSTech CRA and CCSE