Commit 4ce68cf5 authored by Steve Plimpton's avatar Steve Plimpton
Browse files

added diagnostic info for memory and balance info

parent 9de02621

src/RIGID/fix_shake.cpp

+4 −4
Original line number Diff line number Diff line
@@ -1070,7 +1070,7 @@ void FixShake::atom_owners() 
  char *buf;

  comm->rendezvous(1,nlocal,(char *) idbuf,sizeof(IDRvous),

                   0,proclist,

                   rendezvous_ids,0,buf,0,(void *) this);

                   rendezvous_ids,0,buf,0,(void *) this,1);



  memory->destroy(proclist);

  memory->sfree(idbuf);
@@ -1178,7 +1178,7 @@ void FixShake::partner_info(int *npartner, tagint **partner_tag, 
                                 0,proclist,

                                 rendezvous_partners_info,

                                 0,buf,sizeof(PartnerInfo),

                                 (void *) this);

                                 (void *) this,1);

  PartnerInfo *outbuf = (PartnerInfo *) buf;

    

  memory->destroy(proclist);
@@ -1266,7 +1266,7 @@ void FixShake::nshake_info(int *npartner, tagint **partner_tag, 
  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(NShakeInfo),

                                 0,proclist,

                                 rendezvous_nshake,0,buf,sizeof(NShakeInfo),

                                 (void *) this);

                                 (void *) this,1);

  NShakeInfo *outbuf = (NShakeInfo *) buf;

    

  memory->destroy(proclist);
@@ -1359,7 +1359,7 @@ void FixShake::shake_info(int *npartner, tagint **partner_tag, 
  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(ShakeInfo),

                                 0,proclist,

                                 rendezvous_shake,0,buf,sizeof(ShakeInfo),

                                 (void *) this);

                                 (void *) this,1);

  ShakeInfo *outbuf = (ShakeInfo *) buf;

    

  memory->destroy(proclist);

src/comm.cpp

+167 −18
Original line number Diff line number Diff line
@@ -766,18 +766,14 @@ int Comm:: 
rendezvous(int which, int n, char *inbuf, int insize,

           int inorder, int *procs,

           int (*callback)(int, char *, int &, int *&, char *&, void *),

           int outorder, char *&outbuf, int outsize, void *ptr)

           int outorder, char *&outbuf, int outsize, void *ptr, int statflag)

{

  int nout;



  if (which == 0)

    nout = rendezvous_irregular(n,inbuf,insize,inorder,procs,callback,

                                outorder,outbuf,outsize,ptr);

    return rendezvous_irregular(n,inbuf,insize,inorder,procs,callback,

                                outorder,outbuf,outsize,ptr,statflag);

  else

    nout = rendezvous_all2all(n,inbuf,insize,inorder,procs,callback,

                              outorder,outbuf,outsize,ptr);



  return nout;

    return rendezvous_all2all(n,inbuf,insize,inorder,procs,callback,

                              outorder,outbuf,outsize,ptr,statflag);

}



/* ---------------------------------------------------------------------- */
@@ -786,7 +782,7 @@ int Comm:: 
rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs,

                     int (*callback)(int, char *, int &, int *&, char *&, void *),

                     int outorder, char *&outbuf, 

                     int outsize, void *ptr)

                     int outsize, void *ptr, int statflag)

{

  // irregular comm of inbuf from caller decomp to rendezvous decomp
@@ -837,8 +833,82 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs, 
  memory->destroy(procs_rvous);

  memory->sfree(outbuf_rvous);



  // approximate memory tally

  // return number of output datums



  if (!statflag) return nout;



  // memory info for caller and rendezvous decompositions



  bigint size_in_all,size_in_max,size_in_min;

  bigint size_out_all,size_out_max,size_out_min;

  bigint size_inrvous_all,size_inrvous_max,size_inrvous_min;

  bigint size_outrvous_all,size_outrvous_max,size_outrvous_min;



  bigint size = (bigint) n*insize;

  MPI_Allreduce(&size,&size_in_all,1,MPI_LMP_BIGINT,MPI_SUM,world);

  MPI_Allreduce(&size,&size_in_max,1,MPI_LMP_BIGINT,MPI_MAX,world);

  MPI_Allreduce(&size,&size_in_min,1,MPI_LMP_BIGINT,MPI_MIN,world);



  size = (bigint) nout*outsize;

  MPI_Allreduce(&size,&size_out_all,1,MPI_LMP_BIGINT,MPI_SUM,world);

  MPI_Allreduce(&size,&size_out_max,1,MPI_LMP_BIGINT,MPI_MAX,world);

  MPI_Allreduce(&size,&size_out_min,1,MPI_LMP_BIGINT,MPI_MIN,world);



  size = (bigint) nrvous*insize;

  MPI_Allreduce(&size,&size_inrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world);

  MPI_Allreduce(&size,&size_inrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);

  MPI_Allreduce(&size,&size_inrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);



  size = (bigint) nrvous_out*insize;

  MPI_Allreduce(&size,&size_outrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world);

  MPI_Allreduce(&size,&size_outrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);

  MPI_Allreduce(&size,&size_outrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);



  int mbytes = 1024*1024;



  if (me == 0) {

    if (screen) {

      fprintf(screen,"Rendezvous balance and memory info:\n");

      fprintf(screen,"  input datum count "

              "(tot,ave,max,min): " BIGINT_FORMAT " %g "

              BIGINT_FORMAT " " BIGINT_FORMAT "\n",

              size_in_all/insize,1.0*size_in_all/nprocs/insize,

              size_in_max/insize,size_in_min/insize);

      fprintf(screen,"  input data (MB) "

              "(tot,ave,max,min): %g %g %g %g\n",

              1.0*size_in_all/mbytes,1.0*size_in_all/nprocs/mbytes,

              1.0*size_in_max/mbytes,1.0*size_in_min/mbytes);

      fprintf(screen,"  output datum count "

              "(tot,ave,max,min): " BIGINT_FORMAT " %g "

              BIGINT_FORMAT " " BIGINT_FORMAT "\n",

              size_out_all/outsize,1.0*size_out_all/nprocs/outsize,

              size_out_max/outsize,size_out_min/outsize);

      fprintf(screen,"  output data (MB) "

              "(tot,ave,max,min): %g %g %g %g\n",

              1.0*size_out_all/mbytes,1.0*size_out_all/nprocs/mbytes,

              1.0*size_out_max/mbytes,1.0*size_out_min/mbytes);

      fprintf(screen,"  input rvous datum count "

              "(tot,ave,max,min): " BIGINT_FORMAT " %g "

              BIGINT_FORMAT " " BIGINT_FORMAT "\n",

              size_inrvous_all/insize,1.0*size_inrvous_all/nprocs/insize,

              size_inrvous_max/insize,size_inrvous_min/insize);

      fprintf(screen,"  input rvous data (MB) "

              "(tot,ave,max,min): %g %g %g %g\n",

              1.0*size_inrvous_all/mbytes,1.0*size_inrvous_all/nprocs/mbytes,

              1.0*size_inrvous_max/mbytes,1.0*size_inrvous_min/mbytes);

      fprintf(screen,"  output rvous datum count "

              "(tot,ave,max,min): " BIGINT_FORMAT " %g "

              BIGINT_FORMAT " " BIGINT_FORMAT "\n",

              size_outrvous_all/outsize,1.0*size_outrvous_all/nprocs/outsize,

              size_outrvous_max/outsize,size_outrvous_min/outsize);

      fprintf(screen,"  output rvous data (MB) "

              "(tot,ave,max,min): %g %g %g %g\n",

              1.0*size_outrvous_all/mbytes,1.0*size_outrvous_all/nprocs/mbytes,

              1.0*size_outrvous_max/mbytes,1.0*size_outrvous_min/mbytes);

    }

  }



  /*

  bigint rvous_bytes = 0;

  rvous_bytes += n*insize;                                // inbuf

  rvous_bytes += nout*outsize;                            // outbuf
@@ -846,8 +916,7 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs, 
  rvous_bytes += nrvous_out*outsize;                      // outbuf_rvous

  rvous_bytes += nrvous_out*sizeof(int);                  // procs_rvous

  rvous_bytes += MAX(irregular1_bytes,irregular2_bytes);  // max of 2 comms



  // return number of output datums

  */



  return nout;

}
@@ -857,7 +926,8 @@ rendezvous_irregular(int n, char *inbuf, int insize, int inorder, int *procs, 
int Comm::

rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs,

                   int (*callback)(int, char *, int &, int *&, char *&, void *),

                   int outorder, char *&outbuf, int outsize, void *ptr)

                   int outorder, char *&outbuf, int outsize, void *ptr,

                   int statflag)

{

  int iproc;

  bigint all2all1_bytes,all2all2_bytes;
@@ -956,7 +1026,13 @@ rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs, 
                            procs_rvous,outbuf_rvous,ptr);



  if (flag != 1) memory->sfree(inbuf_rvous);  // outbuf_rvous = inbuf_vous

  if (flag == 0) return 0;    // all nout_rvous are 0, no 2nd irregular

  if (flag == 0) {

    memory->destroy(sendcount);

    memory->destroy(recvcount);

    memory->destroy(sdispls);

    memory->destroy(rdispls);

    return 0;    // all nout_rvous are 0, no 2nd irregular

  }



  // create procs and outbuf for All2all if necesary
@@ -1044,8 +1120,82 @@ rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs, 
  memory->destroy(sdispls);

  memory->destroy(rdispls);



  // approximate memory tally

  // return number of output datums



  if (!statflag) return nout;



  // memory info for caller and rendezvous decompositions



  bigint size_in_all,size_in_max,size_in_min;

  bigint size_out_all,size_out_max,size_out_min;

  bigint size_inrvous_all,size_inrvous_max,size_inrvous_min;

  bigint size_outrvous_all,size_outrvous_max,size_outrvous_min;



  bigint size = (bigint) n*insize;

  MPI_Allreduce(&size,&size_in_all,1,MPI_LMP_BIGINT,MPI_SUM,world);

  MPI_Allreduce(&size,&size_in_max,1,MPI_LMP_BIGINT,MPI_MAX,world);

  MPI_Allreduce(&size,&size_in_min,1,MPI_LMP_BIGINT,MPI_MIN,world);



  size = (bigint) nout*outsize;

  MPI_Allreduce(&size,&size_out_all,1,MPI_LMP_BIGINT,MPI_SUM,world);

  MPI_Allreduce(&size,&size_out_max,1,MPI_LMP_BIGINT,MPI_MAX,world);

  MPI_Allreduce(&size,&size_out_min,1,MPI_LMP_BIGINT,MPI_MIN,world);



  size = (bigint) nrvous*insize;

  MPI_Allreduce(&size,&size_inrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world);

  MPI_Allreduce(&size,&size_inrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);

  MPI_Allreduce(&size,&size_inrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);



  size = (bigint) nrvous_out*insize;

  MPI_Allreduce(&size,&size_outrvous_all,1,MPI_LMP_BIGINT,MPI_SUM,world);

  MPI_Allreduce(&size,&size_outrvous_max,1,MPI_LMP_BIGINT,MPI_MAX,world);

  MPI_Allreduce(&size,&size_outrvous_min,1,MPI_LMP_BIGINT,MPI_MIN,world);



  int mbytes = 1024*1024;



  if (me == 0) {

    if (screen) {

      fprintf(screen,"Rendezvous balance and memory info:\n");

      fprintf(screen,"  input datum count "

              "(tot,ave,max,min): " BIGINT_FORMAT " %g "

              BIGINT_FORMAT " " BIGINT_FORMAT "\n",

              size_in_all/insize,1.0*size_in_all/nprocs/insize,

              size_in_max/insize,size_in_min/insize);

      fprintf(screen,"  input data (MB) "

              "(tot,ave,max,min): %g %g %g %g\n",

              1.0*size_in_all/mbytes,1.0*size_in_all/nprocs/mbytes,

              1.0*size_in_max/mbytes,1.0*size_in_min/mbytes);

      fprintf(screen,"  output datum count "

              "(tot,ave,max,min): " BIGINT_FORMAT " %g "

              BIGINT_FORMAT " " BIGINT_FORMAT "\n",

              size_out_all/outsize,1.0*size_out_all/nprocs/outsize,

              size_out_max/outsize,size_out_min/outsize);

      fprintf(screen,"  output data (MB) "

              "(tot,ave,max,min): %g %g %g %g\n",

              1.0*size_out_all/mbytes,1.0*size_out_all/nprocs/mbytes,

              1.0*size_out_max/mbytes,1.0*size_out_min/mbytes);

      fprintf(screen,"  input rvous datum count "

              "(tot,ave,max,min): " BIGINT_FORMAT " %g "

              BIGINT_FORMAT " " BIGINT_FORMAT "\n",

              size_inrvous_all/insize,1.0*size_inrvous_all/nprocs/insize,

              size_inrvous_max/insize,size_inrvous_min/insize);

      fprintf(screen,"  input rvous data (MB) "

              "(tot,ave,max,min): %g %g %g %g\n",

              1.0*size_inrvous_all/mbytes,1.0*size_inrvous_all/nprocs/mbytes,

              1.0*size_inrvous_max/mbytes,1.0*size_inrvous_min/mbytes);

      fprintf(screen,"  output rvous datum count "

              "(tot,ave,max,min): " BIGINT_FORMAT " %g "

              BIGINT_FORMAT " " BIGINT_FORMAT "\n",

              size_outrvous_all/outsize,1.0*size_outrvous_all/nprocs/outsize,

              size_outrvous_max/outsize,size_outrvous_min/outsize);

      fprintf(screen,"  output rvous data (MB) "

              "(tot,ave,max,min): %g %g %g %g\n",

              1.0*size_outrvous_all/mbytes,1.0*size_outrvous_all/nprocs/mbytes,

              1.0*size_outrvous_max/mbytes,1.0*size_outrvous_min/mbytes);

    }

  }



  /*

  bigint rvous_bytes = 0;

  rvous_bytes += n*insize;                                // inbuf

  rvous_bytes += nout*outsize;                            // outbuf
@@ -1054,8 +1204,7 @@ rendezvous_all2all(int n, char *inbuf, int insize, int inorder, int *procs, 
  rvous_bytes += nrvous_out*sizeof(int);                  // procs_rvous

  rvous_bytes += 4*nprocs*sizeof(int);                    // all2all vectors

  rvous_bytes += MAX(all2all1_bytes,all2all2_bytes);      // reorder ops



  // return number of datums

  */



  return nout;

}

src/comm.h

+3 −3
Original line number Diff line number Diff line
@@ -111,7 +111,7 @@ class Comm : protected Pointers { 
            void *, void *, int self = 1);

  int rendezvous(int, int, char *, int, int, int *, 

                 int (*)(int, char *, int &, int *&, char *&, void *), 

                 int, char *&, int, void *);

                 int, char *&, int, void *, int statflag=0);



  int read_lines_from_file(FILE *, int, int, char *);

  int read_lines_from_file_universe(FILE *, int, int, char *);
@@ -149,10 +149,10 @@ class Comm : protected Pointers { 


  int rendezvous_irregular(int, char *, int, int, int *, 

                           int (*)(int, char *, int &, int *&, char *&, void *), 

                           int, char *&, int, void *);

                           int, char *&, int, void *, int);

  int rendezvous_all2all(int, char *, int, int, int *, 

                         int (*)(int, char *, int &, int *&, char *&, void *), 

                         int, char *&, int, void *);

                         int, char *&, int, void *, int);



 public:

  enum{MULTIPLE};

src/irregular.cpp

+4 −3
Original line number Diff line number Diff line
@@ -966,10 +966,11 @@ void Irregular::exchange_data(char *sendbuf, int nbytes, char *recvbuf) 
  if (nrecv_proc) MPI_Waitall(nrecv_proc,request,status);



  // approximate memory tally

  // DEBUG lines



  bigint irregular_bytes = 2*nprocs*sizeof(int);

  irregular_bytes += maxindex*sizeof(int);

  irregular_bytes += maxbuf;

  //bigint irregular_bytes = 2*nprocs*sizeof(int);

  //irregular_bytes += maxindex*sizeof(int);

  //irregular_bytes += maxbuf;

}



/* ----------------------------------------------------------------------

src/special.cpp

+6 −6
Original line number Diff line number Diff line
@@ -186,7 +186,7 @@ void Special::atom_owners() 
  

  char *buf;

  comm->rendezvous(1,nlocal,(char *) idbuf,sizeof(IDRvous),0,proclist,

                   rendezvous_ids,0,buf,0,(void *) this);

                   rendezvous_ids,0,buf,0,(void *) this,1);



  memory->destroy(proclist);

  memory->sfree(idbuf);
@@ -244,7 +244,7 @@ void Special::onetwo_build_newton() 
  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),

                                 0,proclist,

                                 rendezvous_pairs,0,buf,sizeof(PairRvous),

                                 (void *) this);

                                 (void *) this,1);

  PairRvous *outbuf = (PairRvous *) buf;

    

  memory->destroy(proclist);
@@ -379,7 +379,7 @@ void Special::onethree_build() 
  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),

                                 0,proclist,

                                 rendezvous_pairs,0,buf,sizeof(PairRvous),

                                 (void *) this);

                                 (void *) this,1);

  PairRvous *outbuf = (PairRvous *) buf;



  memory->destroy(proclist);
@@ -484,7 +484,7 @@ void Special::onefour_build() 
  int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),

                                 0,proclist,

                                 rendezvous_pairs,0,buf,sizeof(PairRvous),

                                 (void *) this);

                                 (void *) this,1);

  PairRvous *outbuf = (PairRvous *) buf;



  memory->destroy(proclist);
@@ -907,7 +907,7 @@ void Special::angle_trim() 
    int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),

                                   0,proclist,

                                   rendezvous_pairs,0,buf,sizeof(PairRvous),

                                   (void *) this);

                                   (void *) this,1);

    PairRvous *outbuf = (PairRvous *) buf;



    memory->destroy(proclist);
@@ -1126,7 +1126,7 @@ void Special::dihedral_trim() 
    int nreturn = comm->rendezvous(1,nsend,(char *) inbuf,sizeof(PairRvous),

                                   0,proclist,

                                   rendezvous_pairs,0,buf,sizeof(PairRvous),

                                   (void *) this);

                                   (void *) this,1);

    PairRvous *outbuf = (PairRvous *) buf;



    memory->destroy(proclist);

CRA Git | Maintained and supported by SUSTech CRA and CCSE