Commit 90f9ad3d authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Adding multiple rounds of reshard shuffle

parent 619b5d05
Loading
Loading
Loading
Loading
+11 −10
Original line number Diff line number Diff line
@@ -327,11 +327,12 @@ class Dataset(object):
                   metadata_rows=metadata_rows,
                   verbosity=self.verbosity)

  def reshard_shuffle(self, reshard_size=10):
  def reshard_shuffle(self, reshard_size=10, num_reshards=3):
    """Shuffles by resharding, shuffling shards, undoing resharding."""
    #########################################################  TIMING
    time1 = time.time()
    #########################################################  TIMING
    for i in range(num_reshards):
      orig_shard_size = self.get_shard_size()
      log("Resharding to shard-size %d." % reshard_size, self.verbosity)
      self.reshard(shard_size=reshard_size)