Changes

    shutil.rmtree(self.data_dir)

    shutil.move(reshard_dir, self.data_dir)

    self.metadata_df = resharded_dataset.metadata_df

    # Note that this resets the cache internally

    self.save_to_disk()

  def get_data_shape(self) -> Shape:

    return DiskDataset.from_numpy(Xs, ys, ws, ids, data_dir=data_dir)

  def shuffle_each_shard(self, shard_basenames: Optional[str] = None) -> None:

  def shuffle_each_shard(self,

                         shard_basenames: Optional[List[str]] = None) -> None:

    """Shuffles elements within each shard of the datset.

    Parameters

    ----------

    shard_basenames: Optional[str], optional (default None)

    shard_basenames: Optional[List[str]], optional (default None)

      The basenames for each shard. If this isn't specified, will assume the

      default basenames of form "shard-i" used by `create_dataset`.

      default basenames of form "shard-i" used by `create_dataset` and

      `reshard`.

    """

    tasks = self.get_task_names()

    # Shuffle the arrays corresponding to each row in metadata_df

    n_rows = len(self.metadata_df.index)

    if shard_basenames is not None:

      if len(shard_basenames) != n_rows:

        raise ValueError("shard_basenames must provide a basename for each shard in this DiskDataset.")

        raise ValueError(

            "shard_basenames must provide a basename for each shard in this DiskDataset."

        )

    else:

      shard_basenames = ["shard-%d" % shard_num for shard_num in range(n_rows)]

    for i, basename in zip(range(n_rows), shard_basenames):

      permutation = np.random.permutation(n)

      X, y, w, ids = (X[permutation], y[permutation], w[permutation],

                      ids[permutation])

      #########################

      print("ids")

      print(ids)

      print("basename")

      print(basename)

      #########################

      DiskDataset.write_data_to_disk(self.data_dir, basename, tasks, X, y, w, ids)

      DiskDataset.write_data_to_disk(self.data_dir, basename, tasks, X, y, w,

                                     ids)

    # Reset cache

    self._cached_shards = None

  def shuffle_shards(self) -> None:

    """Shuffles the order of the shards for this dataset."""

import deepchem as dc

import numpy as np

#def test_shuffle():

#  """Test that datasets can be merged."""

#  current_dir = os.path.dirname(os.path.realpath(__file__))

#  assert y_orig.shape == y_new.shape

#  assert w_orig.shape == w_new.shape

def test_sparse_shuffle():

  """Test that sparse datasets can be shuffled quickly."""

  current_dir = os.path.dirname(os.path.realpath(__file__))

  orig_len = len(dataset)

  dataset.sparse_shuffle()

  X_new, y_new, w_new, new_ids = (dataset.X, dataset.y, dataset.w,

                                  dataset.ids)

  X_new, y_new, w_new, new_ids = (dataset.X, dataset.y, dataset.w, dataset.ids)

  assert len(dataset) == orig_len

  # The shuffling should have switched up the ordering

  assert y_orig.shape == y_new.shape

  assert w_orig.shape == w_new.shape

def test_shuffle_each_shard():

  """Test that shuffle_each_shard works."""

  n_samples = 100

  dataset.shuffle_each_shard()

  X_s, y_s, w_s, ids_s = (dataset.X, dataset.y, dataset.w, dataset.ids)

  ##############

  print("ids_s")

  print(ids_s)

  ##############

  assert X_s.shape == X.shape

  assert y_s.shape == y.shape

  assert ids_s.shape == ids.shape

  assert w_s.shape == w.shape

  ##############

  print("ids")

  print(ids)

  ##############

  assert not (ids_s == ids).all()

  # The ids should now store the performed permutation. Check that the

    np.testing.assert_array_equal(w_s[i], w[ids_s[i]])

    np.testing.assert_array_equal(ids_s[i], ids[ids_s[i]])

def test_shuffle_shards():

  """Test that shuffle_shards works."""

  n_samples = 100