Loading deepchem/data/datasets.py +45 −0 Original line number Diff line number Diff line Loading @@ -937,6 +937,51 @@ class DiskDataset(Dataset): time2 = time.time() log("TIMING: sparse_shuffle took %0.3f s" % (time2 - time1), self.verbose) def complete_shuffle(self, data_dir=None): """ Completely shuffle across all data, across all shards. Note: this loads all the data into ram, and can be prohibitively expensive for larger datasets. Parameters ---------- shard_size: int size of the resulting dataset's size. If None, then the first shard's shard_size will be used. Returns ------- DiskDatasset A DiskDataset with a single shard. """ all_X = [] all_y = [] all_w = [] all_ids = [] for Xs, ys, ws, ids in self.itershards(): all_X.append(Xs) if ys is not None: all_y.append(ys) if ws is not None: all_w.append(ws) all_ids.append(ids) all_X = np.concatenate(all_X) all_y = np.concatenate(all_y) all_w = np.concatenate(all_w) all_ids = np.concatenate(all_ids) perm = np.random.permutation(all_X.shape[0]) all_X = all_X[perm] all_y = all_y[perm] all_w = all_w[perm] all_ids = all_ids[perm] return DiskDataset.from_numpy( all_X, all_y, all_w, all_ids, data_dir=data_dir) def shuffle_each_shard(self): """Shuffles elements within each shard of the datset.""" tasks = self.get_task_names() Loading deepchem/data/tests/test_datasets.py +38 −0 Original line number Diff line number Diff line Loading @@ -238,6 +238,44 @@ class TestDatasets(unittest.TestCase): np.testing.assert_array_equal(w[indices], w_sel) np.testing.assert_array_equal(ids[indices], ids_sel) def test_complete_shuffle(self): shard_sizes = [1, 2, 3, 4, 5] batch_size = 10 all_Xs, all_ys, all_ws, all_ids = [], [], [], [] def shard_generator(): for sz in shard_sizes: X_b = np.random.rand(sz, 1) y_b = np.random.rand(sz, 1) w_b = np.random.rand(sz, 1) ids_b = np.random.rand(sz) all_Xs.append(X_b) all_ys.append(y_b) all_ws.append(w_b) all_ids.append(ids_b) yield X_b, y_b, w_b, ids_b dataset = dc.data.DiskDataset.create_dataset(shard_generator()) res = dataset.complete_shuffle() # approx 1/15! chance of equality np.testing.assert_equal(np.any(np.not_equal(dataset.X, res.X)), True) np.testing.assert_equal(np.any(np.not_equal(dataset.y, res.w)), True) np.testing.assert_equal(np.any(np.not_equal(dataset.w, res.y)), True) np.testing.assert_equal(np.any(np.not_equal(dataset.ids, res.ids)), True) np.testing.assert_array_equal( np.sort(dataset.X, axis=0), np.sort(res.X, axis=0)) np.testing.assert_array_equal( np.sort(dataset.y, axis=0), np.sort(res.y, axis=0)) np.testing.assert_array_equal( np.sort(dataset.w, axis=0), np.sort(res.w, axis=0)) np.testing.assert_array_equal(np.sort(dataset.ids), np.sort(res.ids)) def test_get_shape(self): """Test that get_shape works.""" num_datapoints = 100 Loading Loading
deepchem/data/datasets.py +45 −0 Original line number Diff line number Diff line Loading @@ -937,6 +937,51 @@ class DiskDataset(Dataset): time2 = time.time() log("TIMING: sparse_shuffle took %0.3f s" % (time2 - time1), self.verbose) def complete_shuffle(self, data_dir=None): """ Completely shuffle across all data, across all shards. Note: this loads all the data into ram, and can be prohibitively expensive for larger datasets. Parameters ---------- shard_size: int size of the resulting dataset's size. If None, then the first shard's shard_size will be used. Returns ------- DiskDatasset A DiskDataset with a single shard. """ all_X = [] all_y = [] all_w = [] all_ids = [] for Xs, ys, ws, ids in self.itershards(): all_X.append(Xs) if ys is not None: all_y.append(ys) if ws is not None: all_w.append(ws) all_ids.append(ids) all_X = np.concatenate(all_X) all_y = np.concatenate(all_y) all_w = np.concatenate(all_w) all_ids = np.concatenate(all_ids) perm = np.random.permutation(all_X.shape[0]) all_X = all_X[perm] all_y = all_y[perm] all_w = all_w[perm] all_ids = all_ids[perm] return DiskDataset.from_numpy( all_X, all_y, all_w, all_ids, data_dir=data_dir) def shuffle_each_shard(self): """Shuffles elements within each shard of the datset.""" tasks = self.get_task_names() Loading
deepchem/data/tests/test_datasets.py +38 −0 Original line number Diff line number Diff line Loading @@ -238,6 +238,44 @@ class TestDatasets(unittest.TestCase): np.testing.assert_array_equal(w[indices], w_sel) np.testing.assert_array_equal(ids[indices], ids_sel) def test_complete_shuffle(self): shard_sizes = [1, 2, 3, 4, 5] batch_size = 10 all_Xs, all_ys, all_ws, all_ids = [], [], [], [] def shard_generator(): for sz in shard_sizes: X_b = np.random.rand(sz, 1) y_b = np.random.rand(sz, 1) w_b = np.random.rand(sz, 1) ids_b = np.random.rand(sz) all_Xs.append(X_b) all_ys.append(y_b) all_ws.append(w_b) all_ids.append(ids_b) yield X_b, y_b, w_b, ids_b dataset = dc.data.DiskDataset.create_dataset(shard_generator()) res = dataset.complete_shuffle() # approx 1/15! chance of equality np.testing.assert_equal(np.any(np.not_equal(dataset.X, res.X)), True) np.testing.assert_equal(np.any(np.not_equal(dataset.y, res.w)), True) np.testing.assert_equal(np.any(np.not_equal(dataset.w, res.y)), True) np.testing.assert_equal(np.any(np.not_equal(dataset.ids, res.ids)), True) np.testing.assert_array_equal( np.sort(dataset.X, axis=0), np.sort(res.X, axis=0)) np.testing.assert_array_equal( np.sort(dataset.y, axis=0), np.sort(res.y, axis=0)) np.testing.assert_array_equal( np.sort(dataset.w, axis=0), np.sort(res.w, axis=0)) np.testing.assert_array_equal(np.sort(dataset.ids), np.sort(res.ids)) def test_get_shape(self): """Test that get_shape works.""" num_datapoints = 100 Loading
