Loading deepchem/datasets/__init__.py +22 −3 Original line number Diff line number Diff line Loading @@ -123,10 +123,12 @@ class Dataset(object): ids = load_from_disk(row['ids']) yield (X, y, w, ids) def iterbatches(self, batch_size, epoch=1): def iterbatches(self, batch_size=None, epoch=1): """ Returns minibatches from dataset. """ if batch_size == None: batch_size = len(self) for i, (X, y, w, ids) in enumerate(self._itershards()): log("Iterating on shard-%s/epoch-%s" % (str(i+1), str(epoch+1)), self.verbosity) Loading @@ -145,6 +147,23 @@ class Dataset(object): X_batch, y_batch, w_batch, ids_batch, batch_size) yield (X_batch, y_batch, w_batch, ids_batch) def to_numpy(self): """ Transforms internal data into arrays X, y, w Creates three arrays containing all data in this object. This operation is dangerous (!) for large datasets which don't fit into memory. """ Xs, ys, ws, ids = [], [], [], [] for (X_b, y_b, w_b, ids_b) in self.iterbatches(): Xs.append(X_b) ys.append(y_b) ws.append(w_b) ids.append(np.squeeze(ids_b)) # ids_b should be 1-d. Squeeze to make sure return (np.vstack(Xs), np.vstack(ys), np.vstack(ws), np.squeeze(np.vstack(ids))) def _pad_batch(self, X_b, y_b, w_b, ids_b, batch_size): """Fix batch to have exactly batch_size elements. Loading Loading @@ -184,8 +203,8 @@ class Dataset(object): """Return pandas series of label stds.""" return self.metadata_df["y_stds"] def compute_statistics(self): """Computes statistics of this dataset""" def get_statistics(self): """Computes and returns statistics of this dataset""" df = self.metadata_df X_means, X_stds, y_means, y_stds = compute_mean_and_std(df) return X_means, X_stds, y_means, y_stds Loading deepchem/datasets/tests/test_datasets.py +28 −0 Original line number Diff line number Diff line Loading @@ -13,6 +13,7 @@ import unittest import tempfile import os import shutil import numpy as np from deepchem.datasets import Dataset from deepchem.featurizers.featurize import DataFeaturizer from deepchem.featurizers.fingerprints import CircularFingerprint Loading Loading @@ -105,3 +106,30 @@ class TestAPI(unittest.TestCase): assert y_b.shape == (batch_size,) + (len(tasks),) assert w_b.shape == (batch_size,) + (len(tasks),) assert ids_b.shape == (batch_size,) def test_to_numpy(self): """Test that transformation to numpy arrays is sensible.""" solubility_dataset = self._load_solubility_data() data_shape = solubility_dataset.get_data_shape() tasks = solubility_dataset.get_task_names() X, y, w, ids = solubility_dataset.to_numpy() N_samples = len(solubility_dataset) N_tasks = len(tasks) assert X.shape == (N_samples,) + data_shape assert y.shape == (N_samples, N_tasks) assert w.shape == (N_samples, N_tasks) assert ids.shape == (N_samples,) def test_get_statistics(self): """Test statistics computation of this dataset.""" solubility_dataset = self._load_solubility_data() X, y, _, _ = solubility_dataset.to_numpy() X_means, y_means = np.mean(X, axis=0), np.mean(y, axis=0) X_stds, y_stds = np.std(X, axis=0), np.std(y, axis=0) comp_X_means, comp_X_stds, comp_y_means, comp_y_stds = \ solubility_dataset.get_statistics() np.testing.assert_allclose(comp_X_means, X_means) np.testing.assert_allclose(comp_y_means, y_means) np.testing.assert_allclose(comp_X_stds, X_stds) np.testing.assert_allclose(comp_y_stds, y_stds) Loading
deepchem/datasets/__init__.py +22 −3 Original line number Diff line number Diff line Loading @@ -123,10 +123,12 @@ class Dataset(object): ids = load_from_disk(row['ids']) yield (X, y, w, ids) def iterbatches(self, batch_size, epoch=1): def iterbatches(self, batch_size=None, epoch=1): """ Returns minibatches from dataset. """ if batch_size == None: batch_size = len(self) for i, (X, y, w, ids) in enumerate(self._itershards()): log("Iterating on shard-%s/epoch-%s" % (str(i+1), str(epoch+1)), self.verbosity) Loading @@ -145,6 +147,23 @@ class Dataset(object): X_batch, y_batch, w_batch, ids_batch, batch_size) yield (X_batch, y_batch, w_batch, ids_batch) def to_numpy(self): """ Transforms internal data into arrays X, y, w Creates three arrays containing all data in this object. This operation is dangerous (!) for large datasets which don't fit into memory. """ Xs, ys, ws, ids = [], [], [], [] for (X_b, y_b, w_b, ids_b) in self.iterbatches(): Xs.append(X_b) ys.append(y_b) ws.append(w_b) ids.append(np.squeeze(ids_b)) # ids_b should be 1-d. Squeeze to make sure return (np.vstack(Xs), np.vstack(ys), np.vstack(ws), np.squeeze(np.vstack(ids))) def _pad_batch(self, X_b, y_b, w_b, ids_b, batch_size): """Fix batch to have exactly batch_size elements. Loading Loading @@ -184,8 +203,8 @@ class Dataset(object): """Return pandas series of label stds.""" return self.metadata_df["y_stds"] def compute_statistics(self): """Computes statistics of this dataset""" def get_statistics(self): """Computes and returns statistics of this dataset""" df = self.metadata_df X_means, X_stds, y_means, y_stds = compute_mean_and_std(df) return X_means, X_stds, y_means, y_stds Loading
deepchem/datasets/tests/test_datasets.py +28 −0 Original line number Diff line number Diff line Loading @@ -13,6 +13,7 @@ import unittest import tempfile import os import shutil import numpy as np from deepchem.datasets import Dataset from deepchem.featurizers.featurize import DataFeaturizer from deepchem.featurizers.fingerprints import CircularFingerprint Loading Loading @@ -105,3 +106,30 @@ class TestAPI(unittest.TestCase): assert y_b.shape == (batch_size,) + (len(tasks),) assert w_b.shape == (batch_size,) + (len(tasks),) assert ids_b.shape == (batch_size,) def test_to_numpy(self): """Test that transformation to numpy arrays is sensible.""" solubility_dataset = self._load_solubility_data() data_shape = solubility_dataset.get_data_shape() tasks = solubility_dataset.get_task_names() X, y, w, ids = solubility_dataset.to_numpy() N_samples = len(solubility_dataset) N_tasks = len(tasks) assert X.shape == (N_samples,) + data_shape assert y.shape == (N_samples, N_tasks) assert w.shape == (N_samples, N_tasks) assert ids.shape == (N_samples,) def test_get_statistics(self): """Test statistics computation of this dataset.""" solubility_dataset = self._load_solubility_data() X, y, _, _ = solubility_dataset.to_numpy() X_means, y_means = np.mean(X, axis=0), np.mean(y, axis=0) X_stds, y_stds = np.std(X, axis=0), np.std(y, axis=0) comp_X_means, comp_X_stds, comp_y_means, comp_y_stds = \ solubility_dataset.get_statistics() np.testing.assert_allclose(comp_X_means, X_means) np.testing.assert_allclose(comp_y_means, y_means) np.testing.assert_allclose(comp_X_stds, X_stds) np.testing.assert_allclose(comp_y_stds, y_stds)
