Loading deepchem/datasets/__init__.py +84 −35 Original line number Diff line number Diff line Loading @@ -168,6 +168,28 @@ class Dataset(object): """ raise NotImplementedError() def transform(self, fn, **args): """Construct a new dataset by applying a transformation to every sample in this dataset. The argument is a function that can be called as follows: >>> newx, newy, neww = fn(x, y, w) It might be called only once with the whole dataset, or multiple times with different subsets of the data. Each time it is called, it should transform the samples and return the transformed data. Parameters ---------- fn: function A function to apply to each sample in the dataset Returns ------- a newly constructed Dataset object """ raise NotImplementedError() def get_statistics(self, X_stats=True, y_stats=True): """Compute and return statistics of this dataset.""" X_means = 0.0 Loading Loading @@ -298,6 +320,29 @@ class NumpyDataset(Dataset): n_samples = self._X.shape[0] return ((self._X[i], self._y[i], self._w[i], self._ids[i]) for i in range(n_samples)) def transform(self, fn, **args): """Construct a new dataset by applying a transformation to every sample in this dataset. The argument is a function that can be called as follows: >>> newx, newy, neww = fn(x, y, w) It might be called only once with the whole dataset, or multiple times with different subsets of the data. Each time it is called, it should transform the samples and return the transformed data. Parameters ---------- fn: function A function to apply to each sample in the dataset Returns ------- a newly constructed Dataset object """ newx, newy, neww = fn(self._X, self._y, self._w) return NumpyDataset(newx, newy, neww, self._ids[:], self.verbosity) class DiskDataset(Dataset): """ Loading Loading @@ -560,6 +605,45 @@ class DiskDataset(Dataset): yield (X_shard[i], y_shard[i], w_shard[i], ids_shard[i]) return iterate(self) def transform(self, fn, **args): """Construct a new dataset by applying a transformation to every sample in this dataset. The argument is a function that can be called as follows: >>> newx, newy, neww = fn(x, y, w) It might be called only once with the whole dataset, or multiple times with different subsets of the data. Each time it is called, it should transform the samples and return the transformed data. Parameters ---------- fn: function A function to apply to each sample in the dataset out_dir: string The directory to save the new dataset in. If this is omitted, a temporary directory is created automatically Returns ------- a newly constructed Dataset object """ if 'out_dir' in args: out_dir = args['out_dir'] else: out_dir = tempfile.mkdtemp() tasks = self.get_task_names() metadata_rows = [] for shard_num, row in self.metadata_df.iterrows(): X, y, w, ids = self.get_shard(shard_num) newx, newy, neww = fn(X, y, w) basename = "dataset-%d" % shard_num metadata_rows.append(DiskDataset.write_data_to_disk( out_dir, basename, tasks, newx, newy, neww, ids)) return DiskDataset(data_dir=out_dir, metadata_rows=metadata_rows, verbosity=self.verbosity) def reshard(self, shard_size): """Reshards data to have specified shard size.""" # Create temp directory to store resharded version Loading Loading @@ -940,41 +1024,6 @@ class DiskDataset(Dataset): """Return pandas series of label stds.""" return self.metadata_df["y_stds"] def get_grad_statistics(self): """Computes and returns statistics of this dataset This function assumes that the first task of a dataset holds the energy for an input system, and that the remaining tasks holds the gradient for the system. TODO(rbharath, joegomes): It is unclear whether this should be a Dataset function. Might get refactored out. TODO(rbharath, joegomes): If y_n were an exposed part of the API, this function could be entirely written in userspace. """ if len(self) == 0: return None, None, None, None df = self.metadata_df y = [] y_n = [] for _, row in df.iterrows(): yy = load_from_disk(os.path.join(self.data_dir, row['y'])) y.append(yy) yn = load_from_disk(os.path.join(self.data_dir, row['y_n'])) y_n.append(np.array(yn)) y = np.vstack(y) y_n = np.sum(y_n, axis=0) energy = y[:,0] grad = y[:,1:] for i in range(energy.size): grad[i] *= energy[i] ydely_means = np.sum(grad, axis=0)/y_n[1:] return grad, ydely_means def compute_sums_and_nb_sample(tensor, W=None): """ Computes sums, squared sums of tensor along axis 0. Loading deepchem/datasets/tests/test_datasets.py +51 −1 Original line number Diff line number Diff line Loading @@ -289,7 +289,7 @@ class TestBasicDatasetAPI(TestDatasetAPI): np.testing.assert_array_equal(sw, w[i]) np.testing.assert_array_equal(sid, ids[i]) def test_itersamples_dist(self): def test_itersamples_disk(self): """Test that iterating over samples in a DiskDataset works.""" solubility_dataset = self.load_solubility_data() X = solubility_dataset.X Loading @@ -302,6 +302,56 @@ class TestBasicDatasetAPI(TestDatasetAPI): np.testing.assert_array_equal(sw, w[i]) np.testing.assert_array_equal(sid, ids[i]) def test_transform_numpy(self): """Test that the transform() method works for NumpyDatasets.""" num_datapoints = 100 num_features = 10 num_tasks = 10 # Generate data X = np.random.rand(num_datapoints, num_features) y = np.random.randint(2, size=(num_datapoints, num_tasks)) w = np.random.randint(2, size=(num_datapoints, num_tasks)) ids = np.array(["id"] * num_datapoints) dataset = NumpyDataset(X, y, w, ids) # Transform it def fn(x, y, w): return (2*x, 1.5*y, w) transformed = dataset.transform(fn) np.testing.assert_array_equal(X, dataset.X) np.testing.assert_array_equal(y, dataset.y) np.testing.assert_array_equal(w, dataset.w) np.testing.assert_array_equal(ids, dataset.ids) np.testing.assert_array_equal(2*X, transformed.X) np.testing.assert_array_equal(1.5*y, transformed.y) np.testing.assert_array_equal(w, transformed.w) np.testing.assert_array_equal(ids, transformed.ids) def test_transform_disk(self): """Test that the transform() method works for DiskDatasets.""" dataset = self.load_solubility_data() X = dataset.X y = dataset.y w = dataset.w ids = dataset.ids # Transform it def fn(x, y, w): return (2*x, 1.5*y, w) transformed = dataset.transform(fn) np.testing.assert_array_equal(X, dataset.X) np.testing.assert_array_equal(y, dataset.y) np.testing.assert_array_equal(w, dataset.w) np.testing.assert_array_equal(ids, dataset.ids) np.testing.assert_array_equal(2*X, transformed.X) np.testing.assert_array_equal(1.5*y, transformed.y) np.testing.assert_array_equal(w, transformed.w) np.testing.assert_array_equal(ids, transformed.ids) def test_to_numpy(self): """Test that transformation to numpy arrays is sensible.""" solubility_dataset = self.load_solubility_data() Loading deepchem/transformers/__init__.py +20 −3 Original line number Diff line number Diff line Loading @@ -28,6 +28,23 @@ def undo_grad_transforms(grad, tasks, transformers): grad = transformer.untransform_grad(grad, tasks) return grad def get_grad_statistics(dataset): """Computes and returns statistics of a dataset This function assumes that the first task of a dataset holds the energy for an input system, and that the remaining tasks holds the gradient for the system. """ if len(dataset) == 0: return None, None, None, None y = dataset.y energy = y[:,0] grad = y[:,1:] for i in range(energy.size): grad[i] *= energy[i] ydely_means = np.sum(grad, axis=0)/len(energy) return grad, ydely_means class Transformer(object): """ Abstract base class for different ML models. Loading Loading @@ -117,7 +134,7 @@ class NormalizationTransformer(Transformer): self.y_stds = y_stds self.transform_gradients = transform_gradients if self.transform_gradients: true_grad, ydely_means = dataset.get_grad_statistics() true_grad, ydely_means = get_grad_statistics(dataset) self.grad = np.reshape(true_grad, (true_grad.shape[0],-1,3)) self.ydely_means = ydely_means Loading Loading @@ -195,7 +212,7 @@ class AtomicNormalizationTransformer(Transformer): # Control for pathological case with no variance. y_stds[y_stds == 0] = 1. self.y_stds = y_stds true_grad, ydely_means = dataset.get_grad_statistics() true_grad, ydely_means = get_grad_statistics(dataset) self.grad = np.reshape(true_grad, (true_grad.shape[0],-1,3)) self.ydely_means = ydely_means Loading Loading
deepchem/datasets/__init__.py +84 −35 Original line number Diff line number Diff line Loading @@ -168,6 +168,28 @@ class Dataset(object): """ raise NotImplementedError() def transform(self, fn, **args): """Construct a new dataset by applying a transformation to every sample in this dataset. The argument is a function that can be called as follows: >>> newx, newy, neww = fn(x, y, w) It might be called only once with the whole dataset, or multiple times with different subsets of the data. Each time it is called, it should transform the samples and return the transformed data. Parameters ---------- fn: function A function to apply to each sample in the dataset Returns ------- a newly constructed Dataset object """ raise NotImplementedError() def get_statistics(self, X_stats=True, y_stats=True): """Compute and return statistics of this dataset.""" X_means = 0.0 Loading Loading @@ -298,6 +320,29 @@ class NumpyDataset(Dataset): n_samples = self._X.shape[0] return ((self._X[i], self._y[i], self._w[i], self._ids[i]) for i in range(n_samples)) def transform(self, fn, **args): """Construct a new dataset by applying a transformation to every sample in this dataset. The argument is a function that can be called as follows: >>> newx, newy, neww = fn(x, y, w) It might be called only once with the whole dataset, or multiple times with different subsets of the data. Each time it is called, it should transform the samples and return the transformed data. Parameters ---------- fn: function A function to apply to each sample in the dataset Returns ------- a newly constructed Dataset object """ newx, newy, neww = fn(self._X, self._y, self._w) return NumpyDataset(newx, newy, neww, self._ids[:], self.verbosity) class DiskDataset(Dataset): """ Loading Loading @@ -560,6 +605,45 @@ class DiskDataset(Dataset): yield (X_shard[i], y_shard[i], w_shard[i], ids_shard[i]) return iterate(self) def transform(self, fn, **args): """Construct a new dataset by applying a transformation to every sample in this dataset. The argument is a function that can be called as follows: >>> newx, newy, neww = fn(x, y, w) It might be called only once with the whole dataset, or multiple times with different subsets of the data. Each time it is called, it should transform the samples and return the transformed data. Parameters ---------- fn: function A function to apply to each sample in the dataset out_dir: string The directory to save the new dataset in. If this is omitted, a temporary directory is created automatically Returns ------- a newly constructed Dataset object """ if 'out_dir' in args: out_dir = args['out_dir'] else: out_dir = tempfile.mkdtemp() tasks = self.get_task_names() metadata_rows = [] for shard_num, row in self.metadata_df.iterrows(): X, y, w, ids = self.get_shard(shard_num) newx, newy, neww = fn(X, y, w) basename = "dataset-%d" % shard_num metadata_rows.append(DiskDataset.write_data_to_disk( out_dir, basename, tasks, newx, newy, neww, ids)) return DiskDataset(data_dir=out_dir, metadata_rows=metadata_rows, verbosity=self.verbosity) def reshard(self, shard_size): """Reshards data to have specified shard size.""" # Create temp directory to store resharded version Loading Loading @@ -940,41 +1024,6 @@ class DiskDataset(Dataset): """Return pandas series of label stds.""" return self.metadata_df["y_stds"] def get_grad_statistics(self): """Computes and returns statistics of this dataset This function assumes that the first task of a dataset holds the energy for an input system, and that the remaining tasks holds the gradient for the system. TODO(rbharath, joegomes): It is unclear whether this should be a Dataset function. Might get refactored out. TODO(rbharath, joegomes): If y_n were an exposed part of the API, this function could be entirely written in userspace. """ if len(self) == 0: return None, None, None, None df = self.metadata_df y = [] y_n = [] for _, row in df.iterrows(): yy = load_from_disk(os.path.join(self.data_dir, row['y'])) y.append(yy) yn = load_from_disk(os.path.join(self.data_dir, row['y_n'])) y_n.append(np.array(yn)) y = np.vstack(y) y_n = np.sum(y_n, axis=0) energy = y[:,0] grad = y[:,1:] for i in range(energy.size): grad[i] *= energy[i] ydely_means = np.sum(grad, axis=0)/y_n[1:] return grad, ydely_means def compute_sums_and_nb_sample(tensor, W=None): """ Computes sums, squared sums of tensor along axis 0. Loading
deepchem/datasets/tests/test_datasets.py +51 −1 Original line number Diff line number Diff line Loading @@ -289,7 +289,7 @@ class TestBasicDatasetAPI(TestDatasetAPI): np.testing.assert_array_equal(sw, w[i]) np.testing.assert_array_equal(sid, ids[i]) def test_itersamples_dist(self): def test_itersamples_disk(self): """Test that iterating over samples in a DiskDataset works.""" solubility_dataset = self.load_solubility_data() X = solubility_dataset.X Loading @@ -302,6 +302,56 @@ class TestBasicDatasetAPI(TestDatasetAPI): np.testing.assert_array_equal(sw, w[i]) np.testing.assert_array_equal(sid, ids[i]) def test_transform_numpy(self): """Test that the transform() method works for NumpyDatasets.""" num_datapoints = 100 num_features = 10 num_tasks = 10 # Generate data X = np.random.rand(num_datapoints, num_features) y = np.random.randint(2, size=(num_datapoints, num_tasks)) w = np.random.randint(2, size=(num_datapoints, num_tasks)) ids = np.array(["id"] * num_datapoints) dataset = NumpyDataset(X, y, w, ids) # Transform it def fn(x, y, w): return (2*x, 1.5*y, w) transformed = dataset.transform(fn) np.testing.assert_array_equal(X, dataset.X) np.testing.assert_array_equal(y, dataset.y) np.testing.assert_array_equal(w, dataset.w) np.testing.assert_array_equal(ids, dataset.ids) np.testing.assert_array_equal(2*X, transformed.X) np.testing.assert_array_equal(1.5*y, transformed.y) np.testing.assert_array_equal(w, transformed.w) np.testing.assert_array_equal(ids, transformed.ids) def test_transform_disk(self): """Test that the transform() method works for DiskDatasets.""" dataset = self.load_solubility_data() X = dataset.X y = dataset.y w = dataset.w ids = dataset.ids # Transform it def fn(x, y, w): return (2*x, 1.5*y, w) transformed = dataset.transform(fn) np.testing.assert_array_equal(X, dataset.X) np.testing.assert_array_equal(y, dataset.y) np.testing.assert_array_equal(w, dataset.w) np.testing.assert_array_equal(ids, dataset.ids) np.testing.assert_array_equal(2*X, transformed.X) np.testing.assert_array_equal(1.5*y, transformed.y) np.testing.assert_array_equal(w, transformed.w) np.testing.assert_array_equal(ids, transformed.ids) def test_to_numpy(self): """Test that transformation to numpy arrays is sensible.""" solubility_dataset = self.load_solubility_data() Loading
deepchem/transformers/__init__.py +20 −3 Original line number Diff line number Diff line Loading @@ -28,6 +28,23 @@ def undo_grad_transforms(grad, tasks, transformers): grad = transformer.untransform_grad(grad, tasks) return grad def get_grad_statistics(dataset): """Computes and returns statistics of a dataset This function assumes that the first task of a dataset holds the energy for an input system, and that the remaining tasks holds the gradient for the system. """ if len(dataset) == 0: return None, None, None, None y = dataset.y energy = y[:,0] grad = y[:,1:] for i in range(energy.size): grad[i] *= energy[i] ydely_means = np.sum(grad, axis=0)/len(energy) return grad, ydely_means class Transformer(object): """ Abstract base class for different ML models. Loading Loading @@ -117,7 +134,7 @@ class NormalizationTransformer(Transformer): self.y_stds = y_stds self.transform_gradients = transform_gradients if self.transform_gradients: true_grad, ydely_means = dataset.get_grad_statistics() true_grad, ydely_means = get_grad_statistics(dataset) self.grad = np.reshape(true_grad, (true_grad.shape[0],-1,3)) self.ydely_means = ydely_means Loading Loading @@ -195,7 +212,7 @@ class AtomicNormalizationTransformer(Transformer): # Control for pathological case with no variance. y_stds[y_stds == 0] = 1. self.y_stds = y_stds true_grad, ydely_means = dataset.get_grad_statistics() true_grad, ydely_means = get_grad_statistics(dataset) self.grad = np.reshape(true_grad, (true_grad.shape[0],-1,3)) self.ydely_means = ydely_means Loading
