Merge pull request #135 from rbharath/dataset_tests (ef0ddf6e) · Commits · 钟慕尧 / deepchem

deepchem/datasets/init.py

+22 −3

Original line number	Diff line number	Diff line
		@@ -123,10 +123,12 @@ class Dataset(object):
		ids = load_from_disk(row['ids'])
		yield (X, y, w, ids)

		def iterbatches(self, batch_size, epoch=1):
		def iterbatches(self, batch_size=None, epoch=1):
		"""
		Returns minibatches from dataset.
		"""
		if batch_size == None:
		batch_size = len(self)
		for i, (X, y, w, ids) in enumerate(self._itershards()):
		log("Iterating on shard-%s/epoch-%s" % (str(i+1), str(epoch+1)),
		self.verbosity)
		@@ -145,6 +147,23 @@ class Dataset(object):
		X_batch, y_batch, w_batch, ids_batch, batch_size)
		yield (X_batch, y_batch, w_batch, ids_batch)

		def to_numpy(self):
		"""
		Transforms internal data into arrays X, y, w

		Creates three arrays containing all data in this object. This operation is
		dangerous (!) for large datasets which don't fit into memory.
		"""
		Xs, ys, ws, ids = [], [], [], []
		for (X_b, y_b, w_b, ids_b) in self.iterbatches():
		Xs.append(X_b)
		ys.append(y_b)
		ws.append(w_b)
		ids.append(np.squeeze(ids_b))
		# ids_b should be 1-d. Squeeze to make sure
		return (np.vstack(Xs), np.vstack(ys), np.vstack(ws),
		np.squeeze(np.vstack(ids)))

		def _pad_batch(self, X_b, y_b, w_b, ids_b, batch_size):
		"""Fix batch to have exactly batch_size elements.

		@@ -184,8 +203,8 @@ class Dataset(object):
		"""Return pandas series of label stds."""
		return self.metadata_df["y_stds"]

		def compute_statistics(self):
		"""Computes statistics of this dataset"""
		def get_statistics(self):
		"""Computes and returns statistics of this dataset"""
		self.update_moments()
		df = self.metadata_df
		X_means, X_stds, y_means, y_stds = compute_mean_and_std(df)

deepchem/datasets/tests/init.py

0 → 100644

+0 −0

Empty file added.

deepchem/datasets/tests/test_datasets.py

0 → 100644

+135 −0

Original line number	Diff line number	Diff line
		"""
		Tests for dataset creation
		"""
		from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals

		__author__ = "Bharath Ramsundar"
		__copyright__ = "Copyright 2016, Stanford University"
		__license__ = "LGPL"

		import unittest
		import tempfile
		import os
		import shutil
		import numpy as np
		from deepchem.datasets import Dataset
		from deepchem.featurizers.featurize import DataFeaturizer
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.transformers import NormalizationTransformer

		class TestAPI(unittest.TestCase):
		"""
		Test top-level API for ML models."
		"""
		def setUp(self):
		self.current_dir = os.path.dirname(os.path.abspath(__file__))
		self.test_data_dir = os.path.join(self.current_dir, "../../models/test")
		self.smiles_field = "smiles"
		self.feature_dir = tempfile.mkdtemp()
		self.samples_dir = tempfile.mkdtemp()
		self.data_dir = tempfile.mkdtemp()

		def tearDown(self):
		shutil.rmtree(self.feature_dir)
		shutil.rmtree(self.samples_dir)
		shutil.rmtree(self.data_dir)

		# TODO(rbharath): There should be a more natural way to create a dataset
		# object, perhaps just starting from (Xs, ys, ws)
		def _create_dataset(self, compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, tasks,
		protein_pdb_field=None, ligand_pdb_field=None,
		user_specified_features=None,
		split_field=None,
		shard_size=100):
		# Featurize input
		featurizers = compound_featurizers + complex_featurizers

		input_file = os.path.join(self.test_data_dir, input_file)
		featurizer = DataFeaturizer(tasks=tasks,
		smiles_field=self.smiles_field,
		protein_pdb_field=protein_pdb_field,
		ligand_pdb_field=ligand_pdb_field,
		compound_featurizers=compound_featurizers,
		complex_featurizers=complex_featurizers,
		user_specified_features=user_specified_features,
		split_field=split_field,
		verbosity="low")

		samples = featurizer.featurize(input_file, self.feature_dir, self.samples_dir,
		shard_size=shard_size)
		use_user_specified_features = (user_specified_features is not None)
		dataset = Dataset(data_dir=self.data_dir, samples=samples,
		featurizers=featurizers, tasks=tasks,
		use_user_specified_features=use_user_specified_features)
		return dataset

		def _load_solubility_data(self):
		"""Loads solubility data from example.csv"""
		compound_featurizers = [CircularFingerprint(size=1024)]
		complex_featurizers = []
		input_transformer_classes = []
		output_transformer_classes = [NormalizationTransformer]
		task_types = {"log-solubility": "regression"}
		input_file = "example.csv"
		return self._create_dataset(
		compound_featurizers, complex_featurizers,
		input_transformer_classes, output_transformer_classes,
		input_file, task_types.keys())

		def test_get_task_names(self):
		"""Test that get_task_names returns correct task_names"""
		solubility_dataset = self._load_solubility_data()
		assert solubility_dataset.get_task_names() == ["log-solubility"]

		def test_get_data_shape(self):
		"""Test that get_data_shape returns currect data shape"""
		solubility_dataset = self._load_solubility_data()
		assert solubility_dataset.get_data_shape() == (1024,)

		def test_len(self):
		"""Test that len(dataset) works."""
		solubility_dataset = self._load_solubility_data()
		assert len(solubility_dataset) == 10

		def test_iterbatches(self):
		"""Test that iterating over batches of data works."""
		solubility_dataset = self._load_solubility_data()
		batch_size = 2
		data_shape = solubility_dataset.get_data_shape()
		tasks = solubility_dataset.get_task_names()
		for (X_b, y_b, w_b, ids_b) in solubility_dataset.iterbatches(batch_size):
		assert X_b.shape == (batch_size,) + data_shape
		assert y_b.shape == (batch_size,) + (len(tasks),)
		assert w_b.shape == (batch_size,) + (len(tasks),)
		assert ids_b.shape == (batch_size,)

		def test_to_numpy(self):
		"""Test that transformation to numpy arrays is sensible."""
		solubility_dataset = self._load_solubility_data()
		data_shape = solubility_dataset.get_data_shape()
		tasks = solubility_dataset.get_task_names()
		X, y, w, ids = solubility_dataset.to_numpy()
		N_samples = len(solubility_dataset)
		N_tasks = len(tasks)

		assert X.shape == (N_samples,) + data_shape
		assert y.shape == (N_samples, N_tasks)
		assert w.shape == (N_samples, N_tasks)
		assert ids.shape == (N_samples,)

		def test_get_statistics(self):
		"""Test statistics computation of this dataset."""
		solubility_dataset = self._load_solubility_data()
		X, y, _, _ = solubility_dataset.to_numpy()
		X_means, y_means = np.mean(X, axis=0), np.mean(y, axis=0)
		X_stds, y_stds = np.std(X, axis=0), np.std(y, axis=0)
		comp_X_means, comp_X_stds, comp_y_means, comp_y_stds = \
		solubility_dataset.get_statistics()
		np.testing.assert_allclose(comp_X_means, X_means)
		np.testing.assert_allclose(comp_y_means, y_means)
		np.testing.assert_allclose(comp_X_stds, X_stds)
		np.testing.assert_allclose(comp_y_stds, y_stds)

deepchem/transformers/init.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -78,14 +78,14 @@ class NormalizationTransformer(Transformer):
		super(NormalizationTransformer, self).__init__(transform_X=transform_X,
		transform_y=transform_y,
		dataset=dataset)
		X_means, X_stds, y_means, y_stds = dataset.compute_statistics()
		X_means, X_stds, y_means, y_stds = dataset.get_statistics()
		self.X_means = X_means
		self.X_stds = X_stds
		self.y_means = y_means
		self.y_stds = y_stds

		def transform(self, dataset, parallel=False):
		X_means, X_stds, y_means, y_stds = dataset.compute_statistics()
		X_means, X_stds, y_means, y_stds = dataset.get_statistics()
		self.X_means = X_means
		self.X_stds = X_stds
		self.y_means = y_means

Admin message