Refactoring continues! (f5cfcea4) · Commits · 钟慕尧 / deepchem

deepchem/datasets/datasets.py

+6 −3

Original line number	Diff line number	Diff line
		@@ -896,7 +896,7 @@ class DiskDataset(Dataset):

		# TODO(rbharath): This change for general object types seems a little
		# kludgey. Is there a more principled approach to support general objects?
		def select(self, select_dir, indices, compute_feature_statistics=False):
		def select(self, indices, select_dir=None, compute_feature_statistics=False):
		"""Creates a new dataset from a selection of indices from self.

		Parameters
		@@ -909,8 +909,11 @@ class DiskDataset(Dataset):
		Whether or not to compute moments of features. Only meaningful if features
		are np.ndarrays. Not meaningful for other featurizations.
		"""
		if select_dir is not None:
		if not os.path.exists(select_dir):
		os.makedirs(select_dir)
		else:
		select_dir = tempfile.mkdtemp()
		# Handle edge case with empty indices
		if not len(indices):
		return DiskDataset(

deepchem/datasets/tests/init.py

+38 −39

Original line number	Diff line number	Diff line
		@@ -14,33 +14,48 @@ import tempfile
		import os
		import shutil
		import numpy as np
		from deepchem.datasets import Dataset
		from deepchem.featurizers.featurize import DataLoader
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.featurizers import UserDefinedFeaturizer
		from deepchem.transformers import NormalizationTransformer
		from deepchem.models.tests import TestAPI
		import deepchem as dc
		#from deepchem.datasets import Dataset
		#from deepchem.featurizers.featurize import DataLoader
		#from deepchem.featurizers.fingerprints import CircularFingerprint
		#from deepchem.featurizers import UserDefinedFeaturizer
		#from deepchem.transformers import NormalizationTransformer
		#from deepchem.models.tests import TestAPI

		class TestDatasetAPI(TestAPI):
		"""
		Shared API for testing with dataset objects.
		"""

		def load_solubility_data(self):
		"""Loads solubility data from example.csv"""
		if os.path.exists(self.data_dir):
		shutil.rmtree(self.data_dir)
		featurizer = CircularFingerprint(size=1024)
		def load_solubility_data():
		"""Loads solubility dataset"""
		current_dir = os.path.dirname(os.path.abspath(__file__))
		featurizer = dc.featurizers.CircularFingerprint(size=1024)
		tasks = ["log-solubility"]
		task_type = "regression"
		input_file = os.path.join(self.current_dir, "../../models/tests/example.csv")
		featurizer = DataLoader(
		input_file = os.path.join(current_dir, "../../models/tests/example.csv")
		featurizer = dc.loaders.DataLoader(
		tasks=tasks,
		smiles_field=self.smiles_field,
		smiles_field="smiles",
		featurizer=featurizer,
		verbosity="low")
		return featurizer.featurize(input_file)

		def load_multitask_data():
		"""Load example multitask data."""
		current_dir = os.path.dirname(os.path.abspath(__file__))
		featurizer = dc.featurizers.CircularFingerprint(size=1024)
		tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",
		"task7", "task8", "task9", "task10", "task11", "task12",
		"task13", "task14", "task15", "task16"]
		input_file = os.path.join(
		current_dir, "../../models/tests/multitask_example.csv")
		loader = dc.loaders.DataLoader(
		tasks=tasks,
		smiles_field="smiles",
		featurizer=featurizer,
		verbosity="low")
		return loader.featurize(input_file)

		return featurizer.featurize(input_file, self.data_dir)
		class TestDatasets(unittest.TestCase):
		"""
		Shared API for testing with dataset objects.
		"""

		def load_classification_data(self):
		"""Loads classification data from example.csv"""
		@@ -58,22 +73,6 @@ class TestDatasetAPI(TestAPI):
		verbosity="low")
		return loader.featurize(input_file, self.data_dir)

		def load_multitask_data(self):
		"""Load example multitask data."""
		if os.path.exists(self.data_dir):
		shutil.rmtree(self.data_dir)
		featurizer = CircularFingerprint(size=1024)
		tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",
		"task7", "task8", "task9", "task10", "task11", "task12",
		"task13", "task14", "task15", "task16"]
		input_file = os.path.join(
		self.current_dir, "../../models/tests/multitask_example.csv")
		loader = DataLoader(
		tasks=tasks,
		smiles_field=self.smiles_field,
		featurizer=featurizer,
		verbosity="low")
		return loader.featurize(input_file, self.data_dir)

		def load_sparse_multitask_dataset(self):
		"""Load sparse tox multitask data, sample dataset."""

deepchem/datasets/tests/test_datasets.py

+51 −56

Original line number	Diff line number	Diff line
		@@ -14,17 +14,9 @@ import tempfile
		import os
		import shutil
		import numpy as np
		from deepchem.datasets import sparsify_features
		from deepchem.datasets import densify_features
		from deepchem.datasets import pad_batch
		from deepchem.datasets import pad_features
		from deepchem.datasets import DiskDataset, NumpyDataset
		from deepchem.featurizers.featurize import DataLoader
		from deepchem.featurizers.fingerprints import CircularFingerprint
		from deepchem.transformers import NormalizationTransformer
		from deepchem.datasets.tests import TestDatasetAPI

		class TestBasicDatasetAPI(TestDatasetAPI):
		import deepchem as dc

		class TestBasicDatasets(unittest.TestCase):
		"""
		Test basic top-level API for dataset objects.
		"""
		@@ -35,22 +27,22 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		num_samples = 10
		num_features = num_samples
		X = np.eye(num_samples)
		X_sparse = sparsify_features(X)
		X_reconstructed = densify_features(X_sparse, num_features)
		X_sparse = dc.datasets.sparsify_features(X)
		X_reconstructed = dc.datasets.densify_features(X_sparse, num_features)
		np.testing.assert_array_equal(X, X_reconstructed)

		# Generate random sparse features dataset
		np.random.seed(123)
		p = .05
		X = np.random.binomial(1, p, size=(num_samples, num_features))
		X_sparse = sparsify_features(X)
		X_reconstructed = densify_features(X_sparse, num_features)
		X_sparse = dc.datasets.sparsify_features(X)
		X_reconstructed = dc.datasets.densify_features(X_sparse, num_features)
		np.testing.assert_array_equal(X, X_reconstructed)

		# Test edge case with array of all zeros
		X = np.zeros((num_samples, num_features))
		X_sparse = sparsify_features(X)
		X_reconstructed = densify_features(X_sparse, num_features)
		X_sparse = dc.datasets.sparsify_features(X)
		X_reconstructed = dc.datasets.densify_features(X_sparse, num_features)
		np.testing.assert_array_equal(X, X_reconstructed)

		def test_pad_features(self):
		@@ -63,31 +55,31 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		n_samples = 29
		X_b = np.zeros((n_samples, num_features))

		X_out = pad_features(batch_size, X_b)
		X_out = dc.datasets.pad_features(batch_size, X_b)
		assert len(X_out) == batch_size

		# Test cases where n_samples < batch_size
		n_samples = 79
		X_b = np.zeros((n_samples, num_features))
		X_out = pad_features(batch_size, X_b)
		X_out = dc.datasets.pad_features(batch_size, X_b)
		assert len(X_out) == batch_size

		# Test case where n_samples == batch_size
		n_samples = 100
		X_b = np.zeros((n_samples, num_features))
		X_out = pad_features(batch_size, X_b)
		X_out = dc.datasets.pad_features(batch_size, X_b)
		assert len(X_out) == batch_size

		# Test case for object featurization.
		n_samples = 2
		X_b = np.array([{"a": 1}, {"b": 2}])
		X_out = pad_features(batch_size, X_b)
		X_out = dc.datasets.pad_features(batch_size, X_b)
		assert len(X_out) == batch_size

		# Test case for more complicated object featurization
		n_samples = 2
		X_b = np.array([(1, {"a": 1}), (2, {"b": 2})])
		X_out = pad_features(batch_size, X_b)
		X_out = dc.datasets.pad_features(batch_size, X_b)
		assert len(X_out) == batch_size

		# Test case with multidimensional data
		@@ -95,7 +87,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		num_atoms = 15
		d = 3
		X_b = np.zeros((n_samples, num_atoms, d))
		X_out = pad_features(batch_size, X_b)
		X_out = dc.datasets.pad_features(batch_size, X_b)
		assert len(X_out) == batch_size


		@@ -112,7 +104,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		w_b = np.zeros((n_samples, num_tasks))
		ids_b = np.zeros((n_samples,))

		X_out, y_out, w_out, ids_out = pad_batch(
		X_out, y_out, w_out, ids_out = dc.datasets.pad_batch(
		batch_size, X_b, y_b, w_b, ids_b)
		assert len(X_out) == len(y_out) == len(w_out) == len(ids_out) == batch_size

		@@ -123,7 +115,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		w_b = np.zeros((n_samples, num_tasks))
		ids_b = np.zeros((n_samples,))

		X_out, y_out, w_out, ids_out = pad_batch(
		X_out, y_out, w_out, ids_out = dc.datasets.pad_batch(
		batch_size, X_b, y_b, w_b, ids_b)
		assert len(X_out) == len(y_out) == len(w_out) == len(ids_out) == batch_size

		@@ -134,7 +126,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		w_b = np.zeros((n_samples, num_tasks))
		ids_b = np.zeros((n_samples,))

		X_out, y_out, w_out, ids_out = pad_batch(
		X_out, y_out, w_out, ids_out = dc.datasets.pad_batch(
		batch_size, X_b, y_b, w_b, ids_b)
		assert len(X_out) == len(y_out) == len(w_out) == len(ids_out) == batch_size

		@@ -144,7 +136,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		y_b = np.zeros((n_samples, num_tasks))
		w_b = np.zeros((n_samples, num_tasks))
		ids_b = np.zeros((n_samples,))
		X_out, y_out, w_out, ids_out = pad_batch(
		X_out, y_out, w_out, ids_out = dc.datasets.pad_batch(
		batch_size, X_b, y_b, w_b, ids_b)
		assert len(X_out) == len(y_out) == len(w_out) == len(ids_out) == batch_size

		@@ -154,7 +146,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		y_b = np.zeros((n_samples, num_tasks))
		w_b = np.zeros((n_samples, num_tasks))
		ids_b = np.zeros((n_samples,))
		X_out, y_out, w_out, ids_out = pad_batch(
		X_out, y_out, w_out, ids_out = dc.datasets.pad_batch(
		batch_size, X_b, y_b, w_b, ids_b)
		assert len(X_out) == len(y_out) == len(w_out) == len(ids_out) == batch_size

		@@ -167,16 +159,16 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		w_b = np.zeros((n_samples, num_tasks))
		ids_b = np.zeros((n_samples,))

		X_out, y_out, w_out, ids_out = pad_batch(
		X_out, y_out, w_out, ids_out = dc.datasets.pad_batch(
		batch_size, X_b, y_b, w_b, ids_b)
		assert len(X_out) == len(y_out) == len(w_out) == len(ids_out) == batch_size

		def test_get_task_names(self):
		"""Test that get_task_names returns correct task_names"""
		solubility_dataset = self.load_solubility_data()
		solubility_dataset = dc.datasets.tests.load_solubility_data()
		assert solubility_dataset.get_task_names() == ["log-solubility"]

		multitask_dataset = self.load_multitask_data()
		multitask_dataset = dc.datasets.tests.load_multitask_data()
		assert sorted(multitask_dataset.get_task_names()) == sorted(["task0",
		"task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8",
		"task9", "task10", "task11", "task12", "task13", "task14", "task15",
		@@ -184,29 +176,32 @@ class TestBasicDatasetAPI(TestDatasetAPI):

		def test_get_data_shape(self):
		"""Test that get_data_shape returns currect data shape"""
		solubility_dataset = self.load_solubility_data()
		solubility_dataset = dc.datasets.tests.load_solubility_data()
		assert solubility_dataset.get_data_shape() == (1024,)

		multitask_dataset = self.load_multitask_data()
		multitask_dataset = dc.datasets.tests.load_multitask_data()
		assert multitask_dataset.get_data_shape() == (1024,)

		def test_len(self):
		"""Test that len(dataset) works."""
		solubility_dataset = self.load_solubility_data()
		solubility_dataset = dc.datasets.tests.load_solubility_data()
		assert len(solubility_dataset) == 10

		def test_reshard(self):
		"""Test that resharding the dataset works."""
		solubility_dataset = self.load_solubility_data()
		X, y, w, ids = (solubility_dataset.X, solubility_dataset.y, solubility_dataset.w, solubility_dataset.ids)
		solubility_dataset = dc.datasets.tests.load_solubility_data()
		X, y, w, ids = (solubility_dataset.X, solubility_dataset.y,
		solubility_dataset.w, solubility_dataset.ids)
		assert solubility_dataset.get_number_shards() == 1
		solubility_dataset.reshard(shard_size=1)
		assert solubility_dataset.get_shard_size() == 1
		X_r, y_r, w_r, ids_r = (solubility_dataset.X, solubility_dataset.y, solubility_dataset.w, solubility_dataset.ids)
		X_r, y_r, w_r, ids_r = (solubility_dataset.X, solubility_dataset.y,
		solubility_dataset.w, solubility_dataset.ids)
		assert solubility_dataset.get_number_shards() == 10
		solubility_dataset.reshard(shard_size=10)
		assert solubility_dataset.get_shard_size() == 10
		X_rr, y_rr, w_rr, ids_rr = (solubility_dataset.X, solubility_dataset.y, solubility_dataset.w, solubility_dataset.ids)
		X_rr, y_rr, w_rr, ids_rr = (solubility_dataset.X, solubility_dataset.y,
		solubility_dataset.w, solubility_dataset.ids)

		# Test first resharding worked
		np.testing.assert_array_equal(X, X_r)
		@@ -229,17 +224,17 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		y = np.random.randint(2, size=(num_datapoints, num_tasks))
		w = np.ones((num_datapoints, num_tasks))
		ids = np.array(["id"] * num_datapoints)
		dataset = DiskDataset.from_numpy(self.data_dir, X, y, w, ids)
		dataset = dc.datasets.DiskDataset.from_numpy(
		tempfile.mkdtemp(), X, y, w, ids)

		select_dir = tempfile.mkdtemp()
		indices = [0, 4, 5, 8]
		select_dataset = dataset.select(select_dir, indices)
		X_sel, y_sel, w_sel, ids_sel = (select_dataset.X, select_dataset.y, select_dataset.w, select_dataset.ids)
		select_dataset = dataset.select(indices)
		X_sel, y_sel, w_sel, ids_sel = (select_dataset.X, select_dataset.y,
		select_dataset.w, select_dataset.ids)
		np.testing.assert_array_equal(X[indices], X_sel)
		np.testing.assert_array_equal(y[indices], y_sel)
		np.testing.assert_array_equal(w[indices], w_sel)
		np.testing.assert_array_equal(ids[indices], ids_sel)
		shutil.rmtree(select_dir)

		def test_get_shape(self):
		"""Test that get_shape works."""
		@@ -252,7 +247,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		w = np.random.randint(2, size=(num_datapoints, num_tasks))
		ids = np.array(["id"] * num_datapoints)

		dataset = NumpyDataset(X, y, w, ids)
		dataset = dc.datasets.NumpyDataset(X, y, w, ids)

		X_shape, y_shape, w_shape, ids_shape = dataset.get_shape()
		assert X_shape == X.shape
		@@ -262,7 +257,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):

		def test_iterbatches(self):
		"""Test that iterating over batches of data works."""
		solubility_dataset = self.load_solubility_data()
		solubility_dataset = dc.datasets.tests.load_solubility_data()
		batch_size = 2
		data_shape = solubility_dataset.get_data_shape()
		tasks = solubility_dataset.get_task_names()
		@@ -282,7 +277,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		y = np.random.randint(2, size=(num_datapoints, num_tasks))
		w = np.random.randint(2, size=(num_datapoints, num_tasks))
		ids = np.array(["id"] * num_datapoints)
		dataset = NumpyDataset(X, y, w, ids)
		dataset = dc.datasets.NumpyDataset(X, y, w, ids)
		for i, (sx, sy, sw, sid) in enumerate(dataset.itersamples()):
		np.testing.assert_array_equal(sx, X[i])
		np.testing.assert_array_equal(sy, y[i])
		@@ -291,7 +286,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):

		def test_itersamples_disk(self):
		"""Test that iterating over samples in a DiskDataset works."""
		solubility_dataset = self.load_solubility_data()
		solubility_dataset = dc.datasets.tests.load_solubility_data()
		X = solubility_dataset.X
		y = solubility_dataset.y
		w = solubility_dataset.w
		@@ -309,12 +304,11 @@ class TestBasicDatasetAPI(TestDatasetAPI):
		num_tasks = 10

		# Generate data

		X = np.random.rand(num_datapoints, num_features)
		y = np.random.randint(2, size=(num_datapoints, num_tasks))
		w = np.random.randint(2, size=(num_datapoints, num_tasks))
		ids = np.array(["id"] * num_datapoints)
		dataset = NumpyDataset(X, y, w, ids)
		dataset = dc.datasets.NumpyDataset(X, y, w, ids)

		# Transform it

		@@ -332,14 +326,13 @@ class TestBasicDatasetAPI(TestDatasetAPI):

		def test_transform_disk(self):
		"""Test that the transform() method works for DiskDatasets."""
		dataset = self.load_solubility_data()
		dataset = dc.datasets.tests.load_solubility_data()
		X = dataset.X
		y = dataset.y
		w = dataset.w
		ids = dataset.ids

		# Transform it

		def fn(x, y, w):
		return (2x, 1.5y, w)
		transformed = dataset.transform(fn)
		@@ -354,10 +347,11 @@ class TestBasicDatasetAPI(TestDatasetAPI):

		def test_to_numpy(self):
		"""Test that transformation to numpy arrays is sensible."""
		solubility_dataset = self.load_solubility_data()
		solubility_dataset = dc.datasets.tests.load_solubility_data()
		data_shape = solubility_dataset.get_data_shape()
		tasks = solubility_dataset.get_task_names()
		X, y, w, ids = (solubility_dataset.X, solubility_dataset.y, solubility_dataset.w, solubility_dataset.ids)
		X, y, w, ids = (solubility_dataset.X, solubility_dataset.y,
		solubility_dataset.w, solubility_dataset.ids)
		N_samples = len(solubility_dataset)
		N_tasks = len(tasks)

		@@ -368,7 +362,7 @@ class TestBasicDatasetAPI(TestDatasetAPI):

		def test_consistent_ordering(self):
		"""Test that ordering of labels is consistent over time."""
		solubility_dataset = self.load_solubility_data()
		solubility_dataset = dc.datasets.tests.load_solubility_data()

		ids1 = solubility_dataset.ids
		ids2 = solubility_dataset.ids
		@@ -377,8 +371,9 @@ class TestBasicDatasetAPI(TestDatasetAPI):

		def test_get_statistics(self):
		"""Test statistics computation of this dataset."""
		solubility_dataset = self.load_solubility_data()
		X, y, _, _ = (solubility_dataset.X, solubility_dataset.y, solubility_dataset.w, solubility_dataset.ids)
		solubility_dataset = dc.datasets.tests.load_solubility_data()
		X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,
		solubility_dataset.w, solubility_dataset.ids)
		X_means, y_means = np.mean(X, axis=0), np.mean(y, axis=0)
		X_stds, y_stds = np.std(X, axis=0), np.std(y, axis=0)
		comp_X_means, comp_X_stds, comp_y_means, comp_y_stds = \

deepchem/datasets/tests/test_drop.py

+8 −16

Original line number	Diff line number	Diff line
		import os
		import shutil
		import unittest
		import tempfile
		import deepchem as dc
		import numpy as np
		from deepchem.models.tests import TestAPI
		from deepchem.utils.save import load_from_disk
		from deepchem.featurizers.featurize import DataLoader
		from deepchem.datasets import Dataset
		from sklearn.ensemble import RandomForestClassifier
		from deepchem.models.sklearn_models import SklearnModel
		from deepchem.featurizers.fingerprints import CircularFingerprint

		class TestDrop(TestAPI):
		class TestDrop(unittest.TestCase):
		"""
		Test how loading of malformed compounds is handled.

		@@ -25,23 +21,19 @@ class TestDrop(TestAPI):
		len_full = 25

		current_dir = os.path.dirname(os.path.realpath(__file__))
		data_dir = os.path.join(self.base_dir, "dataset")
		model_dir = os.path.join(self.base_dir, "model")

		print("About to load emols dataset.")
		dataset_file = os.path.join(
		current_dir, "mini_emols.csv")

		# Featurize emols dataset
		print("About to featurize datasets.")
		featurizer = CircularFingerprint(size=1024)
		featurizer = dc.featurizers.CircularFingerprint(size=1024)
		emols_tasks = ['activity']

		loader = DataLoader(tasks=emols_tasks,
		smiles_field="smiles",
		featurizer=featurizer,
		verbosity=verbosity)
		dataset = loader.featurize(dataset_file, data_dir, debug=True, logging=False)
		loader = dc.loaders.DataLoader(
		tasks=emols_tasks, smiles_field="smiles",
		featurizer=featurizer, verbosity=verbosity)
		dataset = loader.featurize(dataset_file, debug=True, logging=False)

		X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids)
		print("ids.shape, X.shape, y.shape, w.shape")

deepchem/datasets/tests/test_load.py

+38 −63

File changed.

Preview size limit exceeded, changes collapsed.

Admin message