Fixing failing tests (d4227698) · Commits · 钟慕尧 / deepchem

deepchem/data/tests/test_datasets.py

+717 −690

Original line number	Diff line number	Diff line
		@@ -55,16 +55,34 @@ def load_multitask_data():

		class TestTransformer(dc.trans.Transformer):

		def transform_array(self, X, y, w):
		return (2 * X, 1.5 * y, w)
		def transform_array(self, X, y, w, ids):
		return (2 * X, 1.5 * y, w, ids)


		class TestDatasets(test_util.TensorFlowTestCase):
		"""
		Test basic top-level API for dataset objects.
		"""
		def test_transform_disk():
		"""Test that the transform() method works for DiskDatasets."""
		dataset = load_solubility_data()
		X = dataset.X
		y = dataset.y
		w = dataset.w
		ids = dataset.ids

		# Transform it

		def test_sparsify_and_densify(self):
		transformer = TestTransformer(transform_X=True, transform_y=True)
		for parallel in (True, False):
		transformed = dataset.transform(transformer, parallel=parallel)
		np.testing.assert_array_equal(X, dataset.X)
		np.testing.assert_array_equal(y, dataset.y)
		np.testing.assert_array_equal(w, dataset.w)
		np.testing.assert_array_equal(ids, dataset.ids)
		np.testing.assert_array_equal(2 * X, transformed.X)
		np.testing.assert_array_equal(1.5 * y, transformed.y)
		np.testing.assert_array_equal(w, transformed.w)
		np.testing.assert_array_equal(ids, transformed.ids)


		def test_sparsify_and_densify():
		"""Test that sparsify and densify work as inverses."""
		# Test on identity matrix
		num_samples = 10
		@@ -88,7 +106,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		X_reconstructed = dc.data.densify_features(X_sparse, num_features)
		np.testing.assert_array_equal(X, X_reconstructed)

		def test_pad_features(self):

		def test_pad_features():
		"""Test that pad_features pads features correctly."""
		batch_size = 100
		num_features = 10
		@@ -133,7 +152,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		X_out = dc.data.pad_features(batch_size, X_b)
		assert len(X_out) == batch_size

		def test_pad_batches(self):

		def test_pad_batches():
		"""Test that pad_batch pads batches correctly."""
		batch_size = 100
		num_features = 10
		@@ -205,7 +225,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		ids_b)
		assert len(X_out) == len(y_out) == len(w_out) == len(ids_out) == batch_size

		def test_get_task_names(self):

		def test_get_task_names():
		"""Test that get_task_names returns correct task_names"""
		solubility_dataset = load_solubility_data()
		assert solubility_dataset.get_task_names() == ["log-solubility"]
		@@ -217,7 +238,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		"task15", "task16"
		])

		def test_get_data_shape(self):

		def test_get_data_shape():
		"""Test that get_data_shape returns currect data shape"""
		solubility_dataset = load_solubility_data()
		assert solubility_dataset.get_data_shape() == (1024,)
		@@ -225,12 +247,14 @@ class TestDatasets(test_util.TensorFlowTestCase):
		multitask_dataset = load_multitask_data()
		assert multitask_dataset.get_data_shape() == (1024,)

		def test_len(self):

		def test_len():
		"""Test that len(dataset) works."""
		solubility_dataset = load_solubility_data()
		assert len(solubility_dataset) == 10

		def test_reshard(self):

		def test_reshard():
		"""Test that resharding the dataset works."""
		solubility_dataset = load_solubility_data()
		X, y, w, ids = (solubility_dataset.X, solubility_dataset.y,
		@@ -258,7 +282,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(w, w_rr)
		np.testing.assert_array_equal(ids, ids_rr)

		def test_select(self):

		def test_select():
		"""Test that dataset select works."""
		num_datapoints = 10
		num_features = 10
		@@ -278,7 +303,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(w[indices], w_sel)
		np.testing.assert_array_equal(ids[indices], ids_sel)

		def test_complete_shuffle(self):

		def test_complete_shuffle():
		shard_sizes = [1, 2, 3, 4, 5]
		batch_size = 10

		@@ -316,7 +342,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.sort(dataset.w, axis=0), np.sort(res.w, axis=0))
		np.testing.assert_array_equal(np.sort(dataset.ids), np.sort(res.ids))

		def test_get_shape(self):

		def test_get_shape():
		"""Test that get_shape works."""
		num_datapoints = 100
		num_features = 10
		@@ -335,7 +362,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		assert w_shape == w.shape
		assert ids_shape == ids.shape

		def test_iterbatches(self):

		def test_iterbatches():
		"""Test that iterating over batches of data works."""
		solubility_dataset = load_solubility_data()
		batch_size = 2
		@@ -347,7 +375,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		assert w_b.shape == (batch_size,) + (len(tasks),)
		assert ids_b.shape == (batch_size,)

		def test_itersamples_numpy(self):

		def test_itersamples_numpy():
		"""Test that iterating over samples in a NumpyDataset works."""
		num_datapoints = 100
		num_features = 10
		@@ -364,7 +393,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(sw, w[i])
		np.testing.assert_array_equal(sid, ids[i])

		def test_itersamples_disk(self):

		def test_itersamples_disk():
		"""Test that iterating over samples in a DiskDataset works."""
		solubility_dataset = load_solubility_data()
		X = solubility_dataset.X
		@@ -377,7 +407,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(sw, w[i])
		np.testing.assert_array_equal(sid, ids[i])

		def test_transform_numpy(self):

		def test_transform_numpy():
		"""Test that the transform() method works for NumpyDatasets."""
		num_datapoints = 100
		num_features = 10
		@@ -403,29 +434,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(w, transformed.w)
		np.testing.assert_array_equal(ids, transformed.ids)

		def test_transform_disk(self):
		"""Test that the transform() method works for DiskDatasets."""
		dataset = load_solubility_data()
		X = dataset.X
		y = dataset.y
		w = dataset.w
		ids = dataset.ids

		# Transform it

		transformer = TestTransformer(transform_X=True, transform_y=True)
		for parallel in (True, False):
		transformed = dataset.transform(transformer, parallel=parallel)
		np.testing.assert_array_equal(X, dataset.X)
		np.testing.assert_array_equal(y, dataset.y)
		np.testing.assert_array_equal(w, dataset.w)
		np.testing.assert_array_equal(ids, dataset.ids)
		np.testing.assert_array_equal(2 * X, transformed.X)
		np.testing.assert_array_equal(1.5 * y, transformed.y)
		np.testing.assert_array_equal(w, transformed.w)
		np.testing.assert_array_equal(ids, transformed.ids)

		def test_to_numpy(self):
		def test_to_numpy():
		"""Test that transformation to numpy arrays is sensible."""
		solubility_dataset = load_solubility_data()
		data_shape = solubility_dataset.get_data_shape()
		@@ -440,7 +450,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		assert w.shape == (N_samples, N_tasks)
		assert ids.shape == (N_samples,)

		def test_consistent_ordering(self):

		def test_consistent_ordering():
		"""Test that ordering of labels is consistent over time."""
		solubility_dataset = load_solubility_data()

		@@ -449,7 +460,8 @@ class TestDatasets(test_util.TensorFlowTestCase):

		assert np.array_equal(ids1, ids2)

		def test_get_statistics(self):

		def test_get_statistics():
		"""Test statistics computation of this dataset."""
		solubility_dataset = load_solubility_data()
		X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,
		@@ -463,7 +475,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_allclose(comp_X_stds, X_stds)
		np.testing.assert_allclose(comp_y_stds, y_stds)

		def test_disk_iterate_batch_size(self):

		def test_disk_iterate_batch_size():
		solubility_dataset = load_solubility_data()
		X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,
		solubility_dataset.w, solubility_dataset.ids)
		@@ -471,9 +484,10 @@ class TestDatasets(test_util.TensorFlowTestCase):
		for X, y, _, _ in solubility_dataset.iterbatches(
		3, epochs=2, pad_batches=False, deterministic=True):
		batch_sizes.append(len(X))
		self.assertEqual([3, 3, 3, 1, 3, 3, 3, 1], batch_sizes)
		assert [3, 3, 3, 1, 3, 3, 3, 1] == batch_sizes

		def test_disk_pad_batches(self):

		def test_disk_pad_batches():
		shard_sizes = [21, 11, 41, 21, 51]
		batch_size = 10

		@@ -531,7 +545,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(all_ws, test_ws[:total_size, :])
		np.testing.assert_array_equal(all_ids, test_ids[:total_size])

		def test_disk_iterate_y_w_None(self):

		def test_disk_iterate_y_w_None():
		shard_sizes = [21, 11, 41, 21, 51]
		batch_size = 10

		@@ -575,7 +590,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(all_Xs, test_Xs[:total_size, :])
		np.testing.assert_array_equal(all_ids, test_ids[:total_size])

		def test_disk_iterate_batch(self):

		def test_disk_iterate_batch():

		all_batch_sizes = [None, 32, 17, 11]
		all_shard_sizes = [[7, 3, 12, 4, 5], [1, 1, 1, 1, 1], [31, 31, 31, 31, 31],
		@@ -688,19 +704,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(
		np.sort(all_ids, axis=0), np.sort(test_ids, axis=0))

		def test_numpy_iterate_batch_size(self):
		solubility_dataset = load_solubility_data()
		X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,
		solubility_dataset.w, solubility_dataset.ids)
		solubility_dataset = dc.data.NumpyDataset.from_DiskDataset(
		solubility_dataset)
		batch_sizes = []
		for X, y, _, _ in solubility_dataset.iterbatches(
		3, epochs=2, pad_batches=False, deterministic=True):
		batch_sizes.append(len(X))
		self.assertEqual([3, 3, 3, 1, 3, 3, 3, 1], batch_sizes)

		def test_merge(self):
		def test_merge():
		"""Test that dataset merge works."""
		num_datapoints = 10
		num_features = 10
		@@ -722,7 +727,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		assert new_data.y.shape == (num_datapoints * num_datasets, num_tasks)
		assert len(new_data.tasks) == len(datasets[0].tasks)

		def test_make_tf_dataset(self):

		def test_make_tf_dataset():
		"""Test creating a Tensorflow Iterator from a Dataset."""
		X = np.random.random((100, 5))
		y = np.random.random((100, 1))
		@@ -736,7 +742,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(np.ones((10, 1)), batch_w)
		assert i == 19

		def _validate_pytorch_dataset(self, dataset):

		def _validate_pytorch_dataset(dataset):
		X = dataset.X
		y = dataset.y
		w = dataset.w
		@@ -780,32 +787,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		id_count[iter_id[0]] += 1
		assert all(id_count[id] == 2 for id in ids)

		@unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')
		def test_make_pytorch_dataset_from_numpy(self):
		"""Test creating a PyTorch Dataset from a NumpyDataset."""
		X = np.random.random((100, 5))
		y = np.random.random((100, 1))
		ids = [str(i) for i in range(100)]
		dataset = dc.data.NumpyDataset(X, y, ids=ids)
		self._validate_pytorch_dataset(dataset)

		@unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')
		def test_make_pytorch_dataset_from_images(self):
		"""Test creating a PyTorch Dataset from an ImageDataset."""
		path = os.path.join(os.path.dirname(__file__), 'images')
		files = [os.path.join(path, f) for f in os.listdir(path)]
		y = np.random.random((10, 1))
		ids = [str(i) for i in range(len(files))]
		dataset = dc.data.ImageDataset(files, y, ids=ids)
		self._validate_pytorch_dataset(dataset)

		@unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')
		def test_make_pytorch_dataset_from_disk(self):
		"""Test creating a PyTorch Dataset from a DiskDataset."""
		dataset = load_solubility_data()
		self._validate_pytorch_dataset(dataset)

		def test_dataframe(self):
		def test_dataframe():
		"""Test converting between Datasets and DataFrames."""
		dataset = load_solubility_data()

		@@ -827,7 +810,8 @@ class TestDatasets(test_util.TensorFlowTestCase):
		np.testing.assert_array_equal(
		np.stack([dataset.y[:, 0], dataset.X[:, 0]], axis=1), dataset3.w)

		def test_to_str(self):

		def test_to_str():
		"""Tests to string representation of Dataset."""
		dataset = dc.data.NumpyDataset(
		X=np.random.rand(5, 3), y=np.random.rand(5,), ids=np.arange(5))
		@@ -853,3 +837,46 @@ class TestDatasets(test_util.TensorFlowTestCase):
		X=np.random.rand(50, 3), y=np.random.rand(50,), ids=np.arange(50))
		ref_str = '<NumpyDataset X.shape: (50, 3), y.shape: (50,), w.shape: (50,), task_names: [0]>'
		assert str(dataset) == ref_str


		class TestDatasets(test_util.TensorFlowTestCase):
		"""
		Test basic top-level API for dataset objects.
		"""

		def test_numpy_iterate_batch_size(self):
		solubility_dataset = load_solubility_data()
		X, y, _, _ = (solubility_dataset.X, solubility_dataset.y,
		solubility_dataset.w, solubility_dataset.ids)
		solubility_dataset = dc.data.NumpyDataset.from_DiskDataset(
		solubility_dataset)
		batch_sizes = []
		for X, y, _, _ in solubility_dataset.iterbatches(
		3, epochs=2, pad_batches=False, deterministic=True):
		batch_sizes.append(len(X))
		self.assertEqual([3, 3, 3, 1, 3, 3, 3, 1], batch_sizes)

		@unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')
		def test_make_pytorch_dataset_from_numpy(self):
		"""Test creating a PyTorch Dataset from a NumpyDataset."""
		X = np.random.random((100, 5))
		y = np.random.random((100, 1))
		ids = [str(i) for i in range(100)]
		dataset = dc.data.NumpyDataset(X, y, ids=ids)
		_validate_pytorch_dataset(dataset)

		@unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')
		def test_make_pytorch_dataset_from_images(self):
		"""Test creating a PyTorch Dataset from an ImageDataset."""
		path = os.path.join(os.path.dirname(__file__), 'images')
		files = [os.path.join(path, f) for f in os.listdir(path)]
		y = np.random.random((10, 1))
		ids = [str(i) for i in range(len(files))]
		dataset = dc.data.ImageDataset(files, y, ids=ids)
		_validate_pytorch_dataset(dataset)

		@unittest.skipIf(PYTORCH_IMPORT_FAILED, 'PyTorch is not installed')
		def test_make_pytorch_dataset_from_disk(self):
		"""Test creating a PyTorch Dataset from a DiskDataset."""
		dataset = load_solubility_data()
		_validate_pytorch_dataset(dataset)

deepchem/trans/tests/test_transformers.py→deepchem/trans/tests/test_data_transforms.py

+2 −1

Original line number	Diff line number	Diff line
		@@ -21,7 +21,8 @@ class TestDataTransforms(unittest.TestCase):
		"""
		init to load the MNIST data for DataTransforms Tests
		"""
		super(TestTransformers, self).setUp()
		import tensorflow as tf
		super(TestDataTransforms, self).setUp()
		self.current_dir = os.path.dirname(os.path.abspath(__file__))
		(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
		train = dc.data.NumpyDataset(x_train, y_train)

deepchem/trans/transformers.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -1380,9 +1380,9 @@ class CoulombFitTransformer(Transformer):
		X = self.normalize(self.expand(self.realize(X)))
		return X

		def transform_array(self, X, y, w):
		def transform_array(self, X, y, w, ids):
		X = self.X_transform(X)
		return (X, y, w)
		return (X, y, w, ids)

		def untransform(self, z):
		raise NotImplementedError(

Admin message