chnages (b04449fd) · Commits · 钟慕尧 / deepchem

deepchem/metrics/init.py

+19 −9

Original line number	Diff line number	Diff line
		@@ -154,6 +154,13 @@ def normalize_labels_shape(y, mode=None, n_tasks=None, n_classes=None):
		raise ValueError("n_classes must be specified")
		if not isinstance(y, np.ndarray):
		raise ValueError("y must be a np.ndarray")
		# Handle n_classes/n_task shape ambiguity
		if mode == "classification" and len(y.shape) == 2:
		if n_classes == y.shape[1] and n_tasks != 1 and n_classes != n_tasks:
		raise ValueError("Shape of input doesn't match expected n_tasks=1")
		elif n_classes == y.shape[1] and n_tasks == 1:
		# Add in task dimension
		y = np.expand_dims(y, 1)
		if len(y.shape) == 1 and n_tasks != 1:
		raise ValueError("n_tasks must equal 1 for a 1D set of labels.")
		if (len(y.shape) == 2 or len(y.shape) == 3) and n_tasks != y.shape[1]:
		@@ -169,10 +176,12 @@ def normalize_labels_shape(y, mode=None, n_tasks=None, n_classes=None):
		elif len(y.shape) == 2:
		y_out = y
		elif len(y.shape) == 3:
		# If 3D and last dimension isn't 1, assume this is one-hot encoded and return as-is.
		if y.shape[-1] != 1:
		raise ValueError(
		"y must be a float scalar or a ndarray of shape `(N,)` or `(N, n_tasks)` or `(N, n_tasks, 1)`."
		)
		return y
		#raise ValueError(
		# "y must be a float scalar or a ndarray of shape `(N,)` or `(N, n_tasks)` or `(N, n_tasks, 1)`."
		#)
		y_out = np.squeeze(y, axis=-1)
		# Handle classification. We need to convert labels into one-hot
		# representation.
		@@ -236,8 +245,9 @@ def normalize_prediction_shape(y, mode=None, n_tasks=None, n_classes=None):
		raise ValueError("y must be a np.ndarray")
		# Handle n_classes/n_task shape ambiguity
		if mode == "classification" and len(y.shape) == 2:
		if n_classes == y.shape[1] and n_tasks != 1:
		if n_classes == y.shape[1] and n_tasks != 1 and n_classes != n_tasks:
		raise ValueError("Shape of input doesn't match expected n_tasks=1")
		elif n_classes == y.shape[1] and n_tasks == 1:
		# Add in task dimension
		y = np.expand_dims(y, 1)
		if (len(y.shape) == 2 or len(y.shape) == 3) and n_tasks != y.shape[1]:
		@@ -527,9 +537,9 @@ def kappa_score(y_true, y_pred):
		yp = np.asarray(y_pred, dtype=int)
		if not set(np.unique(yt)).issubset(set([0, 1])):
		raise ValueError("Class labels must be binary 0, 1")
		#assert np.array_equal(
		# np.unique(yt),
		# [0, 1]), ('Class labels must be binary: %s' % np.unique(yt))
		assert np.array_equal(
		np.unique(yt),
		[0, 1]), ('Class labels must be binary: %s' % np.unique(yt))
		observed_agreement = np.true_divide(
		np.count_nonzero(np.equal(yt, yp)), len(yt))
		expected_agreement = np.true_divide(

deepchem/models/tests/test_graph_models.py

+330 −331

Original line number	Diff line number	Diff line
		@@ -13,12 +13,7 @@ from deepchem.feat import ConvMolFeaturizer
		from flaky import flaky


		class TestGraphModels(unittest.TestCase):

		def get_dataset(self,
		mode='classification',
		featurizer='GraphConv',
		num_tasks=2):
		def get_dataset(mode='classification', featurizer='GraphConv', num_tasks=2):
		data_points = 10
		if mode == 'classification':
		tasks, all_dataset, transformers = load_bace_classification(featurizer)
		@@ -43,9 +38,10 @@ class TestGraphModels(unittest.TestCase):

		return tasks, ds, transformers, metric

		def test_graph_conv_model(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'classification', 'GraphConv')

		def test_graph_conv_model():
		tasks, dataset, transformers, metric = get_dataset('classification',
		'GraphConv')

		batch_size = 10
		model = GraphConvModel(
		@@ -58,9 +54,10 @@ class TestGraphModels(unittest.TestCase):
		scores = model.evaluate(dataset, [metric], transformers)
		assert scores['mean-roc_auc_score'] >= 0.9

		def test_neural_fingerprint_retrieval(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'classification', 'GraphConv')

		def test_neural_fingerprint_retrieval():
		tasks, dataset, transformers, metric = get_dataset('classification',
		'GraphConv')

		fp_size = 3

		@@ -74,11 +71,11 @@ class TestGraphModels(unittest.TestCase):
		model.fit(dataset, nb_epoch=1)
		neural_fingerprints = model.predict_embedding(dataset)
		neural_fingerprints = np.array(neural_fingerprints)[:len(dataset)]
		self.assertEqual((len(dataset), fp_size * 2), neural_fingerprints.shape)
		assert (len(dataset), fp_size * 2) == neural_fingerprints.shape

		def test_graph_conv_regression_model(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'regression', 'GraphConv')

		def test_graph_conv_regression_model():
		tasks, dataset, transformers, metric = get_dataset('regression', 'GraphConv')

		batch_size = 10
		model = GraphConvModel(
		@@ -89,11 +86,11 @@ class TestGraphModels(unittest.TestCase):

		model.fit(dataset, nb_epoch=100)
		scores = model.evaluate(dataset, [metric], transformers)
		assert all(s < 0.1 for s in scores['mean_absolute_error'])
		assert scores['mean_absolute_error'] < 0.1


		def test_graph_conv_regression_uncertainty(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'regression', 'GraphConv')
		def test_graph_conv_regression_uncertainty():
		tasks, dataset, transformers, metric = get_dataset('regression', 'GraphConv')

		batch_size = 10
		model = GraphConvModel(
		@@ -115,8 +112,9 @@ class TestGraphModels(unittest.TestCase):
		assert mean_std > 0.5 * mean_error
		assert mean_std < mean_value

		def test_graph_conv_atom_features(self):
		tasks, dataset, transformers, metric = self.get_dataset(

		def test_graph_conv_atom_features():
		tasks, dataset, transformers, metric = get_dataset(
		'regression', 'Raw', num_tasks=1)

		atom_feature_name = 'feature'
		@@ -125,8 +123,7 @@ class TestGraphModels(unittest.TestCase):
		atom_features = []
		for atom in mol.GetAtoms():
		val = np.random.normal()
		mol.SetProp("atom %08d %s" % (atom.GetIdx(), atom_feature_name),
		str(val))
		mol.SetProp("atom %08d %s" % (atom.GetIdx(), atom_feature_name), str(val))
		atom_features.append(np.random.normal())
		y.append([np.sum(atom_features)])

		@@ -143,10 +140,10 @@ class TestGraphModels(unittest.TestCase):
		model.fit(dataset, nb_epoch=1)
		y_pred1 = model.predict(dataset)


		@pytest.mark.slow
		def test_weave_model(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'classification', 'Weave')
		def test_weave_model():
		tasks, dataset, transformers, metric = get_dataset('classification', 'Weave')

		batch_size = 10
		model = WeaveModel(len(tasks), batch_size=batch_size, mode='classification')
		@@ -154,21 +151,22 @@ class TestGraphModels(unittest.TestCase):
		scores = model.evaluate(dataset, [metric], transformers)
		assert scores['mean-roc_auc_score'] >= 0.9


		@flaky
		def test_weave_regression_model(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'regression', 'Weave')
		def test_weave_regression_model():
		tasks, dataset, transformers, metric = get_dataset('regression', 'Weave')

		batch_size = 10
		model = WeaveModel(len(tasks), batch_size=batch_size, mode='regression')
		model.fit(dataset, nb_epoch=80)
		scores = model.evaluate(dataset, [metric], transformers)
		assert all(s < 0.1 for s in scores['mean_absolute_error'])
		assert scores['mean_absolute_error'] < 0.1


		@pytest.mark.slow
		def test_dag_model(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'classification', 'GraphConv')
		def test_dag_model():
		tasks, dataset, transformers, metric = get_dataset('classification',
		'GraphConv')

		batch_size = 10
		max_atoms = max([mol.get_num_atoms() for mol in dataset.X])
		@@ -187,13 +185,13 @@ class TestGraphModels(unittest.TestCase):
		scores = model.evaluate(dataset, [metric], transformers)
		assert scores['mean-roc_auc_score'] >= 0.9


		@pytest.mark.slow
		def test_dag_regression_model(self):
		def test_dag_regression_model():
		import tensorflow as tf
		np.random.seed(1234)
		tf.random.set_seed(1234)
		tasks, dataset, transformers, metric = self.get_dataset(
		'regression', 'GraphConv')
		tasks, dataset, transformers, metric = get_dataset('regression', 'GraphConv')

		batch_size = 10
		max_atoms = max([mol.get_num_atoms() for mol in dataset.X])
		@@ -210,15 +208,15 @@ class TestGraphModels(unittest.TestCase):

		model.fit(dataset, nb_epoch=1200)
		scores = model.evaluate(dataset, [metric], transformers)
		assert all(s < 0.15 for s in scores['mean_absolute_error'])
		assert scores['mean_absolute_error'] < 0.15


		@pytest.mark.slow
		def test_dag_regression_uncertainty(self):
		def test_dag_regression_uncertainty():
		import tensorflow as tf
		np.random.seed(1234)
		tf.random.set_seed(1234)
		tasks, dataset, transformers, metric = self.get_dataset(
		'regression', 'GraphConv')
		tasks, dataset, transformers, metric = get_dataset('regression', 'GraphConv')

		batch_size = 10
		max_atoms = max([mol.get_num_atoms() for mol in dataset.X])
		@@ -250,10 +248,10 @@ class TestGraphModels(unittest.TestCase):
		assert mean_std > 0.5 * mean_error
		assert mean_std < mean_value


		@pytest.mark.slow
		def test_mpnn_model(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'classification', 'Weave')
		def test_mpnn_model():
		tasks, dataset, transformers, metric = get_dataset('classification', 'Weave')

		batch_size = 10
		model = MPNNModel(
		@@ -270,10 +268,10 @@ class TestGraphModels(unittest.TestCase):
		scores = model.evaluate(dataset, [metric], transformers)
		assert scores['mean-roc_auc_score'] >= 0.9


		@pytest.mark.slow
		def test_mpnn_regression_model(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'regression', 'Weave')
		def test_mpnn_regression_model():
		tasks, dataset, transformers, metric = get_dataset('regression', 'Weave')

		batch_size = 10
		model = MPNNModel(
		@@ -288,12 +286,12 @@ class TestGraphModels(unittest.TestCase):

		model.fit(dataset, nb_epoch=60)
		scores = model.evaluate(dataset, [metric], transformers)
		assert all(s < 0.1 for s in scores['mean_absolute_error'])
		assert scores['mean_absolute_error'] < 0.1


		@pytest.mark.slow
		def test_mpnn_regression_uncertainty(self):
		tasks, dataset, transformers, metric = self.get_dataset(
		'regression', 'Weave')
		def test_mpnn_regression_uncertainty():
		tasks, dataset, transformers, metric = get_dataset('regression', 'Weave')

		batch_size = 10
		model = MPNNModel(
		@@ -319,8 +317,9 @@ class TestGraphModels(unittest.TestCase):
		assert mean_std > 0.5 * mean_error
		assert mean_std < mean_value


		@flaky
		def test_dtnn_regression_model(self):
		def test_dtnn_regression_model():
		current_dir = os.path.dirname(os.path.abspath(__file__))
		input_file = os.path.join(current_dir, "example_DTNN.mat")
		dataset = scipy.io.loadmat(input_file)

deepchem/models/tests/test_overfit.py

+3 −1

Original line number	Diff line number	Diff line
		@@ -199,6 +199,7 @@ def test_residual_classification_overfit():
		assert scores[classification_metric.name] > .9


		@flaky
		def test_fittransform_regression_overfit():
		"""Test that MultitaskFitTransformRegressor can overfit simple regression datasets."""
		n_samples = 10
		@@ -207,6 +208,7 @@ def test_fittransform_regression_overfit():

		# Generate dummy dataset
		np.random.seed(123)
		tf.random.set_seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features, n_features)
		y = np.zeros((n_samples, n_tasks))
		@@ -345,7 +347,7 @@ def test_sklearn_multitask_classification_overfit():
		assert scores[classification_metric.name] > .9


		#@flaky
		@flaky
		def test_multitask_classification_overfit():
		"""Test MultitaskClassifier overfits tiny data."""
		n_tasks = 10

deepchem/utils/test/test_evaluate.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -320,7 +320,8 @@ def test_gc_binary_classification():

		def test_gc_binary_kappa_classification():
		"""Test multiclass classification evaluation."""
		smiles = ["C", "CC"]
		np.random.seed(1234)
		smiles = ["C", "CC", "CO", "CCC", "CCCC"]
		featurizer = dc.feat.ConvMolFeaturizer()
		X = featurizer.featurize(smiles)
		y = np.random.randint(2, size=(len(smiles),))
		@@ -343,7 +344,6 @@ def test_gc_multiclass_classification():
		y = np.random.randint(5, size=(len(smiles),))
		dataset = dc.data.NumpyDataset(X, y)
		model = dc.models.GraphConvModel(1, mode="classification", n_classes=5)
		# TODO: Fix this case with correct thresholding
		evaluator = Evaluator(model, dataset, [])
		multitask_scores = evaluator.compute_model_performance(
		dc.metrics.accuracy_score, n_classes=5)

deepchem/utils/test/test_generator_evaluator.py

+98 −97

Original line number	Diff line number	Diff line
		@@ -9,10 +9,8 @@ import deepchem as dc
		from deepchem.data import NumpyDataset


		class TestGeneratorEvaluator(TestCase):

		@flaky
		def test_compute_model_performance_multitask_classifier(self):
		def test_compute_model_performance_multitask_classifier():
		n_data_points = 20
		n_features = 1
		n_tasks = 2
		@@ -50,7 +48,9 @@ class TestGeneratorEvaluator(TestCase):
		# Loosening atol to see if tests stop failing sporadically
		assert np.all(np.isclose(scores, [1.0, 1.0], atol=0.50))

		def test_compute_model_performance_singletask_classifier(self):

		def test_compute_model_performance_singletask_classifier():
		"""Computes model performance on singletask dataset with one-hot label encoding."""
		n_data_points = 20
		n_features = 10

		@@ -71,14 +71,15 @@ class TestGeneratorEvaluator(TestCase):

		model.fit(dataset, nb_epoch=1000)
		metric = dc.metrics.Metric(
		dc.metrics.roc_auc_score, np.mean, mode="classification")
		dc.metrics.roc_auc_score, np.mean, mode="classification", n_tasks=1)

		scores = model.evaluate_generator(
		model.default_generator(dataset), [metric], per_task_metrics=True)
		scores = list(scores[1].values())
		assert np.isclose(scores, [1.0], atol=0.05)

		def test_compute_model_performance_multitask_regressor(self):

		def test_compute_model_performance_multitask_regressor():
		random_seed = 42
		n_data_points = 20
		n_features = 2

Admin message