Merge pull request #124 from rbharath/metrics (0094be44) · Commits · 钟慕尧 / deepchem

deepchem/metrics/init.py

+54 −38

Original line number	Diff line number	Diff line
		@@ -19,18 +19,22 @@ import collections


		import numpy as np
		from sklearn import metrics

		def compute_metric(num_tasks, y_true, y_pred, metric_str, threshold=0.5):
		import warnings
		from sklearn.metrics import roc_auc_score
		from sklearn.metrics import matthews_corrcoef
		from sklearn.metrics import recall_score
		from sklearn.metrics import accuracy_score
		from sklearn.metrics import r2_score
		from sklearn.metrics import mean_squared_error
		from sklearn.metrics import mean_absolute_error

		def compute_metrics(num_tasks, y_true, y_pred, metric):
		"""Compute a performance metric for each task.

		Args:
		y_true: A list of arrays containing true values for each task.
		y_pred: A list of arrays containing predicted values for each task.
		metric_str: String description of the metric to compute. Must be in
		metrics.METRICS.
		threshold: Float threshold to apply to probabilities for positive/negative
		class assignment.
		metric: Must be a class that inherits from Metric

		Returns:
		A numpy array containing metric values for each task.
		@@ -40,15 +44,31 @@ def compute_metric(num_tasks, y_true, y_pred, metric_str, threshold=0.5):
		yt = y_true[task]
		yp = y_pred[task]
		try:
		metric_value = compute_metric(yt, yp, metric_str,
		threshold=threshold)
		metric_value = metric.compute(yt, yp)
		except (AssertionError, ValueError) as e:
		warnings.warn('Error calculating metric %s for task %d: %s'
		warnings.warn("Error calculating metric %s for task %d: %s"
		% (metric_str, task, e))
		metric_value = np.nan
		computed_metrics.append(metric_value)
		return computed_metrics

		def compute_roc_auc_scores(y, y_pred):
		"""Transforms the results dict into roc-auc-scores and prints scores.

		Parameters
		----------
		results: dict
		task_types: dict
		dict mapping task names to output type. Each output type must be either
		"classification" or "regression".
		"""
		try:
		score = roc_auc_score(y, y_pred)
		except ValueError:
		warnings.warn("ROC AUC score calculation failed.")
		score = 0.5
		return score

		def kappa_score(y_true, y_pred):
		"""Calculate Cohen's kappa for classification tasks.

		@@ -82,17 +102,28 @@ def kappa_score(y_true, y_pred):
		1.0 - expected_agreement)
		return kappa

		class Metric(object):
		"""Wrapper class for computing user-defined metrics."""

		def compute_metric(y_true, y_pred, metric_str, threshold=0.5):
		def __init__(self, metric, name=None, threshold=None):
		"""
		Args:
		metric: function that takes args y_true, y_pred (in that order) and
		computes desired score.
		"""
		self.metric = metric
		if name is None:
		self.name = self.metric.__name__
		else:
		self.name = name
		self.threshold = threshold

		def compute_metric(self, y_true, y_pred):
		"""Compute a metric value.

		Args:
		y_true: A list of arrays containing true values for each task.
		y_pred: A list of arrays containing predicted values for each task.
		metric_str: String description of the metric to compute. Must be in
		biology_metrics.METRICS.
		threshold: Float threshold to apply to probabilities for positive/negative
		class assignment.

		Returns:
		Float metric value.
		@@ -100,27 +131,12 @@ def compute_metric(y_true, y_pred, metric_str, threshold=0.5):
		Raises:
		NotImplementedError: If metric_str is not in METRICS.
		"""
		if metric_str not in METRICS:
		raise NotImplementedError('Unsupported metric %s' % metric_str)
		metric_tuple = METRICS[metric_str]
		if metric_tuple.threshold:
		if self.threshold is not None:
		y_pred = np.greater(y_pred, threshold)
		return metric_tuple.func(y_true, y_pred)


		class Metric(collections.namedtuple('MetricTuple', ['func', 'threshold'])):
		"""A named tuple used to organize model evaluation metrics.

		Args:
		func: Function to call. Should take true and predicted values (in that
		order) and compute the metric.
		threshold: Boolean indicating whether float values should be converted to
		binary labels prior to computing the metric, e.g. accuracy.
		"""

		METRICS = {
		'accuracy': Metric(metrics.accuracy_score, True),
		'auc': Metric(metrics.roc_auc_score, False),
		'kappa': Metric(kappa_score, True),
		'r2': Metric(metrics.r2_score, False),
		}
		try:
		metric_value = self.metric(y_true, y_pred)
		except (AssertionError, ValueError) as e:
		warnings.warn("Error calculating metric %s: %s"
		% (self.name, e))
		metric_value = np.nan
		return metric_value

deepchem/models/test/init.py

+5 −9

Original line number	Diff line number	Diff line
		@@ -48,35 +48,31 @@ class TestAPI(unittest.TestCase):
		shutil.rmtree(self.samples_dir)
		shutil.rmtree(self.train_dir)
		shutil.rmtree(self.test_dir)
		# TODO(rbharath): Removing this causes crashes for some reason. Need to
		# debug.
		#shutil.rmtree(self.model_dir)

		def _create_model(self, train_dataset, test_dataset, model, transformers,
		test_model_creator=None):
		metrics):
		"""Helper method to create model for test."""

		# Fit trained model
		model.fit(train_dataset)
		model.save(self.model_dir)

		# Now create test model
		if test_model_creator is not None:
		test_model = test_model_creator()
		test_model.load(self.model_dir)
		model = test_model

		# Eval model on train
		evaluator = Evaluator(model, train_dataset, transformers, verbose=True)
		with tempfile.NamedTemporaryFile() as train_csv_out:
		with tempfile.NamedTemporaryFile() as train_stats_out:
		_, _ = evaluator.compute_model_performance(
		train_csv_out, train_stats_out)
		metrics, train_csv_out, train_stats_out)

		# Eval model on test
		evaluator = Evaluator(model, test_dataset, transformers, verbose=True)
		with tempfile.NamedTemporaryFile() as test_csv_out:
		with tempfile.NamedTemporaryFile() as test_stats_out:
		_, _ = evaluator.compute_model_performance(
		test_csv_out, test_stats_out)
		metrics, test_csv_out, test_stats_out)

		def _featurize_train_test_split(self, splittype, compound_featurizers,
		complex_featurizers,

deepchem/models/test/test_api.py

+45 −13

Original line number	Diff line number	Diff line
		@@ -26,8 +26,10 @@ from deepchem.models.sklearn_models import SklearnModel
		from deepchem.transformers import NormalizationTransformer
		from deepchem.transformers import LogTransformer
		from deepchem.transformers import ClippingTransformer
		from sklearn.ensemble import RandomForestRegressor
		from deepchem.models.test import TestAPI
		from deepchem import metrics
		from deepchem.metrics import Metric
		from sklearn.ensemble import RandomForestRegressor

		class TestKerasSklearnAPI(TestAPI):
		"""
		@@ -48,9 +50,14 @@ class TestKerasSklearnAPI(TestAPI):
		complex_featurizers, input_transformers,
		output_transformers, input_file, task_types.keys())
		model_params["data_shape"] = train_dataset.get_data_shape()
		regression_metrics = [Metric(metrics.r2_score),
		Metric(metrics.mean_squared_error),
		Metric(metrics.mean_absolute_error)]

		model = SklearnModel(task_types, model_params, model_instance=RandomForestRegressor())
		self._create_model(train_dataset, test_dataset, model, transformers)
		model = SklearnModel(task_types, model_params,
		model_instance=RandomForestRegressor())
		self._create_model(train_dataset, test_dataset, model, transformers,
		regression_metrics)

		def test_singletask_sklearn_rf_user_specified_regression_API(self):
		"""Test of singletask RF ECFP regression API."""
		@@ -71,9 +78,14 @@ class TestKerasSklearnAPI(TestAPI):
		user_specified_features=user_specified_features,
		split_field=split_field)
		model_params["data_shape"] = train_dataset.get_data_shape()
		regression_metrics = [Metric(metrics.r2_score),
		Metric(metrics.mean_squared_error),
		Metric(metrics.mean_absolute_error)]

		model = SklearnModel(task_types, model_params, model_instance=RandomForestRegressor())
		self._create_model(train_dataset, test_dataset, model, transformers)
		model = SklearnModel(task_types, model_params,
		model_instance=RandomForestRegressor())
		self._create_model(train_dataset, test_dataset, model, transformers,
		regression_metrics)

		def test_singletask_sklearn_rf_ECFP_regression_sharded_API(self):
		"""Test of singletask RF ECFP regression API: sharded edition."""
		@@ -92,11 +104,15 @@ class TestKerasSklearnAPI(TestAPI):
		shard_size=50)
		# We set shard size above to force the creation of multiple shards of the data.
		# pdbbind_core has ~200 examples.

		model_params["data_shape"] = train_dataset.get_data_shape()
		regression_metrics = [Metric(metrics.r2_score),
		Metric(metrics.mean_squared_error),
		Metric(metrics.mean_absolute_error)]

		model = SklearnModel(task_types, model_params, model_instance=RandomForestRegressor())
		self._create_model(train_dataset, test_dataset, model, transformers)
		model = SklearnModel(task_types, model_params,
		model_instance=RandomForestRegressor())
		self._create_model(train_dataset, test_dataset, model, transformers,
		regression_metrics)

		def test_singletask_sklearn_rf_RDKIT_descriptor_regression_API(self):
		"""Test of singletask RF RDKIT-descriptor regression API."""
		@@ -113,9 +129,14 @@ class TestKerasSklearnAPI(TestAPI):
		complex_featurizers, input_transformers,
		output_transformers, input_file, task_types.keys())
		model_params["data_shape"] = train_dataset.get_data_shape()
		regression_metrics = [Metric(metrics.r2_score),
		Metric(metrics.mean_squared_error),
		Metric(metrics.mean_absolute_error)]

		model = SklearnModel(task_types, model_params, model_instance=RandomForestRegressor())
		self._create_model(train_dataset, test_dataset, model, transformers)
		model = SklearnModel(task_types, model_params,
		model_instance=RandomForestRegressor())
		self._create_model(train_dataset, test_dataset, model, transformers,
		regression_metrics)

		'''
		# TODO(rbharath): This fails on many systems with an Illegal Instruction
		@@ -181,12 +202,18 @@ class TestKerasSklearnAPI(TestAPI):
		ligand_pdb_field=ligand_pdb_field,
		user_specified_features=user_specified_features)
		model_params["data_shape"] = train_dataset.get_data_shape()
		regression_metrics = [Metric(metrics.r2_score),
		Metric(metrics.mean_squared_error),
		Metric(metrics.mean_absolute_error)]

		model = SingleTaskDNN(task_types, model_params)
		self._create_model(train_dataset, test_dataset, model, transformers)
		self._create_model(train_dataset, test_dataset, model, transformers,
		regression_metrics)


		#TODO(enf/rbharath): 3D CNN's are broken and must be fixed.
		#TODO(enf/rbharath): This should be uncommented now that 3D CNNs are in
		# keras. Need to upgrade the base version of keras for
		# deepchem.
		'''
		def test_singletask_cnn_GridFeaturizer_regression_API(self):
		"""Test of singletask 3D ConvNet regression API."""
		@@ -249,6 +276,11 @@ class TestKerasSklearnAPI(TestAPI):
		complex_featurizers, input_transformers,
		output_transformers, input_file, task_types.keys())
		model_params["data_shape"] = train_dataset.get_data_shape()
		classification_metrics = [Metric(metrics.roc_auc_score),
		Metric(metrics.matthews_corrcoef),
		Metric(metrics.recall_score),
		Metric(metrics.accuracy_score)]

		model = MultiTaskDNN(task_types, model_params)
		self._create_model(train_dataset, test_dataset, model, transformers)
		self._create_model(train_dataset, test_dataset, model, transformers,
		classification_metrics)

deepchem/models/test/test_tf_api.py

+10 −2

Original line number	Diff line number	Diff line
		@@ -28,8 +28,10 @@ from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifie
		from deepchem.transformers import NormalizationTransformer
		from deepchem.transformers import LogTransformer
		from deepchem.transformers import ClippingTransformer
		from sklearn.ensemble import RandomForestRegressor
		from deepchem.models.test import TestAPI
		from deepchem import metrics
		from deepchem.metrics import Metric
		from sklearn.ensemble import RandomForestRegressor

		class TestTensorflowAPI(TestAPI):
		"""
		@@ -72,7 +74,13 @@ class TestTensorflowAPI(TestAPI):
		"learning_rate": .001,
		"data_shape": train_dataset.get_data_shape()
		}
		classification_metrics = [Metric(metrics.roc_auc_score),
		Metric(metrics.matthews_corrcoef),
		Metric(metrics.recall_score),
		Metric(metrics.accuracy_score)]

		model = TensorflowModel(
		task_types, model_params, self.model_dir,
		tf_class=TensorflowMultiTaskClassifier)
		self._create_model(train_dataset, test_dataset, model, transformers)
		self._create_model(train_dataset, test_dataset, model, transformers,
		classification_metrics)

deepchem/utils/evaluate.py

+7 −52

Original line number	Diff line number	Diff line
		@@ -8,13 +8,6 @@ from __future__ import unicode_literals
		import numpy as np
		import warnings
		from deepchem.utils.save import log
		from sklearn.metrics import mean_squared_error
		from sklearn.metrics import mean_absolute_error
		from sklearn.metrics import roc_auc_score
		from sklearn.metrics import r2_score
		from sklearn.metrics import matthews_corrcoef
		from sklearn.metrics import recall_score
		from sklearn.metrics import accuracy_score
		import pandas as pd

		__author__ = "Bharath Ramsundar"
		@@ -52,23 +45,6 @@ def threshold_predictions(y, threshold):
		y_out[ind] = 1 if pred > threshold else 0
		return y_out

		def compute_roc_auc_scores(y, y_pred):
		"""Transforms the results dict into roc-auc-scores and prints scores.

		Parameters
		----------
		results: dict
		task_types: dict
		dict mapping task names to output type. Each output type must be either
		"classification" or "regression".
		"""
		try:
		score = roc_auc_score(y, y_pred)
		except ValueError:
		warnings.warn("ROC AUC score calculation failed.")
		score = 0.5
		return score

		class Evaluator(object):
		"""Class that evaluates a model on a given dataset."""

		@@ -80,24 +56,14 @@ class Evaluator(object):
		self.task_type = model.get_task_type()
		self.verbose = verbose

		def compute_model_performance(self, csv_out, stats_file, threshold=None):
		def compute_model_performance(self, metrics, csv_out, stats_file, threshold=None):
		"""
		Computes statistics of model on test data and saves results to csv.
		"""
		pred_y_df = self.model.predict(self.dataset, self.transformers)

		task_type = self.task_type
		if threshold is not None:
		task_type = "classification"

		if task_type == "classification":
		colnames = ["task_name", "roc_auc_score", "matthews_corrcoef",
		"recall_score", "accuracy_score"]
		elif task_type == "regression":
		colnames = ["task_name", "r2_score", "rms_error", "mae"]
		else:
		raise ValueError("Unrecognized task type: %s" % task_type)

		colnames = ["task_name"] + [metric.name for metric in metrics]
		performance_df = pd.DataFrame(columns=colnames)

		for i, task_name in enumerate(self.task_names):
		@@ -114,22 +80,11 @@ class Evaluator(object):
		# Sometimes all samples have zero weight. In this case, continue.
		if not len(y):
		continue
		auc = compute_roc_auc_scores(y, y_pred)
		mcc = matthews_corrcoef(y, y_pred)
		recall = recall_score(y, y_pred)
		accuracy = accuracy_score(y, y_pred)
		performance_df.loc[i] = [task_name, auc, mcc, recall, accuracy]

		elif task_type == "regression":
		try:
		r2s = r2_score(y, y_pred)
		rms = np.sqrt(mean_squared_error(y, y_pred))
		mae = mean_absolute_error(y, y_pred)
		except ValueError:
		r2s = np.nan
		rms = np.nan
		mae = np.nan
		performance_df.loc[i] = [task_name, r2s, rms, mae]

		scores = []
		for metric in metrics:
		scores.append(metric.compute_metric(y, y_pred))
		performance_df.loc[i] = [task_name] + scores

		log("Saving predictions to %s" % csv_out, self.verbose)
		pred_y_df.to_csv(csv_out)

Admin message