Commit 0f90eb32 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Adds multitask metric support

parent 5aaee3ff

deepchem/hyperparameters/__init__.py

+15 −7
Original line number Diff line number Diff line
@@ -51,19 +51,24 @@ class HyperparamOpt(object): 
        model_params[hyperparam] = hyperparam_val



      model_dir = tempfile.mkdtemp()

      model = self.model_class(self.task_types, model_params, logdir=logdir,

                               train=True)

      if logdir is not None:

        model = self.model_class(self.task_types, model_params, logdir=logdir)

      else:

        model = self.model_class(self.task_types, model_params)

      model.fit(train_dataset)

      model.save(model_dir)

    

      evaluator = Evaluator(model, valid_dataset, output_transformers)

      df, score = evaluator.compute_model_performance(

      df, scores_df, multitask_scores = evaluator.compute_model_performance(

          [metric], valid_csv_out, valid_stats_out)

      valid_score = score.iloc[0][metric.name]

      if not metric.is_multitask:

        valid_score = scores_df.iloc[0][metric.name]

      else:

        valid_score = multitask_scores[metric.name]

      all_scores[hyperparameter_tuple] = valid_score

    

      if (use_max and valid_score > best_validation_score) or (

          not use_max and valid_score < best_validation_score):

      if (use_max and valid_score >= best_validation_score) or (

          not use_max and valid_score <= best_validation_score):

        best_validation_score = valid_score

        best_hyperparams = hyperparameter_tuple

        if best_model_dir is not None:
@@ -82,9 +87,12 @@ class HyperparamOpt(object): 
    train_csv_out = tempfile.NamedTemporaryFile()

    train_stats_out = tempfile.NamedTemporaryFile()

    train_evaluator = Evaluator(best_model, train_dataset, output_transformers)

    train_df, train_score = train_evaluator.compute_model_performance(

    train_df, train_score, multitask_scores = train_evaluator.compute_model_performance(

        [metric], train_csv_out, train_stats_out)

    if not metric.is_multitask:

      train_score = train_score.iloc[0][metric.name]

    else:

      train_score = multitask_scores[metric.name]

    log("Best hyperparameters: %s" % str(zip(hyperparams, best_hyperparams)),

        self.verbosity)

    log("train_score: %f" % train_score, self.verbosity)

deepchem/hyperparameters/tests/test_hyperparam_opt.py

+17 −13
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ import os 
import unittest

import tempfile

import shutil

import numpy as np

from deepchem.models.test import TestAPI

from deepchem.models.sklearn_models import SklearnModel

from deepchem.featurizers.fingerprints import CircularFingerprint
@@ -21,8 +22,7 @@ from deepchem import metrics 
from deepchem.metrics import Metric

from sklearn.ensemble import RandomForestRegressor



def rf_model_builder(task_types, params_dict, logdir=None,

                     train=True):

def rf_model_builder(task_types, params_dict, logdir=None):

    """Builds random forests given hyperparameters.



    Last two arguments only for tensorflow models and ignored.
@@ -70,14 +70,6 @@ class TestHyperparamOptAPI(TestAPI): 
    output_transformers = []

    input_transformers = []

    task_type = "classification"

    # TODO(rbharath): There should be some automatic check to ensure that all

    # required model_params are specified.

    model_params = {"nb_hidden": 10, "activation": "relu",

                    "dropout": .5, "learning_rate": .01,

                    "momentum": .9, "nesterov": False,

                    "decay": 1e-4, "batch_size": 5,

                    "nb_epoch": 2, "init": "glorot_uniform",

                    "nb_layers": 1, "batchnorm": False}



    input_file = os.path.join(self.current_dir, "multitask_example.csv")

    tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",
@@ -88,12 +80,24 @@ class TestHyperparamOptAPI(TestAPI): 
    compound_featurizers = [CircularFingerprint(size=1024)]

    complex_featurizers = []



    train_dataset, test_dataset, _, transformers = self._featurize_train_test_split(

    train_dataset, valid_dataset, _, transformers = self._featurize_train_test_split(

        splittype, compound_featurizers, 

        complex_featurizers, input_transformers,

        output_transformers, input_file, task_types.keys())

    model_params["data_shape"] = train_dataset.get_data_shape()

    metric = Metric(metrics.mean_roc_auc_score)

    metric = Metric(metrics.matthews_corrcoef, np.mean)

    params_dict= {"nb_hidden": [5, 10],

                  "activation": ["relu"],

                  "dropout": [.5],

                  "learning_rate": [.01],

                  "momentum": [.9],

                  "nesterov": [False],

                  "decay": [1e-4],

                  "batch_size": [5],

                  "nb_epoch": [2],

                  "init": ["glorot_uniform"],

                  "nb_layers": [1],

                  "batchnorm": [False],

                  "data_shape": [train_dataset.get_data_shape()]}

    

    self._hyperparam_opt(MultiTaskDNN, params_dict, train_dataset,

                         valid_dataset, output_transformers, task_types,

deepchem/metrics/__init__.py

+39 −25
Original line number Diff line number Diff line
@@ -25,30 +25,6 @@ from sklearn.metrics import r2_score 
from sklearn.metrics import mean_squared_error

from sklearn.metrics import mean_absolute_error



def compute_metrics(num_tasks, y_true, y_pred, metric):

  """Compute a performance metric for each task.



  Args:

    y_true: A list of arrays containing true values for each task.

    y_pred: A list of arrays containing predicted values for each task.

    metric: Must be a class that inherits from Metric 



  Returns:

    A numpy array containing metric values for each task.

  """

  computed_metrics = []

  for task in xrange(num_tasks):

    yt = y_true[task]

    yp = y_pred[task]

    try:

      metric_value = metric.compute(yt, yp)

    except (AssertionError, ValueError) as e:

      warnings.warn("Error calculating metric %s for task %d: %s"

                    % (metric_str, task, e))

      metric_value = np.nan

    computed_metrics.append(metric_value)

  return computed_metrics



def compute_roc_auc_scores(y, y_pred):

  """Transforms the results dict into roc-auc-scores and prints scores.
@@ -102,20 +78,58 @@ def kappa_score(y_true, y_pred): 
class Metric(object):

  """Wrapper class for computing user-defined metrics."""



  def __init__(self, metric, name=None, threshold=None):

  def __init__(self, metric, task_averager=None, name=None, threshold=None):

    """

    Args:

      metric: function that takes args y_true, y_pred (in that order) and

              computes desired score.

      task_averager: If not None, should be a function that averages metrics

              across tasks. For example, task_averager=np.mean. If task_averager

              is provided, this task will be inherited as a multitask metric.

    """

    self.metric = metric

    self.task_averager = task_averager

    self.is_multitask = (self.task_averager is not None)

    if name is None:

      if not self.is_multitask:  

        self.name = self.metric.__name__

      else:

        self.name = self.task_averager.__name__ + "-" + self.metric.__name__

    else:

      self.name = name

    self.threshold = threshold



  def compute_metric(self, y_true, y_pred):

    """Compute a performance metric for each task.



    Args:

      num_tasks: Number of tasks

      y_true: A list of arrays containing true values for each task.

      y_pred: A list of arrays containing predicted values for each task.

      metric: Must be a class that inherits from Metric 



    Returns:

      A numpy array containing metric values for each task.

    """

    assert len(y_true) == len(y_pred)

    num_tasks = len(y_true)

    computed_metrics = []

    for task in xrange(num_tasks):

      yt = y_true[task]

      yp = y_pred[task]

      try:

        metric_value = self.compute_singletask_metric(yt, yp)

      except (AssertionError, ValueError) as e:

        warnings.warn("Error calculating metric %s for task %d: %s"

                      % (metric_str, task, e))

        metric_value = np.nan

      computed_metrics.append(metric_value)

    if not self.is_multitask:

      return computed_metrics

    else:

      return self.task_averager(computed_metrics)



  def compute_singletask_metric(self, y_true, y_pred):

    """Compute a metric value.



    Args:

deepchem/models/test/multitask_example.csv

+4 −4
Original line number Diff line number Diff line 
mol_id,smiles,task0,task1,task2,task3,task4,task5,task6,task7,task8,task9,task10,task11,task12,task13,task14,task15,task16

CID2999678,Cc1cccc(N2CCN(C(=O)C34CC5CC(CC(C5)C3)C4)CC2)c1C,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1

CID2999679,Cn1ccnc1SCC(=O)Nc1ccc(Oc2ccccc2)cc1,0,0,,0,,0,,,,,0,0,,,0,0,0

CID5390003,COc1ccccc1NC(=O)C1=C(C)N=C2N=CNN2C1c1ccc(C)o1,0,,,,,,,0,,0,,0,,,,,

CID5390002,O=C1c2ccccc2/C(=C\NC2CCS(=O)(=O)C2)C(=O)N1c1ccccc1,,0,,,,0,,0,,,,0,,,,,

CID2999679,Cn1ccnc1SCC(=O)Nc1ccc(Oc2ccccc2)cc1,1,0,,0,,0,,,,,0,0,,,0,0,0

CID5390003,COc1ccccc1NC(=O)C1=C(C)N=C2N=CNN2C1c1ccc(C)o1,0,,,,,,,0,,1,,0,,,,,

CID5390002,O=C1c2ccccc2/C(=C\NC2CCS(=O)(=O)C2)C(=O)N1c1ccccc1,,0,,,,0,,1,,,,0,,,,,

CID2999670,NC(=O)NC(Cc1ccccc1)C(=O)O,0,,,,,,,,,0,,,,,0,0,

CID5390000,Cc1[nH]nc(NC=C2C(=O)N(C)C(=O)N(C)C2=O)c1-c1ccccc1,,,,,,,,0,,,,,,,,,

CID5390000,Cc1[nH]nc(NC=C2C(=O)N(C)C(=O)N(C)C2=O)c1-c1ccccc1,,,,,,,,1,,,,,,,,,

CID1511280,CC(=O)N1CCC2(CC1)NC(=O)N(c1ccccc1)N2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1

CID5390006,COc1ccc(C2C(C(=O)NCc3ccccc3)=C(C)N=C3N=CNN32)cc1OC,,,,,,,,,,0,0,,,,,,

CID5390005,CCc1cc2c(CN3CCN(C)CC3)cc(=O)oc2cc1O,,,,,,,,,,0,,,,,,,

deepchem/utils/evaluate.py

+25 −4
Original line number Diff line number Diff line
@@ -61,11 +61,13 @@ class Evaluator(object): 
    Computes statistics of model on test data and saves results to csv.

    """

    pred_y_df = self.model.predict(self.dataset, self.transformers)

    multitask_scores = {}



    task_type = self.task_type

    colnames = ["task_name"] + [metric.name for metric in metrics]

    performance_df = pd.DataFrame(columns=colnames)



    ys, y_preds, ws = [], [], []

    for i, task_name in enumerate(self.task_names):

      y = pred_y_df[task_name].values

      y_pred = pred_y_df["%s_pred" % task_name].values
@@ -80,15 +82,34 @@ class Evaluator(object): 
        # Sometimes all samples have zero weight. In this case, continue.

        if not len(y):

          continue

      ys.append(y)

      y_preds.append(y_pred)

      ws.append(w)



      scores = []

    # Compute multitask metrics

    for metric in metrics:

        scores.append(metric.compute_metric(y, y_pred))

      performance_df.loc[i] = [task_name] + scores

      if metric.is_multitask:

        multitask_scores[metric.name] = metric.compute_metric(ys, y_preds)



    all_scores = []

    for metric in metrics:

      if not metric.is_multitask:

        all_scores.append(metric.compute_metric(ys, y_preds))

    # Note that all_scores will be of shape num_singletask_metrics x num_tasks

    all_scores = np.array(all_scores)

    # If there are any singletask_metrics

    if all_scores.shape[0] > 0:

      for i, task_name in enumerate(self.task_names):

        performance_df.loc[i] = [task_name] + list(all_scores[i])



    #  scores = []

    #  for metric in metrics:

    #    scores.append(metric.compute_metric(y, y_pred))

    #  performance_df.loc[i] = [task_name] + scores



    log("Saving predictions to %s" % csv_out, self.verbose)

    pred_y_df.to_csv(csv_out)

    log("Saving model performance scores to %s" % stats_file, self.verbose)

    performance_df.to_csv(stats_file)



    return pred_y_df, performance_df
 
    return pred_y_df, performance_df, multitask_scores

CRA Git | Maintained and supported by SUSTech CRA and CCSE