Commit 5c35018f authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Fix to multitask/classification metrics

parent 15eab458

deepchem/datasets/__init__.py

+1 −0
Original line number Diff line number Diff line
@@ -121,6 +121,7 @@ class Dataset(object): 
        self.save_to_disk()



    else:

      log("Loading pre-existing metadata file.", self.verbosity)

      if os.path.exists(self._get_metadata_filename()):

        self.metadata_df = load_from_disk(self._get_metadata_filename())

      else:

deepchem/hyperparameters/__init__.py

+14 −8
Original line number Diff line number Diff line
@@ -76,11 +76,14 @@ class HyperparamOpt(object): 
      model.save()

    

      evaluator = Evaluator(model, valid_dataset, output_transformers)

      df, scores_df, multitask_scores = evaluator.compute_model_performance(

      #df, scores_df, multitask_scores = evaluator.compute_model_performance(

      #    [metric], valid_csv_out, valid_stats_out)

      multitask_scores = evaluator.compute_model_performance(

          [metric], valid_csv_out, valid_stats_out)

      if not metric.is_multitask:

        valid_score = scores_df.iloc[0][metric.name]

      else:

      #if not metric.is_multitask:

      #  valid_score = scores_df.iloc[0][metric.name]

      #else:

      #  valid_score = multitask_scores[metric.name]

      valid_score = multitask_scores[metric.name]

      all_scores[hyperparameter_tuple] = valid_score
@@ -107,11 +110,14 @@ class HyperparamOpt(object): 
    train_csv_out = tempfile.NamedTemporaryFile()

    train_stats_out = tempfile.NamedTemporaryFile()

    train_evaluator = Evaluator(best_model, train_dataset, output_transformers)

    train_df, train_score, multitask_scores = train_evaluator.compute_model_performance(

    #train_df, train_score, multitask_scores = train_evaluator.compute_model_performance(

    #    [metric], train_csv_out, train_stats_out)

    multitask_scores = train_evaluator.compute_model_performance(

        [metric], train_csv_out, train_stats_out)

    if not metric.is_multitask:

      train_score = train_score.iloc[0][metric.name]

    else:

    #if not metric.is_multitask:

    #  train_score = train_score.iloc[0][metric.name]

    #else:

    #  train_score = multitask_scores[metric.name]

    train_score = multitask_scores[metric.name]

    log("Best hyperparameters: %s" % str(best_hyperparams),

        self.verbosity, "low")

deepchem/metrics/__init__.py

+56 −10
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ 


import numpy as np

import warnings

from deepchem.utils.save import log

from sklearn.metrics import roc_auc_score

from sklearn.metrics import matthews_corrcoef

from sklearn.metrics import recall_score
@@ -10,6 +11,24 @@ from sklearn.metrics import r2_score 
from sklearn.metrics import mean_squared_error

from sklearn.metrics import mean_absolute_error



# DEBUG

def to_one_hot(y):

  """Transforms label vector into one-hot encoding.



  Turns y into vector of shape [n_samples, 2] (assuming binary labels).



  y: np.ndarray

    A vector of shape [n_samples, 1]

  """

  n_samples = np.shape(y)[0]

  y_hot = np.zeros((n_samples, 2))

  for index, val in enumerate(y):

    if val == 0:

      y_hot[index] = np.array([1, 0])

    elif val == 1:

      y_hot[index] = np.array([0, 1])

  return y_hot



def compute_roc_auc_scores(y, y_pred):

  """Transforms the results dict into roc-auc-scores and prints scores.
@@ -63,7 +82,8 @@ def kappa_score(y_true, y_pred): 
class Metric(object):

  """Wrapper class for computing user-defined metrics."""



  def __init__(self, metric, task_averager=None, name=None, threshold=None):

  def __init__(self, metric, task_averager=None, name=None, threshold=None,

               verbosity=None, mode="classification"):

    """

    Args:

      metric: function that takes args y_true, y_pred (in that order) and
@@ -82,9 +102,12 @@ class Metric(object): 
        self.name = self.task_averager.__name__ + "-" + self.metric.__name__

    else:

      self.name = name

    self.verbosity = verbosity

    self.threshold = threshold

    assert mode in ["classification", "regression"]

    self.mode = mode



  def compute_metric(self, y_true, y_pred):

  def compute_metric(self, y_true, y_pred, w):

    """Compute a performance metric for each task.



    Args:
@@ -96,25 +119,34 @@ class Metric(object): 
    Returns:

      A numpy array containing metric values for each task.

    """

    assert len(y_true) == len(y_pred)

    num_tasks = len(y_true)

    print("y_true.shape, y_pred.shape, w.shape")

    print(y_true.shape, y_pred.shape, w.shape)

    assert y_true.shape[0] == y_pred.shape[0] == w.shape[0]

    num_tasks = y_true.shape[1] 

    computed_metrics = []

    for task in xrange(num_tasks):

      yt = y_true[task]

      yp = y_pred[task]

      y_task = y_true[:, task]

      y_pred_task = y_pred[:, task]

      w_task = w[:, task]

    

      try:

        metric_value = self.compute_singletask_metric(yt, yp)

        import sklearn

        #print("sklearn.metrics.roc_auc_score(y_task, y_pred_task)")

        #print(sklearn.metrics.roc_auc_score(y_task, y_pred_task))

        metric_value = self.compute_singletask_metric(

            y_task, y_pred_task, w_task)

      except (AssertionError, ValueError) as e:

        warnings.warn("Error calculating metric %s for task %d: %s"

                      % (metric_str, task, e))

        warnings.warn("Error calculating metric for task %d: %s"

                      % (task, e))

        metric_value = np.nan

      computed_metrics.append(metric_value)

    log("computed_metrics: %s" % str(computed_metrics), self.verbosity)

    if not self.is_multitask:

      return computed_metrics

    else:

      return self.task_averager(computed_metrics)



  def compute_singletask_metric(self, y_true, y_pred):

  def compute_singletask_metric(self, y_true, y_pred, w):

    """Compute a metric value.



    Args:
@@ -127,8 +159,22 @@ class Metric(object): 
    Raises:

      NotImplementedError: If metric_str is not in METRICS.

    """

    print("compute_singletask_metric()")

    print("y_true.shape, y_pred.shape, w.shape")

    print(y_true.shape, y_pred.shape, w.shape)

    y_true = y_true[w != 0]

    y_pred = y_pred[w != 0]

    if self.mode == "classification":

      y_true = to_one_hot(y_true).astype(int)

      y_pred = y_pred[:, np.newaxis]

    if self.threshold is not None:

      y_pred = np.greater(y_pred, threshold)

    print("y_true.shape, y_pred.shape")

    print(y_true.shape, y_pred.shape)

    import sklearn

    print("compute_singletask_metric()")

    print("sklearn.metrics.roc_auc_score(y_true, y_pred)")

    print(sklearn.metrics.roc_auc_score(y_true, y_pred))

    try:

      metric_value = self.metric(y_true, y_pred)

    except (AssertionError, ValueError) as e:

deepchem/models/__init__.py

+108 −52
Original line number Diff line number Diff line
@@ -14,6 +14,26 @@ from deepchem.datasets import Dataset 
from deepchem.utils.save import load_from_disk

from deepchem.utils.save import save_to_disk

from deepchem.utils.save import log

import sklearn





# DEBUG COPY!

def to_one_hot(y):

  """Transforms label vector into one-hot encoding.



  Turns y into vector of shape [n_samples, 2] (assuming binary labels).



  y: np.ndarray

    A vector of shape [n_samples, 1]

  """

  n_samples = np.shape(y)[0]

  y_hot = np.zeros((n_samples, 2))

  for index, val in enumerate(y):

    if val == 0:

      y_hot[index] = np.array([1, 0])

    elif val == 1:

      y_hot[index] = np.array([0, 1])

  return y_hot



def undo_transforms(y, transformers):

  """Undoes all transformations applied."""
@@ -105,7 +125,8 @@ class Model(object): 
          X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch,

                                            w_batch)

        losses.append(self.fit_on_batch(X_batch, y_batch, w_batch))

      log("Avg loss for epoch %d: %f" % (epoch+1,np.array(losses).mean()),self.verbosity)

      log("Avg loss for epoch %d: %f"

          % (epoch+1,np.array(losses).mean()),self.verbosity)





  def transform_on_batch(self, X, y, w):
@@ -124,57 +145,92 @@ class Model(object): 
    """

    Uses self to make predictions on provided Dataset object.

    """

    task_names = dataset.get_task_names()

    pred_task_names = ["%s_pred" % task_name for task_name in task_names]

    w_task_names = ["%s_weight" % task_name for task_name in task_names]

    raw_task_names = [task_name+"_raw" for task_name in task_names]

    raw_pred_task_names = [pred_task_name+"_raw" for pred_task_name in pred_task_names]

    column_names = (['ids'] + raw_task_names + task_names

                    + raw_pred_task_names + pred_task_names + w_task_names

                    + ["y_means", "y_stds"])

    pred_y_df = pd.DataFrame(columns=column_names)



    batch_size = self.model_params["batch_size"]

    for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):



      # HACK(JG): This was a hack to perform n-fold averaging of y_pred on

      # a given X_batch.  If fit_transformers exist, we will apply them to

      # X_batch 1 times and average the resulting y_pred before we undo 

      # transforms on y_pred and y.  In the future the averaging will be

      # performed n_sample times, where n_sample can be user-specified.



      if self.fit_transformers:



        y_preds = []

        for i in xrange(1):

          X_b, y_b, w_b = self.transform_on_batch(X_batch, y_batch, w_batch)

          y_pred = self.predict_on_batch(X_b)

          y_pred = np.reshape(y_pred, np.shape(y_b))

          y_preds.append(y_pred)



        y_pred = np.array(y_preds).mean(axis=0)



      else:



        y_pred = self.predict_on_batch(X_batch)

        y_pred = np.reshape(y_pred, np.shape(y_batch))



      # Now undo transformations on y, y_pred



      y_raw, y_pred_raw = y_batch, y_pred

      y_batch = undo_transforms(y_batch, transformers)

      y_pred = undo_transforms(y_pred, transformers)



      batch_df = pd.DataFrame(columns=column_names)

      batch_df['ids'] = ids_batch

      batch_df[raw_task_names] = y_raw

      batch_df[task_names] = y_batch

      batch_df[raw_pred_task_names] = y_pred_raw

      batch_df[pred_task_names] = y_pred

      batch_df[w_task_names] = w_batch

      pred_y_df = pd.concat([pred_y_df, batch_df])



    return pred_y_df

    X, y, w, ids = dataset.to_numpy()

    print("X.shape, y.shape, w.shape, ids.shape")

    print(X.shape, y.shape, w.shape, ids.shape)

    

    #X = X[w.flatten() != 0, :]



    y_pred = self.predict_on_batch(X)



    y = y[w.flatten() != 0, :]

    y = to_one_hot(y)

    y_pred_d = y_pred[w.flatten() != 0, :]



    print("X.shape, y.shape, y_pred_d.shape")

    print(X.shape, y.shape, y_pred_d.shape)

    print("Model.predict()")

    print("sklearn.metrics.roc_auc_score(y, y_pred_d)")

    print(sklearn.metrics.roc_auc_score(y, y_pred_d))



    return y_pred



    #task_names = dataset.get_task_names()

    #pred_task_names = ["%s_pred" % task_name for task_name in task_names]

    #w_task_names = ["%s_weight" % task_name for task_name in task_names]

    #raw_task_names = [task_name+"_raw" for task_name in task_names]

    #raw_pred_task_names = [pred_task_name+"_raw" for pred_task_name in pred_task_names]

    #column_names = (['ids'] + raw_task_names + task_names

    #                + raw_pred_task_names + pred_task_names + w_task_names

    #                + ["y_means", "y_stds"])

    #pred_y_df = pd.DataFrame(columns=column_names)



    #batch_size = self.model_params["batch_size"]

    #for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):



    #  # HACK(JG): This was a hack to perform n-fold averaging of y_pred on

    #  # a given X_batch.  If fit_transformers exist, we will apply them to

    #  # X_batch 1 times and average the resulting y_pred before we undo 

    #  # transforms on y_pred and y.  In the future the averaging will be

    #  # performed n_sample times, where n_sample can be user-specified.



    #  if self.fit_transformers:



    #    y_preds = []

    #    for i in xrange(1):

    #      X_b, y_b, w_b = self.transform_on_batch(X_batch, y_batch, w_batch)

    #      y_pred = self.predict_on_batch(X_b)

    #      y_pred = np.reshape(y_pred, np.shape(y_b))

    #      y_preds.append(y_pred)



    #    y_pred = np.array(y_preds).mean(axis=0)



    #  else:



    #    # DEBUG

    #    X_batch = X_batch[w_batch.flatten() != 0, :]

    #    y_batch = y_batch[w_batch.flatten() != 0]



    #    y_pred = self.predict_on_batch(X_batch)

    #    y_pred = np.reshape(y_pred, np.shape(y_batch))



    #    # DEBUG:

    #    print("y_batch.shape, y_pred.shape")

    #    print(y_batch.shape, y_pred.shape)

    #    #y_batch_d = y_batch[w_batch != 0]

    #    #y_pred_d = y_pred[w_batch != 0]

    #    #print("y_batch_d.shape, y_pred_d.shape")

    #    #print(y_batch_d.shape, y_pred_d.shape)

    #    if np.count_nonzero(y_batch) > 0:

    #      print("sklearn.metrics.roc_auc_score(y_batch, y_pred)")

    #      print(sklearn.metrics.roc_auc_score(y_batch, y_pred))



    #  # Now undo transformations on y, y_pred



    #  y_raw, y_pred_raw = y_batch, y_pred

    #  y_batch = undo_transforms(y_batch, transformers)

    #  y_pred = undo_transforms(y_pred, transformers)



    #  batch_df = pd.DataFrame(columns=column_names)

    #  #batch_df['ids'] = ids_batch

    #  #batch_df[raw_task_names] = y_raw

    #  #batch_df[task_names] = y_batch

    #  #batch_df[raw_pred_task_names] = y_pred_raw

    #  #batch_df[pred_task_names] = y_pred

    #  #batch_df[w_task_names] = w_batch

    #  pred_y_df = pd.concat([pred_y_df, batch_df])



    #return pred_y_df



  def get_task_type(self):

    """

deepchem/models/multitask.py

+26 −4
Original line number Diff line number Diff line
@@ -9,6 +9,8 @@ import os 
import numpy as np

from deepchem.utils.save import log

from deepchem.models import Model

# DEBUG

import sklearn



class SingletaskToMultitask(Model):

  """
@@ -29,7 +31,8 @@ class SingletaskToMultitask(Model): 
    if not os.path.exists(self.model_dir):

      os.makedirs(self.model_dir)

    self.fit_transformers = False

    for task, task_type in self.task_types.iteritems():

    for task in self.tasks:

      task_type = self.task_types[task]

      task_model_dir = os.path.join(self.model_dir, task)

      if not os.path.exists(task_model_dir):

        os.makedirs(task_model_dir)
@@ -45,11 +48,21 @@ class SingletaskToMultitask(Model): 


    Warning: This current implementation is only functional for sklearn models. 

    """

    X, y, _, _ = dataset.to_numpy()

    X, y, w, _ = dataset.to_numpy()

    for ind, task in enumerate(self.tasks):

      log("Fitting model for task %s" % task, self.verbosity, "high")

      y_task = y[:, ind]

      self.models[task].raw_model.fit(X, y_task)

      w_task = w[:, ind]

      # DEBUG

      X_task = X[w_task != 0, :]

      y_task = y_task[w_task != 0]

      #self.models[task].raw_model.fit(X, y_task, w_task)

      self.models[task].raw_model.fit(X_task, y_task)



      y_pred_task = self.models[task].raw_model.predict(X_task)

      print("Train ROC-AUC for %s: %f"

            % (task, sklearn.metrics.roc_auc_score(y_task.astype(int),

                                                   y_pred_task)))



  def predict_on_batch(self, X):

    """
@@ -59,7 +72,16 @@ class SingletaskToMultitask(Model): 
    N_samples = X.shape[0]

    y_pred = np.zeros((N_samples, N_tasks))

    for ind, task in enumerate(self.tasks):

      y_pred[:, ind] = self.models[task].predict_on_batch(X)

      # DEBUG: ONLY FOR DEBUGGING

      #print("ind, task")

      #print(ind, task)

      #print("self.models[task]")

      #print(self.models[task])

      #print("type(self.models[task].predict_on_batch(X))")

      #print(type(self.models[task].predict_on_batch(X)))

      y_pred[:, ind] = self.models[task].predict_on_batch(X)[:, 0]

      #return self.models[task].predict_on_batch(X)[:, 0]

      #y_pred[task] = self.models[task].predict_on_batch(X)

    return y_pred



  def save(self):

CRA Git | Maintained and supported by SUSTech CRA and CCSE