Commit 3752f9da authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Merge pull request #166 from rbharath/multitask_debug 

Basic Singletask test-suite for ML models learning.
parents 59615278 927ba807

deepchem/datasets/__init__.py

+1 −1
Original line number Diff line number Diff line
@@ -246,7 +246,7 @@ class Dataset(object): 
    """

    ws = []

    for (_, _, w_b, _) in self.itershards():

      ws.append(w_b)

      ws.append(np.array(w_b))

    return np.vstack(ws)



  def _pad_batch(self, X_b, y_b, w_b, ids_b, batch_size):

deepchem/hyperparameters/__init__.py

+2 −2
Original line number Diff line number Diff line
@@ -79,7 +79,7 @@ class HyperparamOpt(object): 
      multitask_scores = evaluator.compute_model_performance(

          [metric], valid_csv_out.name, valid_stats_out)

      valid_score = multitask_scores[metric.name]

      all_scores[hyperparameter_tuple] = valid_score

      all_scores[str(hyperparameter_tuple)] = valid_score

    

      if (use_max and valid_score >= best_validation_score) or (

          not use_max and valid_score <= best_validation_score):
@@ -105,7 +105,7 @@ class HyperparamOpt(object): 
    train_stats_out = tempfile.NamedTemporaryFile()

    train_evaluator = Evaluator(best_model, train_dataset, output_transformers)

    multitask_scores = train_evaluator.compute_model_performance(

        [metric], train_csv_out, train_stats_out)

        [metric], train_csv_out.name, train_stats_out)

    train_score = multitask_scores[metric.name]

    log("Best hyperparameters: %s" % str(best_hyperparams),

        self.verbosity, "low")

deepchem/hyperparameters/tests/test_hyperparam_opt.py

+3 −2
Original line number Diff line number Diff line
@@ -104,7 +104,8 @@ class TestHyperparamOptAPI(TestAPI): 
        "batch_size": [32],

        "data_shape": [train_dataset.get_data_shape()],

    }

    classification_metric = Metric(metrics.matthews_corrcoef, np.mean)

    classification_metric = Metric(metrics.matthews_corrcoef, np.mean,

                                   mode="classification")

    def model_builder(tasks, task_types, model_params, task_model_dir,

                      verbosity=None):

      return SklearnModel(tasks, task_types, model_params, task_model_dir,
@@ -138,7 +139,7 @@ class TestHyperparamOptAPI(TestAPI): 
        splittype, compound_featurizers, 

        complex_featurizers, input_transformers,

        output_transformers, input_file, tasks)

    metric = Metric(metrics.matthews_corrcoef, np.mean)

    metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification")

    params_dict= {"nb_hidden": [5, 10],

                  "activation": ["relu"],

                  "dropout": [.5],

deepchem/hyperparameters/tests/test_tf_hyperparam_opt.py

+1 −1
Original line number Diff line number Diff line
@@ -43,7 +43,7 @@ class TestTFHyperparamOptAPI(TestAPI): 
        splittype, compound_featurizers, 

        complex_featurizers, input_transformers,

        output_transformers, input_file, tasks)

    metric = Metric(metrics.matthews_corrcoef, np.mean)

    metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification")

    params_dict = {"activation": ["relu"],

                    "momentum": [.9],

                    "batch_size": [50],

deepchem/metrics/__init__.py

+43 −15
Original line number Diff line number Diff line
@@ -90,7 +90,7 @@ class Metric(object): 
  """Wrapper class for computing user-defined metrics."""



  def __init__(self, metric, task_averager=None, name=None, threshold=None,

               verbosity=None, mode="classification"):

               verbosity=None, mode=None):

    """

    Args:

      metric: function that takes args y_true, y_pred (in that order) and
@@ -111,14 +111,22 @@ class Metric(object): 
      self.name = name

    self.verbosity = verbosity

    self.threshold = threshold

    if mode is None:

      if self.name in ["roc_auc_score", "matthews_corrcoef", "recall_score",

                       "accuracy_score", "kappa_score"]:

        mode = "classification"

      elif self.name in ["r2_score", "mean_squared_error",

                         "mean_absolute_error"]:

        mode = "regression"

      else:

        raise ValueError("Must specify mode for new metric.")

    assert mode in ["classification", "regression"]

    self.mode = mode



  def compute_metric(self, y_true, y_pred, w):

  def compute_metric(self, y_true, y_pred, w, n_classes=2):

    """Compute a performance metric for each task.



    Args:

      num_tasks: Number of tasks

      y_true: A list of arrays containing true values for each task.

      y_pred: A list of arrays containing predicted values for each task.

      metric: Must be a class that inherits from Metric
@@ -126,24 +134,28 @@ class Metric(object): 
    Returns:

      A numpy array containing metric values for each task.

    """

    print("y_true.shape, y_pred.shape")

    print(y_true.shape, y_pred.shape)

    assert y_true.shape[0] == y_pred.shape[0] == w.shape[0]

    num_tasks = y_true.shape[1] 

    n_samples, n_tasks = y_true.shape[0], y_true.shape[1] 

    if self.mode == "classification":

      y_pred = np.reshape(y_pred, (n_samples, n_tasks, n_classes))

    else:

      y_pred = np.reshape(y_pred, (n_samples, n_tasks))

    computed_metrics = []

    for task in xrange(num_tasks):

    for task in xrange(n_tasks):

      y_task = y_true[:, task]

      if self.mode == "regression":

        y_pred_task = y_pred[:, task]

      else:

        y_pred_task = y_pred[:, task, :]

      w_task = w[:, task]

    

      try:

      metric_value = self.compute_singletask_metric(

          y_task, y_pred_task, w_task)

      except (AssertionError, ValueError) as e:

        warnings.warn("Error calculating metric for task %d: %s"

                      % (task, e))

        metric_value = np.nan

      computed_metrics.append(metric_value)

    log("computed_metrics: %s" % str(computed_metrics), self.verbosity)

    if num_tasks == 1:

    if n_tasks == 1:

      computed_metrics = computed_metrics[0]

    if not self.is_multitask:

      return computed_metrics
@@ -163,14 +175,30 @@ class Metric(object): 
    Raises:

      NotImplementedError: If metric_str is not in METRICS.

    """

    y_true = y_true[w != 0]

    y_pred = y_pred[w != 0]

    print("compute_singletask_metric()")

    y_true = np.array(np.squeeze(y_true[w != 0]))

    y_pred = np.array(np.squeeze(y_pred[w != 0]))

    if len(y_true.shape) == 0:

      n_samples = 1

    else:

      n_samples = y_true.shape[0]

    # If there are no nonzero examples, metric is ill-defined.

    if not len(y_true):

    if not y_true.size:

      return np.nan

    y_true = np.reshape(y_true, (n_samples,))

    if self.mode == "classification":

      n_classes = y_pred.shape[-1]

      if self.name == "roc_auc_score":

        y_true = to_one_hot(y_true).astype(int)

      y_pred = y_pred[:, np.newaxis]

        y_pred = np.reshape(y_pred, (n_samples, n_classes))

      else:

        y_true = y_true.astype(int)

        # Reshape to handle 1-d edge cases

        y_pred = np.reshape(y_pred, (n_samples, n_classes))

        y_pred = from_one_hot(y_pred)

    else:

      y_pred = np.reshape(y_pred, (n_samples,))

      

    if self.threshold is not None:

      y_pred = np.greater(y_pred, threshold)

    try:

CRA Git | Maintained and supported by SUSTech CRA and CCSE