Commit f6a07d88 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Fixes to Metrics. Really need Metrics tests...

parent eaf30e02

deepchem/datasets/__init__.py

+1 −1
Original line number Diff line number Diff line
@@ -57,7 +57,7 @@ class Dataset(object): 
        for ind, (df_file, df) in enumerate(

            zip(samples.dataset_files, samples.iterdataframes())):

          log("Writing data from file %s, number %d/%d"

              % (df_file, ind, len(samples.dataset_files)), self.verbosity)

              % (df_file, ind+1, len(samples.dataset_files)), self.verbosity)

          retval = write_dataset_single_partial((df_file, df))

          if retval is not None:

            metadata_rows.append(retval)

deepchem/metrics/__init__.py

+47 −1
Original line number Diff line number Diff line
@@ -135,6 +135,15 @@ class Metric(object): 
      A numpy array containing metric values for each task.

    """

    assert y_true.shape[0] == y_pred.shape[0] == w.shape[0]

    ######## DEBUG

    #from deepchem.metrics import to_one_hot

    #import sklearn 

    #print("compute_metric")

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    #print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ######## DEBUG

    n_samples, n_tasks = y_true.shape[0], y_true.shape[1] 

    if self.mode == "classification":

      y_pred = np.reshape(y_pred, (n_samples, n_tasks, n_classes))
@@ -148,6 +157,13 @@ class Metric(object): 
      else:

        y_pred_task = y_pred[:, task, :]

      w_task = w[:, task]

      ######## DEBUG

      #print("compute_metric")

      #print("y_task.shape, y_pred_task.shape")

      #print(y_task.shape, y_pred_task.shape)

      #print("sklearn.metrics.roc_auc_score(to_one_hot(y_task), y_pred_task)")

      #print(sklearn.metrics.roc_auc_score(to_one_hot(y_task), y_pred_task))

      ######## DEBUG

    

      metric_value = self.compute_singletask_metric(

          y_task, y_pred_task, w_task)
@@ -175,6 +191,14 @@ class Metric(object): 
    """

    y_true = np.array(np.squeeze(y_true[w != 0]))

    y_pred = np.array(np.squeeze(y_pred[w != 0]))

    ####### DEBUG

    import sklearn 

    print("compute_singletask_metric")

    print("y_true.shape, y_pred.shape")

    print(y_true.shape, y_pred.shape)

    print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ####### DEBUG

    if len(y_true.shape) == 0:

      n_samples = 1

    else:
@@ -183,9 +207,20 @@ class Metric(object): 
    if not y_true.size:

      return np.nan

    y_true = np.reshape(y_true, (n_samples,))

    ######## DEBUG

    #import sklearn 

    #print("compute_singletask_metric after reshape")

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    #print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ######## DEBUG

    if self.mode == "classification":

      n_classes = y_pred.shape[-1]

      if self.name == "roc_auc_score":

      # TODO(rbharath): This has been a major source of bugs. Is there a more

      # robust characterization of which metrics require class-probs and which

      # don't?

      if "roc_auc_score" in self.name:

        y_true = to_one_hot(y_true).astype(int)

        y_pred = np.reshape(y_pred, (n_samples, n_classes))

      else:
@@ -196,6 +231,17 @@ class Metric(object): 
    else:

      y_pred = np.reshape(y_pred, (n_samples,))



    ######## DEBUG

    #import sklearn 

    #print("compute_singletask_metric after classification adjustments")

    #print("self.mode, self.name, n_classes")

    #print(self.mode, self.name, n_classes)

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(y_true, y_pred)")

    #print(sklearn.metrics.roc_auc_score(y_true, y_pred))

    ######## DEBUG

      

    if self.threshold is not None:

      y_pred = np.greater(y_pred, threshold)

    try:

deepchem/utils/evaluate.py

+8 −0
Original line number Diff line number Diff line
@@ -72,6 +72,14 @@ class Evaluator(object): 
      y_pred = self.model.predict(self.dataset, self.transformers)

    multitask_scores = {}



    ####### DEBUG

    from deepchem.metrics import to_one_hot

    print("y.shape, y_pred.shape")

    print(y.shape, y_pred.shape)

    print("sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred)")

    print(sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred))

    ####### DEBUG



    if csv_out is not None:

      log("Saving predictions to %s" % csv_out, self.verbosity)

      self.output_predictions(y_pred, csv_out)

examples/muv_multitask_models.py

+40 −43
Original line number Diff line number Diff line
@@ -22,13 +22,17 @@ from deepchem.hyperparameters import HyperparamOpt 
from deepchem.models.multitask import SingletaskToMultitask

from deepchem import metrics

from deepchem.metrics import Metric

from deepchem.metrics import to_one_hot

from deepchem.models.sklearn_models import SklearnModel

from deepchem.utils.evaluate import relative_difference





np.random.seed(123)



# Set some global variables up top

reload = True

verbosity = "high"

model = "tensorflow"

model = "logistic"



# Create some directories for analysis

# The base_dir holds the results of all analysis
@@ -66,7 +70,7 @@ print(MUV_tasks) 
featurizer = DataFeaturizer(tasks=MUV_tasks,

                            smiles_field="smiles",

                            compound_featurizers=featurizers,

                            verbosity="low")

                            verbosity=verbosity)

featurized_samples = featurizer.featurize(

    dataset_file, feature_dir,

    samples_dir, shard_size=8192,
@@ -75,13 +79,13 @@ featurized_samples = featurizer.featurize( 
assert len(featurized_samples) == len(dataset)

# Train/Valid/Test Split dataset

print("About to perform train/valid/test split.")

#splitter = ScaffoldSplitter(verbosity=verbosity)

splitter = RandomSplitter(verbosity=verbosity)

frac_train, frac_valid, frac_test = .8, .1, .1

train_samples, valid_samples, test_samples = \

    splitter.train_valid_test_split(

        featurized_samples, train_dir, valid_dir, test_dir,

        log_every_n=1000, reload=reload)



len_train_samples, len_valid_samples, len_test_samples = \

  len(train_samples), len(valid_samples), len(test_samples)

assert relative_difference(
@@ -116,12 +120,18 @@ y_test = test_dataset.get_labels() 
len_train_dataset, len_valid_dataset, len_test_dataset = \

  len(train_dataset), len(valid_dataset), len(test_dataset)



print("len(train_samples), len(train_dataset)")

print(len(train_samples), len(train_dataset))

assert relative_difference(

    len(train_samples), len(train_dataset)) < 1e-3

print("len(valid_samples), len(valid_dataset)")

print(len(valid_samples), len(valid_dataset))

assert relative_difference(

    len(valid_samples), len(valid_dataset)) < 1e-3

    len(valid_samples), len(valid_dataset)) < 1e-2

print("len(test_samples), len(test_dataset)")

print(len(test_samples), len(test_dataset))

assert relative_difference(

    len(test_samples), len(test_dataset)) < 1e-3

    len(test_samples), len(test_dataset)) < 1e-2



# Transform data

print("About to transform data")
@@ -137,22 +147,41 @@ for transformer in transformers: 
    transformer.transform(test_dataset)



# Fit Logistic Regression models

print("About to fit logistic regression models")

MUV_task_types = {task: "Classification" for task in MUV_tasks}





classification_metric = Metric(metrics.roc_auc_score, np.mean, verbosity="low")

if model == "logistic":

classification_metric = Metric(metrics.roc_auc_score, np.mean,

                               verbosity=verbosity,

                               mode="classification")

params_dict = { 

    "batch_size": [None],

    "data_shape": [train_dataset.get_data_shape()],

}   



  print("IN LOGISTIC MODE!")

######### DEBUG

#def build_rf(train_dataset, valid_dataset):

#  X_train, y_train, w_train, _ = train_dataset.to_numpy()

#  X_valid, y_valid, w_valid, _ = valid_dataset.to_numpy()

#

#  w_train, w_valid = w_train.flatten(), w_valid.flatten()

#  y_train, y_valid = y_train[w_train != 0], y_valid[w_valid != 0]

#  X_train, X_valid = X_train[w_train != 0], X_valid[w_valid != 0]

#

#  cl = LogisticRegression(class_weight="balanced")

#  y_valid, y_train = y_valid.astype(int), y_train.astype(int)

#  cl.fit(X_train, y_train)

#  y_train_pred = cl.predict_proba(X_train)

#  y_valid_pred = cl.predict_proba(X_valid)

#  print("metrics.roc_auc_score(to_one_hot(y_train), y_train_pred)")

#  print(metrics.roc_auc_score(to_one_hot(y_train), y_train_pred))

#  print("metrics.roc_auc_score(to_one_hot(y_valid), y_valid_pred)")

#  print(metrics.roc_auc_score(to_one_hot(y_valid), y_valid_pred))

#build_rf(train_dataset, valid_dataset)

######### DEBUG



def model_builder(tasks, task_types, model_params, model_dir, verbosity=None):

  return SklearnModel(tasks, task_types, model_params, model_dir,

                      model_instance=LogisticRegression(class_weight="balanced"),

                        #model_instance=RandomForestClassifier(),

                      verbosity=verbosity)

def multitask_model_builder(tasks, task_types, params_dict, model_dir, logdir=None,

                            verbosity=None):
@@ -160,38 +189,6 @@ if model == "logistic": 
                               model_builder, verbosity=verbosity)

optimizer = HyperparamOpt(multitask_model_builder, MUV_tasks, MUV_task_types,

                          verbosity=verbosity)

  best_logistic, best_logistic_hyperparams, all_logistic_results = \

      optimizer.hyperparam_search(

          params_dict, train_dataset, valid_dataset, output_transformers,

          classification_metric, logdir=model_dir, use_max=True)

else:

  print("IN TF MODE!")



  params_dict = { 

      "batch_size": [64],

      "nb_epoch": [10],

      "data_shape": [train_dataset.get_data_shape()],

      "layer_sizes": [[1000]],

      "weight_init_stddevs": [[.01]],

      "bias_init_consts": [[.5]],

      "dropouts": [[.0]],

      "num_classification_tasks": [len(MUV_tasks)],

      "num_classes": [2],

      "penalty": [.0],

      "optimizer": ["sgd"],

      "learning_rate": [.0003],

  }   



  from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

  from deepchem.models.tensorflow_models import TensorflowModel

  def tf_multitask_model_builder(tasks, task_types, params_dict, model_dir, logdir=None,

                              verbosity=None):

    return TensorflowModel(tasks, task_types, params_dict, model_dir,

                           tf_class=TensorflowMultiTaskClassifier,

                           verbosity=verbosity)

  optimizer = HyperparamOpt(tf_multitask_model_builder, MUV_tasks, MUV_task_types,

                            verbosity=verbosity)

  best_dnn, best_dnn_hyperparams, all_dnn_results = \

      optimizer.hyperparam_search(

best_lgstc, best_hyperparams, all_results = optimizer.hyperparam_search(

        params_dict, train_dataset, valid_dataset, output_transformers,

        classification_metric, logdir=model_dir, use_max=True)

examples/muv_tf_models.py

0 → 100644
+181 −0
Original line number Diff line number Diff line 
"""

Script that trains multitask tensorflow models on MUV dataset.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



import os

import numpy as np

from sklearn.linear_model import LogisticRegression

from sklearn.ensemble import RandomForestClassifier

from deepchem.models.test import TestAPI

from deepchem.utils.save import load_from_disk

from deepchem.datasets import Dataset

from deepchem.featurizers.featurize import DataFeaturizer

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.splits import ScaffoldSplitter

from deepchem.splits import RandomSplitter

from deepchem.datasets import Dataset

from deepchem.transformers import BalancingTransformer

from deepchem.hyperparameters import HyperparamOpt

from deepchem.models.multitask import SingletaskToMultitask

from deepchem import metrics

from deepchem.metrics import Metric

from deepchem.metrics import to_one_hot

from deepchem.models.sklearn_models import SklearnModel

from deepchem.utils.evaluate import relative_difference





np.random.seed(123)



# Set some global variables up top

reload = True

verbosity = "high"

model = "logistic"



# Create some directories for analysis

# The base_dir holds the results of all analysis

#base_dir = "/scratch/users/rbharath/muv_multitask_analysis"

base_dir = "/scratch/users/rbharath/small_muv_multitask_analysis"

current_dir = os.path.dirname(os.path.realpath(__file__))

#Make directories to store the raw and featurized datasets.

feature_dir = os.path.join(base_dir, "features")

samples_dir = os.path.join(base_dir, "samples")

full_dir = os.path.join(base_dir, "full_dataset")

train_dir = os.path.join(base_dir, "train_dataset")

valid_dir = os.path.join(base_dir, "valid_dataset")

test_dir = os.path.join(base_dir, "test_dataset")

model_dir = os.path.join(base_dir, "model")



# Load MUV dataset

print("About to load MUV dataset.")

dataset_file = os.path.join(

    current_dir, "../datasets/muv.csv.gz")

dataset = load_from_disk(dataset_file)

print("Columns of dataset: %s" % str(dataset.columns.values))

print("Number of examples in dataset: %s" % str(dataset.shape[0]))



# Featurize MUV dataset

print("About to featurize MUV dataset.")

featurizers = [CircularFingerprint(size=1024)]

MUV_tasks = sorted(['MUV-692', 'MUV-689', 'MUV-846', 'MUV-859', 'MUV-644',

                    'MUV-548', 'MUV-852', 'MUV-600', 'MUV-810', 'MUV-712',

                    'MUV-737', 'MUV-858', 'MUV-713', 'MUV-733', 'MUV-652',

                    'MUV-466', 'MUV-832'])

# For debugging purposes

MUV_tasks = MUV_tasks[0:1]

print("Using following tasks")

print(MUV_tasks)

featurizer = DataFeaturizer(tasks=MUV_tasks,

                            smiles_field="smiles",

                            compound_featurizers=featurizers,

                            verbosity="low")

featurized_samples = featurizer.featurize(

    dataset_file, feature_dir,

    samples_dir, shard_size=8192,

    reload=reload)



assert len(featurized_samples) == len(dataset)

# Train/Valid/Test Split dataset

print("About to perform train/valid/test split.")

splitter = RandomSplitter(verbosity=verbosity)

frac_train, frac_valid, frac_test = .8, .1, .1

train_samples, valid_samples, test_samples = \

    splitter.train_valid_test_split(

        featurized_samples, train_dir, valid_dir, test_dir,

        log_every_n=1000, reload=reload)



len_train_samples, len_valid_samples, len_test_samples = \

  len(train_samples), len(valid_samples), len(test_samples)

assert relative_difference(

    len(train_samples), frac_train * len(featurized_samples)) < 1e-3

assert relative_difference(

    len(valid_samples), frac_valid * len(featurized_samples)) < 1e-3

assert relative_difference(

    len(test_samples), frac_test * len(featurized_samples)) < 1e-3



# Generate datasets

print("About to create datasets")

print("MUV_tasks")

print(MUV_tasks)

full_dataset = Dataset(data_dir=full_dir, samples=featurized_samples, 

                        featurizers=featurizers, tasks=MUV_tasks,

                        verbosity=verbosity, reload=reload)

print("full_dataset.get_task_names()")

print(full_dataset.get_task_names())

y = full_dataset.get_labels()

train_dataset = Dataset(data_dir=train_dir, samples=train_samples, 

                        featurizers=featurizers, tasks=MUV_tasks,

                        verbosity=verbosity, reload=reload)

y_train  = train_dataset.get_labels()

valid_dataset = Dataset(data_dir=valid_dir, samples=valid_samples, 

                        featurizers=featurizers, tasks=MUV_tasks,

                        verbosity=verbosity, reload=reload)

y_valid = valid_dataset.get_labels()

test_dataset = Dataset(data_dir=test_dir, samples=test_samples, 

                       featurizers=featurizers, tasks=MUV_tasks,

                       verbosity=verbosity, reload=reload)

y_test = test_dataset.get_labels()

len_train_dataset, len_valid_dataset, len_test_dataset = \

  len(train_dataset), len(valid_dataset), len(test_dataset)



print("len(train_samples), len(train_dataset)")

print(len(train_samples), len(train_dataset))

assert relative_difference(

    len(train_samples), len(train_dataset)) < 1e-3

print("len(valid_samples), len(valid_dataset)")

print(len(valid_samples), len(valid_dataset))

assert relative_difference(

    len(valid_samples), len(valid_dataset)) < 1e-2

print("len(test_samples), len(test_dataset)")

print(len(test_samples), len(test_dataset))

assert relative_difference(

    len(test_samples), len(test_dataset)) < 1e-2



# Transform data

print("About to transform data")

input_transformers = []

output_transformers = []

weight_transformers = [BalancingTransformer(transform_w=True, dataset=train_dataset)]

transformers = input_transformers + output_transformers + weight_transformers

for transformer in transformers:

    transformer.transform(train_dataset)

for transformer in transformers:

    transformer.transform(valid_dataset)

for transformer in transformers:

    transformer.transform(test_dataset)



# Fit tensorflow models

MUV_task_types = {task: "Classification" for task in MUV_tasks}

classification_metric = Metric(metrics.roc_auc_score, np.mean,

                               verbosity=verbosity,

                               mode="classification")

params_dict = { 

    "batch_size": [64],

    "nb_epoch": [10],

    "data_shape": [train_dataset.get_data_shape()],

    "layer_sizes": [[1000]],

    "weight_init_stddevs": [[.01]],

    "bias_init_consts": [[.5]],

    "dropouts": [[.0]],

    "num_classification_tasks": [len(MUV_tasks)],

    "num_classes": [2],

    "penalty": [.0],

    "optimizer": ["sgd"],

    "learning_rate": [.0003],

}   



from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

from deepchem.models.tensorflow_models import TensorflowModel

def tf_multitask_model_builder(tasks, task_types, params_dict, model_dir, logdir=None,

                            verbosity=None):

  return TensorflowModel(tasks, task_types, params_dict, model_dir,

                         tf_class=TensorflowMultiTaskClassifier,

                         verbosity=verbosity)

optimizer = HyperparamOpt(tf_multitask_model_builder, MUV_tasks, MUV_task_types,

                          verbosity=verbosity)

best_dnn, best_dnn_hyperparams, all_dnn_results = \

    optimizer.hyperparam_search(

        params_dict, train_dataset, valid_dataset, output_transformers,

        classification_metric, logdir=model_dir, use_max=True)

CRA Git | Maintained and supported by SUSTech CRA and CCSE