Commit bf8abcb2 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Cleanup pass

parent 9490fb3e

deepchem/metrics/__init__.py

+2 −42
Original line number Diff line number Diff line
@@ -135,15 +135,6 @@ class Metric(object): 
      A numpy array containing metric values for each task.

    """

    assert y_true.shape[0] == y_pred.shape[0] == w.shape[0]

    ######### DEBUG

    ##from deepchem.metrics import to_one_hot

    ##import sklearn 

    #print("compute_metric")

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    ##print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    ##print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ######### DEBUG

    n_samples, n_tasks = y_true.shape[0], y_true.shape[1] 

    if self.mode == "classification":

      y_pred = np.reshape(y_pred, (n_samples, n_tasks, n_classes))
@@ -157,13 +148,6 @@ class Metric(object): 
      else:

        y_pred_task = y_pred[:, task, :]

      w_task = w[:, task]

      ######## DEBUG

      #print("compute_metric")

      #print("y_task.shape, y_pred_task.shape")

      #print(y_task.shape, y_pred_task.shape)

      #print("sklearn.metrics.roc_auc_score(to_one_hot(y_task), y_pred_task)")

      #print(sklearn.metrics.roc_auc_score(to_one_hot(y_task), y_pred_task))

      ######## DEBUG

    

      metric_value = self.compute_singletask_metric(

          y_task, y_pred_task, w_task)
@@ -191,14 +175,7 @@ class Metric(object): 
    """

    y_true = np.array(np.squeeze(y_true[w != 0]))

    y_pred = np.array(np.squeeze(y_pred[w != 0]))

    ######## DEBUG

    #import sklearn 

    #print("compute_singletask_metric")

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    #print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ######## DEBUG



    if len(y_true.shape) == 0:

      n_samples = 1

    else:
@@ -207,14 +184,7 @@ class Metric(object): 
    if not y_true.size:

      return np.nan

    y_true = np.reshape(y_true, (n_samples,))

    ######## DEBUG

    #import sklearn 

    #print("compute_singletask_metric after reshape")

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    #print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ######## DEBUG



    if self.mode == "classification":

      n_classes = y_pred.shape[-1]

      # TODO(rbharath): This has been a major source of bugs. Is there a more
@@ -231,16 +201,6 @@ class Metric(object): 
    else:

      y_pred = np.reshape(y_pred, (n_samples,))



    ######## DEBUG

    #import sklearn 

    #print("compute_singletask_metric after classification adjustments")

    #print("self.mode, self.name, n_classes")

    #print(self.mode, self.name, n_classes)

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(y_true, y_pred)")

    #print(sklearn.metrics.roc_auc_score(y_true, y_pred))

    ######## DEBUG

      

    if self.threshold is not None:

      y_pred = np.greater(y_pred, threshold)

deepchem/models/sklearn_models/__init__.py

+0 −5
Original line number Diff line number Diff line
@@ -56,11 +56,6 @@ class SklearnModel(Model): 
    """

    Makes per-class predictions on batch of data.

    """

    ######## DEBUG

    print("predict_proba_on_batch()")

    print("self.raw_model.predict_proba(X).shape")

    print(self.raw_model.predict_proba(X).shape)

    ######## DEBUG

    return self.raw_model.predict_proba(X)



  def predict(self, X, transformers):

deepchem/models/test/test_overfit.py

+2 −17
Original line number Diff line number Diff line
@@ -257,10 +257,7 @@ class TestOverfitAPI(TestAPI): 
    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [regression_metric], csv_out.name, stats_out)

    scores = evaluator.compute_model_performance([regression_metric])



    assert scores[regression_metric.name] < .1
@@ -308,10 +305,6 @@ class TestOverfitAPI(TestAPI): 


    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])

    ######### DEBUG

    #print("y_pred_proba_model.shape")

    #print(y_pred_proba_model.shape)

    ######### DEBUG



    # Eval model on train

    transformers = []
@@ -356,7 +349,6 @@ class TestOverfitAPI(TestAPI): 


    verbosity = "high"

    classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)

    #classification_metric = Metric(metrics.recall_score, verbosity=verbosity)

    model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir,

                         verbosity=verbosity)
@@ -366,8 +358,6 @@ class TestOverfitAPI(TestAPI): 


    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])

    #print("y_pred_proba_model.shape")

    #print(y_pred_proba_model.shape)



    # Eval model on train

    transformers = []
@@ -486,11 +476,6 @@ class TestOverfitAPI(TestAPI): 
    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)

    scores = evaluator.compute_model_performance([classification_metric])



    print("scores")

    print(scores)

    assert scores[classification_metric.name] > .8

deepchem/utils/evaluate.py

+0 −12
Original line number Diff line number Diff line
@@ -48,10 +48,6 @@ class Evaluator(object): 
      csvfile: Open file object.

    """

    mol_ids = self.dataset.get_ids()

    ################ DEBUG

    print("len(y_preds), len(mol_ids)")

    print(len(y_preds), len(mol_ids))

    ################ DEBUG

    assert len(y_preds) == len(mol_ids)

    with open(csv_out, "wb") as csvfile:

      csvwriter = csv.writer(csvfile)
@@ -76,14 +72,6 @@ class Evaluator(object): 
      y_pred = self.model.predict(self.dataset, self.transformers)

    multitask_scores = {}



    ######### DEBUG

    ##from deepchem.metrics import to_one_hot

    #print("y.shape, y_pred.shape")

    #print(y.shape, y_pred.shape)

    ##print("sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred)")

    ##print(sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred))

    ######### DEBUG



    if csv_out is not None:

      log("Saving predictions to %s" % csv_out, self.verbosity)

      self.output_predictions(y_pred, csv_out)

examples/muv_keras_models.py

+12 −32
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ from deepchem.metrics import Metric 
from deepchem.metrics import to_one_hot

from deepchem.models.sklearn_models import SklearnModel

from deepchem.utils.evaluate import relative_difference

from deepchem.utils.evaluate import Evaluator

from deepchem.models.keras_models.fcnet import MultiTaskDNN
@@ -153,22 +154,6 @@ classification_metric = Metric(metrics.roc_auc_score, np.mean, 
                               verbosity=verbosity,

                               mode="classification")



#params_dict = {

#    "nb_hidden": [1000],

#    "activation": ["relu"],

#    "dropout": [.25],

#    "learning_rate": [.001],

#    "momentum": [.9],

#    "nesterov": [False],

#    "decay": [1e-4],

#    "batch_size": [64],

#    "nb_epoch": [100],

#    "init": ["glorot_uniform"],

#    "nb_layers": [1],

#    "batchnorm": [False],

#    "data_shape": [train_dataset.get_data_shape()]

#}



params_dict = {

    "nb_hidden": 1000,

    "activation": "relu",
@@ -178,25 +163,13 @@ params_dict = { 
    "nesterov": False,

    "decay": 1e-4,

    "batch_size": 64,

    "nb_epoch": 100,

    "nb_epoch": 1,

    "init": "glorot_uniform",

    "nb_layers": 1,

    "batchnorm": False,

    "data_shape": train_dataset.get_data_shape()

}



#def keras_multitask_model_builder(tasks, task_types, params_dict, model_dir, logdir=None,

#                                  verbosity=None):

#  return MultiTaskDNN(tasks, task_types, params_dict, model_dir,

#                      verbosity=verbosity)

#optimizer = HyperparamOpt(keras_multitask_model_builder, MUV_tasks, MUV_task_types,

#                          verbosity=verbosity)

#

#best_dnn, best_dnn_hyperparams, all_dnn_results = \

#    optimizer.hyperparam_search(

#        params_dict, train_dataset, valid_dataset, output_transformers,

#        classification_metric, logdir=model_dir, use_max=True)



model = MultiTaskDNN(MUV_tasks, MUV_task_types, params_dict, model_dir,

                    verbosity=verbosity)
@@ -204,7 +177,14 @@ model = MultiTaskDNN(MUV_tasks, MUV_task_types, params_dict, model_dir, 
model.fit(train_dataset)

model.save()



evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

scores = evaluator.compute_model_performance([classification_metric])

train_evaluator = Evaluator(model, train_dataset, transformers, verbosity=verbosity)

train_scores = train_evaluator.compute_model_performance([classification_metric])



print("Train scores")

print(train_scores)



valid_evaluator = Evaluator(model, valid_dataset, transformers, verbosity=verbosity)

valid_scores = valid_evaluator.compute_model_performance([classification_metric])



print(scores)
 
print("Validation scores")

print(valid_scores)

CRA Git | Maintained and supported by SUSTech CRA and CCSE