Commit 9490fb3e authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Simplifying tests

parent 5ac4c88e

deepchem/metrics/__init__.py

+27 −27
Original line number Diff line number Diff line
@@ -135,15 +135,15 @@ class Metric(object): 
      A numpy array containing metric values for each task.

    """

    assert y_true.shape[0] == y_pred.shape[0] == w.shape[0]

    ######## DEBUG

    #from deepchem.metrics import to_one_hot

    #import sklearn 

    print("compute_metric")

    print("y_true.shape, y_pred.shape")

    print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    #print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ######## DEBUG

    ######### DEBUG

    ##from deepchem.metrics import to_one_hot

    ##import sklearn 

    #print("compute_metric")

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    ##print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    ##print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ######### DEBUG

    n_samples, n_tasks = y_true.shape[0], y_true.shape[1] 

    if self.mode == "classification":

      y_pred = np.reshape(y_pred, (n_samples, n_tasks, n_classes))
@@ -191,14 +191,14 @@ class Metric(object): 
    """

    y_true = np.array(np.squeeze(y_true[w != 0]))

    y_pred = np.array(np.squeeze(y_pred[w != 0]))

    ####### DEBUG

    import sklearn 

    print("compute_singletask_metric")

    print("y_true.shape, y_pred.shape")

    print(y_true.shape, y_pred.shape)

    print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ####### DEBUG

    ######## DEBUG

    #import sklearn 

    #print("compute_singletask_metric")

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    #print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ######## DEBUG

    if len(y_true.shape) == 0:

      n_samples = 1

    else:
@@ -231,16 +231,16 @@ class Metric(object): 
    else:

      y_pred = np.reshape(y_pred, (n_samples,))



    ####### DEBUG

    import sklearn 

    print("compute_singletask_metric after classification adjustments")

    print("self.mode, self.name, n_classes")

    print(self.mode, self.name, n_classes)

    print("y_true.shape, y_pred.shape")

    print(y_true.shape, y_pred.shape)

    print("sklearn.metrics.roc_auc_score(y_true, y_pred)")

    print(sklearn.metrics.roc_auc_score(y_true, y_pred))

    ####### DEBUG

    ######## DEBUG

    #import sklearn 

    #print("compute_singletask_metric after classification adjustments")

    #print("self.mode, self.name, n_classes")

    #print(self.mode, self.name, n_classes)

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(y_true, y_pred)")

    #print(sklearn.metrics.roc_auc_score(y_true, y_pred))

    ######## DEBUG

      

    if self.threshold is not None:

      y_pred = np.greater(y_pred, threshold)

deepchem/models/__init__.py

+0 −10
Original line number Diff line number Diff line
@@ -162,11 +162,6 @@ class Model(object): 
    n_tasks = len(self.tasks)

    for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):

      y_pred_batch = self.predict_proba_on_batch(X_batch)

      ######## DEBUG

      print("predict_proba()")

      print("y_pred_batch.shape")

      print(y_pred_batch.shape)

      ######## DEBUG

      batch_size = len(y_batch)

      y_pred_batch = np.squeeze(

          np.reshape(y_pred_batch, (batch_size, n_tasks, n_classes)))
@@ -178,11 +173,6 @@ class Model(object): 
    n_samples, n_tasks = len(dataset), len(self.tasks)

    y_pred = y_pred[:n_samples]

    y_pred = np.reshape(y_pred, (n_samples, n_tasks, n_classes))

    ######## DEBUG

    print("predict_proba()")

    print("y_pred.shape")

    print(y_pred.shape)

    ######## DEBUG

    return y_pred



  def get_task_type(self):

deepchem/models/test/test_overfit.py

+20 −40
Original line number Diff line number Diff line
@@ -68,10 +68,7 @@ class TestOverfitAPI(TestAPI): 
    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [regression_metric], csv_out.name, stats_out)

    scores = evaluator.compute_model_performance([regression_metric])



    assert scores[regression_metric.name] > .7
@@ -113,14 +110,11 @@ class TestOverfitAPI(TestAPI): 
    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)

    scores = evaluator.compute_model_performance([classification_metric])



    assert scores[classification_metric.name] > .9



  def test_sklearn_sparse_classification_overfit(self):

  def test_sklearn_skewed_classification_overfit(self):

    """Test sklearn models can overfit 0/1 datasets with few actives."""

    tasks = ["task0"]

    task_types = {task: "classification" for task in tasks}
@@ -136,11 +130,6 @@ class TestOverfitAPI(TestAPI): 
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    ######## DEBUG

    print("np.count_nonzero(y)")

    print(np.count_nonzero(y))

    ######## DEBUG

  

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {
@@ -164,10 +153,7 @@ class TestOverfitAPI(TestAPI): 
    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)

    scores = evaluator.compute_model_performance([classification_metric])



    assert scores[classification_metric.name] > .9
@@ -218,10 +204,7 @@ class TestOverfitAPI(TestAPI): 
    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [regression_metric], csv_out.name, stats_out)

    scores = evaluator.compute_model_performance([regression_metric])



    assert scores[regression_metric.name] > .7
@@ -325,20 +308,19 @@ class TestOverfitAPI(TestAPI): 


    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])

    print("y_pred_proba_model.shape")

    print(y_pred_proba_model.shape)

    ######### DEBUG

    #print("y_pred_proba_model.shape")

    #print(y_pred_proba_model.shape)

    ######### DEBUG



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)

    scores = evaluator.compute_model_performance([classification_metric])



    assert scores[classification_metric.name] > .9



  def test_keras_sparse_classification_overfit(self):

  def test_keras_skewed_classification_overfit(self):

    """Test keras models can overfit 0/1 datasets with few actives."""

    tasks = ["task0"]

    task_types = {task: "classification" for task in tasks}
@@ -384,16 +366,15 @@ class TestOverfitAPI(TestAPI): 


    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])

    print("y_pred_proba_model.shape")

    print(y_pred_proba_model.shape)

    #print("y_pred_proba_model.shape")

    #print(y_pred_proba_model.shape)



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)

    scores = evaluator.compute_model_performance([classification_metric])



    assert scores[classification_metric.name] > .9



  def test_tf_classification_overfit(self):

    """Test that tensorflow models can overfit simple classification datasets."""
@@ -448,14 +429,11 @@ class TestOverfitAPI(TestAPI): 
    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)

    scores = evaluator.compute_model_performance([classification_metric])



    assert scores[classification_metric.name] > .9



  def test_tf_sparse_classification_overfit(self):

  def test_tf_skewed_classification_overfit(self):

    """Test tensorflow models can overfit 0/1 datasets with few actives."""

    tasks = ["task0"]

    task_types = {task: "classification" for task in tasks}
@@ -513,4 +491,6 @@ class TestOverfitAPI(TestAPI): 
        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)



    print("scores")

    print(scores)

    assert scores[classification_metric.name] > .8

deepchem/utils/evaluate.py

+7 −7
Original line number Diff line number Diff line
@@ -76,13 +76,13 @@ class Evaluator(object): 
      y_pred = self.model.predict(self.dataset, self.transformers)

    multitask_scores = {}



    ######## DEBUG

    #from deepchem.metrics import to_one_hot

    print("y.shape, y_pred.shape")

    print(y.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred)")

    #print(sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred))

    ######## DEBUG

    ######### DEBUG

    ##from deepchem.metrics import to_one_hot

    #print("y.shape, y_pred.shape")

    #print(y.shape, y_pred.shape)

    ##print("sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred)")

    ##print(sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred))

    ######### DEBUG



    if csv_out is not None:

      log("Saving predictions to %s" % csv_out, self.verbosity)

examples/muv_keras_models.py

+50 −22
Original line number Diff line number Diff line
@@ -153,30 +153,58 @@ classification_metric = Metric(metrics.roc_auc_score, np.mean, 
                               verbosity=verbosity,

                               mode="classification")



#params_dict = {

#    "nb_hidden": [1000],

#    "activation": ["relu"],

#    "dropout": [.25],

#    "learning_rate": [.001],

#    "momentum": [.9],

#    "nesterov": [False],

#    "decay": [1e-4],

#    "batch_size": [64],

#    "nb_epoch": [100],

#    "init": ["glorot_uniform"],

#    "nb_layers": [1],

#    "batchnorm": [False],

#    "data_shape": [train_dataset.get_data_shape()]

#}



params_dict = {

    "nb_hidden": [1000],

    "activation": ["relu"],

    "dropout": [.25],

    "learning_rate": [.001],

    "momentum": [.9],

    "nesterov": [False],

    "decay": [1e-4],

    "batch_size": [64],

    "nb_epoch": [10],

    "init": ["glorot_uniform"],

    "nb_layers": [1],

    "batchnorm": [False],

    "data_shape": [train_dataset.get_data_shape()]

    "nb_hidden": 1000,

    "activation": "relu",

    "dropout": .25,

    "learning_rate": .001,

    "momentum": .9,

    "nesterov": False,

    "decay": 1e-4,

    "batch_size": 64,

    "nb_epoch": 100,

    "init": "glorot_uniform",

    "nb_layers": 1,

    "batchnorm": False,

    "data_shape": train_dataset.get_data_shape()

}



def keras_multitask_model_builder(tasks, task_types, params_dict, model_dir, logdir=None,

                                  verbosity=None):

  return MultiTaskDNN(tasks, task_types, params_dict, model_dir,

                      verbosity=verbosity)

optimizer = HyperparamOpt(keras_multitask_model_builder, MUV_tasks, MUV_task_types,

#def keras_multitask_model_builder(tasks, task_types, params_dict, model_dir, logdir=None,

#                                  verbosity=None):

#  return MultiTaskDNN(tasks, task_types, params_dict, model_dir,

#                      verbosity=verbosity)

#optimizer = HyperparamOpt(keras_multitask_model_builder, MUV_tasks, MUV_task_types,

#                          verbosity=verbosity)

#

#best_dnn, best_dnn_hyperparams, all_dnn_results = \

#    optimizer.hyperparam_search(

#        params_dict, train_dataset, valid_dataset, output_transformers,

#        classification_metric, logdir=model_dir, use_max=True)



model = MultiTaskDNN(MUV_tasks, MUV_task_types, params_dict, model_dir,

                    verbosity=verbosity)



best_dnn, best_dnn_hyperparams, all_dnn_results = \

    optimizer.hyperparam_search(

        params_dict, train_dataset, valid_dataset, output_transformers,

        classification_metric, logdir=model_dir, use_max=True)
 
# Fit trained model

model.fit(train_dataset)

model.save()



evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

scores = evaluator.compute_model_performance([classification_metric])



print(scores)

CRA Git | Maintained and supported by SUSTech CRA and CCSE