Commit 5ac4c88e authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Sparse ROC-AUC tests pass

parent cc3ad573

deepchem/metrics/__init__.py

+13 −13
Original line number Diff line number Diff line
@@ -138,9 +138,9 @@ class Metric(object): 
    ######## DEBUG

    #from deepchem.metrics import to_one_hot

    #import sklearn 

    #print("compute_metric")

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    print("compute_metric")

    print("y_true.shape, y_pred.shape")

    print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred)")

    #print(sklearn.metrics.roc_auc_score(to_one_hot(y_true), y_pred))

    ######## DEBUG
@@ -231,16 +231,16 @@ class Metric(object): 
    else:

      y_pred = np.reshape(y_pred, (n_samples,))



    ######## DEBUG

    #import sklearn 

    #print("compute_singletask_metric after classification adjustments")

    #print("self.mode, self.name, n_classes")

    #print(self.mode, self.name, n_classes)

    #print("y_true.shape, y_pred.shape")

    #print(y_true.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(y_true, y_pred)")

    #print(sklearn.metrics.roc_auc_score(y_true, y_pred))

    ######## DEBUG

    ####### DEBUG

    import sklearn 

    print("compute_singletask_metric after classification adjustments")

    print("self.mode, self.name, n_classes")

    print(self.mode, self.name, n_classes)

    print("y_true.shape, y_pred.shape")

    print(y_true.shape, y_pred.shape)

    print("sklearn.metrics.roc_auc_score(y_true, y_pred)")

    print(sklearn.metrics.roc_auc_score(y_true, y_pred))

    ####### DEBUG

      

    if self.threshold is not None:

      y_pred = np.greater(y_pred, threshold)

deepchem/models/__init__.py

+17 −6
Original line number Diff line number Diff line
@@ -145,10 +145,12 @@ class Model(object): 
  

    # The iterbatches does padding with zero-weight examples on the last batch.

    # Remove padded examples.

    y_pred = y_pred[:len(dataset)]

    n_samples, n_tasks = len(dataset), len(self.tasks)

    y_pred = y_pred[:n_samples]

    y_pred = np.reshape(y_pred, (n_samples, n_tasks))

    return y_pred



  def predict_proba(self, dataset, transformers):

  def predict_proba(self, dataset, transformers, n_classes=2):

    """

    TODO: Do transformers even make sense here?
@@ -157,12 +159,14 @@ class Model(object): 
    """

    y_preds = []

    batch_size = self.model_params["batch_size"]

    n_classes = None

    n_tasks = len(self.tasks)

    for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):

      y_pred_batch = self.predict_proba_on_batch(X_batch)

      if n_classes is None:

        n_classes = y_pred_batch.shape[-1]

      ######## DEBUG

      print("predict_proba()")

      print("y_pred_batch.shape")

      print(y_pred_batch.shape)

      ######## DEBUG

      batch_size = len(y_batch)

      y_pred_batch = np.squeeze(

          np.reshape(y_pred_batch, (batch_size, n_tasks, n_classes)))
@@ -171,7 +175,14 @@ class Model(object): 
    y_pred = np.vstack(y_preds)

    # The iterbatches does padding with zero-weight examples on the last batch.

    # Remove padded examples.

    y_pred = y_pred[:len(dataset)]

    n_samples, n_tasks = len(dataset), len(self.tasks)

    y_pred = y_pred[:n_samples]

    y_pred = np.reshape(y_pred, (n_samples, n_tasks, n_classes))

    ######## DEBUG

    print("predict_proba()")

    print("y_pred.shape")

    print(y_pred.shape)

    ######## DEBUG

    return y_pred



  def get_task_type(self):

deepchem/models/sklearn_models/__init__.py

+5 −3
Original line number Diff line number Diff line
@@ -56,9 +56,11 @@ class SklearnModel(Model): 
    """

    Makes per-class predictions on batch of data.

    """

    return self.raw_model.predict_proba(X)



  def predict_proba_on_batch(self, X):

    ######## DEBUG

    print("predict_proba_on_batch()")

    print("self.raw_model.predict_proba(X).shape")

    print(self.raw_model.predict_proba(X).shape)

    ######## DEBUG

    return self.raw_model.predict_proba(X)



  def predict(self, X, transformers):

deepchem/models/test/test_overfit.py

+168 −0
Original line number Diff line number Diff line
@@ -120,6 +120,57 @@ class TestOverfitAPI(TestAPI): 


    assert scores[classification_metric.name] > .9



  def test_sklearn_sparse_classification_overfit(self):

    """Test sklearn models can overfit 0/1 datasets with few actives."""

    tasks = ["task0"]

    task_types = {task: "classification" for task in tasks}

    n_samples = 100

    n_features = 3

    n_tasks = len(tasks)

    

    # Generate dummy dataset

    np.random.seed(123)

    p = .05

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.binomial(1, p, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    ######## DEBUG

    print("np.count_nonzero(y)")

    print(np.count_nonzero(y))

    ######## DEBUG

  

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {

      "batch_size": None,

      "data_shape": dataset.get_data_shape()

    }



    verbosity = "high"

    classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)

    model = SklearnModel(tasks, task_types, model_params, self.model_dir,

                         mode="classification",

                         model_instance=RandomForestClassifier())



    # Fit trained model

    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)



    assert scores[classification_metric.name] > .9



  def test_keras_regression_overfit(self):

    """Test that keras models can overfit simple regression datasets."""

    tasks = ["task0"]
@@ -287,6 +338,63 @@ class TestOverfitAPI(TestAPI): 


    assert scores[classification_metric.name] > .9



  def test_keras_sparse_classification_overfit(self):

    """Test keras models can overfit 0/1 datasets with few actives."""

    tasks = ["task0"]

    task_types = {task: "classification" for task in tasks}

    n_samples = 100

    n_features = 3

    n_tasks = len(tasks)

    

    # Generate dummy dataset

    np.random.seed(123)

    p = .05

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.binomial(1, p, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {

        "nb_hidden": 1000,

        "activation": "relu",

        "dropout": .0,

        "learning_rate": .15,

        "momentum": .9,

        "nesterov": False,

        "decay": 1e-4,

        "batch_size": n_samples,

        "nb_epoch": 200,

        "init": "glorot_uniform",

        "nb_layers": 1,

        "batchnorm": False,

        "data_shape": dataset.get_data_shape()

    }



    verbosity = "high"

    classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)

    #classification_metric = Metric(metrics.recall_score, verbosity=verbosity)

    model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir,

                         verbosity=verbosity)



    # Fit trained model

    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])

    print("y_pred_proba_model.shape")

    print(y_pred_proba_model.shape)



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)



  def test_tf_classification_overfit(self):

    """Test that tensorflow models can overfit simple classification datasets."""

    tasks = ["task0"]
@@ -346,3 +454,63 @@ class TestOverfitAPI(TestAPI): 
            [classification_metric], csv_out.name, stats_out)



    assert scores[classification_metric.name] > .9



  def test_tf_sparse_classification_overfit(self):

    """Test tensorflow models can overfit 0/1 datasets with few actives."""

    tasks = ["task0"]

    task_types = {task: "classification" for task in tasks}

    n_samples = 100

    n_features = 3

    n_tasks = len(tasks)

    n_classes = 2

    

    # Generate dummy dataset

    np.random.seed(123)

    p = .05

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.binomial(1, p, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {

      "layer_sizes": [1500],

      "dropouts": [.0],

      "learning_rate": 0.001,

      "momentum": .9,

      "batch_size": n_samples,

      "num_classification_tasks": 1,

      "num_classes": n_classes,

      "num_features": n_features,

      "weight_init_stddevs": [.3],

      "bias_init_consts": [1.],

      "nb_epoch": 200,

      "penalty": 0.0,

      "optimizer": "adam",

      "data_shape": dataset.get_data_shape()

    }



    verbosity = "high"

    classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)

    model = TensorflowModel(

        tasks, task_types, model_params, self.model_dir,

        tf_class=TensorflowMultiTaskClassifier,

        verbosity=verbosity)



    # Fit trained model

    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    with tempfile.NamedTemporaryFile() as csv_out:

      with tempfile.NamedTemporaryFile() as stats_out:

        scores = evaluator.compute_model_performance(

            [classification_metric], csv_out.name, stats_out)



    assert scores[classification_metric.name] > .8

deepchem/utils/evaluate.py

+6 −2
Original line number Diff line number Diff line
@@ -48,6 +48,10 @@ class Evaluator(object): 
      csvfile: Open file object.

    """

    mol_ids = self.dataset.get_ids()

    ################ DEBUG

    print("len(y_preds), len(mol_ids)")

    print(len(y_preds), len(mol_ids))

    ################ DEBUG

    assert len(y_preds) == len(mol_ids)

    with open(csv_out, "wb") as csvfile:

      csvwriter = csv.writer(csvfile)
@@ -74,8 +78,8 @@ class Evaluator(object): 


    ######## DEBUG

    #from deepchem.metrics import to_one_hot

    #print("y.shape, y_pred.shape")

    #print(y.shape, y_pred.shape)

    print("y.shape, y_pred.shape")

    print(y.shape, y_pred.shape)

    #print("sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred)")

    #print(sklearn.metrics.roc_auc_score(to_one_hot(y), y_pred))

    ######## DEBUG

CRA Git | Maintained and supported by SUSTech CRA and CCSE