Commit c5a81aa6 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Merge pull request #178 from rbharath/multitask_overfit 

Multitask Overfit tests
parents 9f3b9c03 ee24ea41

deepchem/models/multitask.py

+8 −2
Original line number Diff line number Diff line
@@ -61,11 +61,17 @@ class SingletaskToMultitask(Model): 
    """

    Concatenates results from all singletask models.

    """

    N_tasks = len(self.tasks)

    n_tasks = len(self.tasks)

    n_samples = X.shape[0]

    y_pred = np.zeros((n_samples, N_tasks))

    y_pred = np.zeros((n_samples, n_tasks))

    for ind, task in enumerate(self.tasks):

      task_type = self.task_types[task]

      if task_type == "classification":

        y_pred[:, ind] = self.models[task].predict_on_batch(X)[:, 0]

      elif task_type == "regression":

        y_pred[:, ind] = self.models[task].predict_on_batch(X)

      else:

        raise ValueError("Invalid task_type")

    return y_pred



  def predict_proba_on_batch(self, X, n_classes=2):

deepchem/models/sklearn_models/__init__.py

+1 −1
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ class SklearnModel(Model): 
    Fits SKLearn model to data.

    """

    X, y, w, _ = dataset.to_numpy()

    y, w = y.flatten(), w.flatten()

    y, w = np.squeeze(y), np.squeeze(w)

    self.raw_model.fit(X, y, w)

    y_pred_raw = self.raw_model.predict(X)

deepchem/models/tensorflow_models/fcnet.py

+1 −1
Original line number Diff line number Diff line
@@ -136,7 +136,7 @@ class TensorflowMultiTaskClassifier(TensorflowClassifier): 
        prev_layer_size = layer_sizes[i]



      self.output = model_ops.MultitaskLogits(

          layer, self.model_params["num_classification_tasks"])

          layer, self.num_tasks)



  def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):

    """Construct a feed dictionary from minibatch data.

deepchem/models/tests/test_overfit.py

+294 −54
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ from deepchem.models.keras_models.fcnet import MultiTaskDNN 
from deepchem.models.tensorflow_models import TensorflowModel

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskRegressor

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

from deepchem.models.multitask import SingletaskToMultitask



class TestOverfitAPI(TestAPI):

  """
@@ -142,9 +143,6 @@ class TestOverfitAPI(TestAPI): 
    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
@@ -194,8 +192,6 @@ class TestOverfitAPI(TestAPI): 
    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
@@ -247,8 +243,6 @@ class TestOverfitAPI(TestAPI): 
    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
@@ -298,9 +292,6 @@ class TestOverfitAPI(TestAPI): 
    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
@@ -351,9 +342,6 @@ class TestOverfitAPI(TestAPI): 
    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
@@ -408,9 +396,6 @@ class TestOverfitAPI(TestAPI): 
    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
@@ -466,9 +451,6 @@ class TestOverfitAPI(TestAPI): 
    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
@@ -486,25 +468,13 @@ class TestOverfitAPI(TestAPI): 
    

    tasks = ["task0"]

    task_types = {task: "classification" for task in tasks}

    #n_samples = 250

    #n_samples = 500

    #n_samples = 1000

    #n_samples = 2000

    #n_samples = 5000

    n_samples = 5120

    #n_features = 3

    n_features = 6

    n_tasks = len(tasks)

    n_classes = 2

    

    # Generate dummy dataset

    np.random.seed(123)

    #p = .002

    #p = .2

    #p = .1

    #p = .05

    #p = .01

    #p = .005

    p = .002

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)
@@ -512,14 +482,6 @@ class TestOverfitAPI(TestAPI): 
    w = np.ones((n_samples, n_tasks))

    print("np.count_nonzero(y)")

    print(np.count_nonzero(y))

    #w = np.random.binomial(1, p, size=(n_samples, n_tasks))

    #print("np.amin(w), np.amax(w)")

    #print(np.amin(w), np.amax(w))



    #print("y_nonzero.shape")

    #print(y_nonzero.shape)

    #print("np.count_nonzero(y_nonzero)")

    #print(np.count_nonzero(y_nonzero))

    ##### DEBUG

    y_flat, w_flat = np.squeeze(y), np.squeeze(w)

    y_nonzero = y_flat[w_flat != 0]
@@ -540,17 +502,7 @@ class TestOverfitAPI(TestAPI): 
      "dropouts": [.0],

      "learning_rate": 0.003,

      "momentum": .9,

      #"batch_size": n_samples,

      #"batch_size": n_samples/2,

      #"batch_size": n_samples/4,

      #"batch_size": n_samples/8,

      #"batch_size": n_samples/16,

      #"batch_size": n_samples/32,

      #"batch_size": n_samples/64,

      "batch_size": 75,

      # TODO(rbharath): Is there a bug in the padding code? Why does it fail to

      # learn for non-multiples?

      #"batch_size": 600,

      "num_classification_tasks": 1,

      "num_classes": n_classes,

      "num_features": n_features,
@@ -561,8 +513,6 @@ class TestOverfitAPI(TestAPI): 
      "optimizer": "adam",

      "data_shape": dataset.get_data_shape()

    }

    print("model_params['batch_size']")

    print(model_params['batch_size'])



    verbosity = "high"

    classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)
@@ -575,12 +525,302 @@ class TestOverfitAPI(TestAPI): 
    model.fit(dataset)

    model.save()



    y_pred_model = model.predict(dataset, transformers=[])

    y_pred_proba_model = model.predict_proba(dataset, transformers=[])



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([classification_metric])



    assert scores[classification_metric.name] > .8



  def test_sklearn_multitask_classification_overfit(self):

    """Test SKLearn singletask-to-multitask overfits tiny data."""

    n_tasks = 10

    tasks = ["task%d" % task for task in range(n_tasks)]

    task_types = {task: "classification" for task in tasks}

    n_samples = 10

    n_features = 3

    

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.randint(2, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {

      "batch_size": None,

      "data_shape": dataset.get_data_shape()

    }



    verbosity = "high"

    classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)

    def model_builder(tasks, task_types, model_params, model_dir, verbosity=None):

      return SklearnModel(tasks, task_types, model_params, model_dir,

                          mode="classification",

                          model_instance=RandomForestClassifier(),

                          verbosity=verbosity)

    model = SingletaskToMultitask(tasks, task_types, model_params, self.model_dir,

                                  model_builder, verbosity=verbosity)



    # Fit trained model

    model.fit(dataset)

    model.save()



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([classification_metric])



    assert scores[classification_metric.name] > .9



  def test_keras_multitask_classification_overfit(self):

    """Test keras multitask overfits tiny data."""

    n_tasks = 10

    tasks = ["task%d" % task for task in range(n_tasks)]

    task_types = {task: "classification" for task in tasks}

    n_samples = 10

    n_features = 3

    

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.randint(2, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {

        "nb_hidden": 1000,

        "activation": "relu",

        "dropout": .0,

        "learning_rate": .15,

        "momentum": .9,

        "nesterov": False,

        "decay": 1e-4,

        "batch_size": n_samples,

        "nb_epoch": 200,

        "init": "glorot_uniform",

        "nb_layers": 1,

        "batchnorm": False,

        "data_shape": dataset.get_data_shape()

    }



    verbosity = "high"

    classification_metric = Metric(metrics.roc_auc_score, verbosity=verbosity)

    model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir,

                         verbosity=verbosity)



    # Fit trained model

    model.fit(dataset)

    model.save()



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([classification_metric])



    assert scores[classification_metric.name] > .9



  def test_tf_multitask_classification_overfit(self):

    """Test tf multitask overfits tiny data."""

    n_tasks = 10

    tasks = ["task%d" % task for task in range(n_tasks)]

    task_types = {task: "classification" for task in tasks}

    n_samples = 10

    n_features = 3

    n_classes = 2

    

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    #y = np.random.randint(n_classes, size=(n_samples, n_tasks))

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {

      "layer_sizes": [1000],

      "dropouts": [.0],

      "learning_rate": 0.0003,

      "momentum": .9,

      "batch_size": n_samples,

      "num_classification_tasks": n_tasks,

      "num_classes": n_classes,

      "num_features": n_features,

      "weight_init_stddevs": [.1],

      "bias_init_consts": [1.],

      "nb_epoch": 100,

      "penalty": 0.0,

      "optimizer": "adam",

      "data_shape": dataset.get_data_shape()

    }



    verbosity = "high"

    classification_metric = Metric(metrics.accuracy_score, verbosity=verbosity)

    model = TensorflowModel(

        tasks, task_types, model_params, self.model_dir,

        tf_class=TensorflowMultiTaskClassifier,

        verbosity=verbosity)



    # Fit trained model

    model.fit(dataset)

    model.save()



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([classification_metric])



    assert scores[classification_metric.name] > .9



  def test_sklearn_multitask_regression_overfit(self):

    """Test SKLearn singletask-to-multitask overfits tiny regression data."""

    n_tasks = 2

    tasks = ["task%d" % task for task in range(n_tasks)]

    task_types = {task: "regression" for task in tasks}

    n_samples = 10

    n_features = 3

    

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.rand(n_samples, n_tasks)

    w = np.ones((n_samples, n_tasks))



    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {

      "batch_size": None,

      "data_shape": dataset.get_data_shape()

    }



    verbosity = "high"

    regression_metric = Metric(metrics.r2_score, verbosity=verbosity)

    def model_builder(tasks, task_types, model_params, model_dir, verbosity=None):

      return SklearnModel(tasks, task_types, model_params, model_dir,

                          mode="regression",

                          model_instance=RandomForestRegressor(),

                          verbosity=verbosity)

    model = SingletaskToMultitask(tasks, task_types, model_params, self.model_dir,

                                  model_builder, verbosity=verbosity)





    # Fit trained model

    model.fit(dataset)

    model.save()



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([regression_metric])



    assert scores[regression_metric.name] > .7



  def test_keras_multitask_regression_overfit(self):

    """Test keras multitask overfits tiny data."""

    n_tasks = 10

    tasks = ["task%d" % task for task in range(n_tasks)]

    task_types = {task: "regression" for task in tasks}

    n_samples = 10

    n_features = 3

    

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.randint(2, size=(n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {

        "nb_hidden": 1000,

        "activation": "relu",

        "dropout": .0,

        "learning_rate": .15,

        "momentum": .9,

        "nesterov": False,

        "decay": 1e-4,

        "batch_size": n_samples,

        "nb_epoch": 200,

        "init": "glorot_uniform",

        "nb_layers": 1,

        "batchnorm": False,

        "data_shape": dataset.get_data_shape()

    }



    verbosity = "high"

    regression_metric = Metric(metrics.r2_score, verbosity=verbosity)

    model = MultiTaskDNN(tasks, task_types, model_params, self.model_dir,

                         verbosity=verbosity)



    # Fit trained model

    model.fit(dataset)

    model.save()



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([regression_metric])



    assert scores[regression_metric.name] > .9



  def test_tf_multitask_regression_overfit(self):

    """Test tf multitask overfits tiny data."""

    n_tasks = 10

    tasks = ["task%d" % task for task in range(n_tasks)]

    task_types = {task: "regression" for task in tasks}

    n_samples = 10

    n_features = 3

    n_classes = 2

    

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    #y = np.random.randint(n_classes, size=(n_samples, n_tasks))

    y = np.zeros((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

  

    dataset = Dataset.from_numpy(self.train_dir, tasks, X, y, w, ids)



    model_params = {

      "layer_sizes": [1000],

      "dropouts": [.0],

      "learning_rate": 0.0003,

      "momentum": .9,

      "batch_size": n_samples,

      "num_regression_tasks": n_tasks,

      "num_classes": n_classes,

      "num_features": n_features,

      "weight_init_stddevs": [.1],

      "bias_init_consts": [1.],

      "nb_epoch": 100,

      "penalty": 0.0,

      "optimizer": "adam",

      "data_shape": dataset.get_data_shape()

    }



    verbosity = "high"

    regression_metric = Metric(metrics.r2_score, verbosity=verbosity)

    model = TensorflowModel(

        tasks, task_types, model_params, self.model_dir,

        tf_class=TensorflowMultiTaskRegressor,

        verbosity=verbosity)



    # Fit trained model

    model.fit(dataset)

    model.save()



    # Eval model on train

    transformers = []

    evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)

    scores = evaluator.compute_model_performance([regression_metric])



    assert scores[regression_metric.name] > .9

CRA Git | Maintained and supported by SUSTech CRA and CCSE