Commit e9a600c2 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Merge pull request #171 from rbharath/tests_refactor 

First round of refactoring for test suite
parents 1746304b b5c6aeac

deepchem/featurizers/tests/test_featurized_samples.py

+4 −4
Original line number Diff line number Diff line
@@ -81,7 +81,7 @@ class TestFeaturizedSamples(unittest.TestCase): 
    tasks = ["log-solubility"]

    task_type = "regression"

    task_types = {task: task_type for task in tasks}

    input_file = "../../models/test/example.csv"

    input_file = "../../models/tests/example.csv"

    train_samples, valid_samples, test_samples = (

        self._featurize_train_valid_test_split(

            splittype, input_file, tasks, frac_train=.8,
@@ -99,7 +99,7 @@ class TestFeaturizedSamples(unittest.TestCase): 
    tasks = ["log-solubility"]

    task_type = "regression"

    task_types = {task: task_type for task in tasks}

    input_file = "../../models/test/example.csv"

    input_file = "../../models/tests/example.csv"

    train_samples, test_samples = (

        self._featurize_train_valid_test_split(

            splittype, input_file, tasks, frac_train=.8,
@@ -116,7 +116,7 @@ class TestFeaturizedSamples(unittest.TestCase): 
    tasks = ["log-solubility"]

    task_type = "regression"

    task_types = {task: task_type for task in tasks}

    input_file = "../../models/test/example.csv"

    input_file = "../../models/tests/example.csv"

    train_samples, valid_samples, test_samples = (

        self._featurize_train_valid_test_split(

            splittype, input_file, tasks, frac_train=.8,
@@ -134,7 +134,7 @@ class TestFeaturizedSamples(unittest.TestCase): 
    tasks = ["log-solubility"]

    task_type = "regression"

    task_types = {task: task_type for task in tasks}

    input_file = "../../models/test/example.csv"

    input_file = "../../models/tests/example.csv"

    train_samples, test_samples = (

        self._featurize_train_valid_test_split(

            splittype, input_file, tasks, frac_train=.8,

deepchem/hyperparameters/tests/test_hyperparam_opt.py

+74 −11
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ import unittest 
import tempfile

import shutil

import numpy as np

from deepchem.models.test import TestAPI

from deepchem.models.tests import TestAPI

from deepchem.models.sklearn_models import SklearnModel

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.transformers import NormalizationTransformer
@@ -24,6 +24,10 @@ from deepchem.models.multitask import SingletaskToMultitask 
from sklearn.linear_model import LogisticRegression

from sklearn.ensemble import RandomForestRegressor 

from deepchem.datasets import Dataset

from deepchem.hyperparameters import HyperparamOpt

from deepchem.models.keras_models.fcnet import MultiTaskDNN

from deepchem.models.tensorflow_models import TensorflowModel

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier



def rf_model_builder(tasks, task_types, params_dict, model_dir, verbosity=None):

    """Builds random forests given hyperparameters.
@@ -65,9 +69,10 @@ class TestHyperparamOptAPI(TestAPI): 
    }

    metric = Metric(metrics.r2_score)



    self._hyperparam_opt(rf_model_builder, params_dict, train_dataset,

                         valid_dataset, output_transformers, tasks, task_types,

                         metric)

    optimizer = HyperparamOpt(rf_model_builder, tasks, task_types, verbosity="low")

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, output_transformers,

      metric, logdir=None)



  def test_singletask_to_multitask_sklearn_hyperparam_opt(self):

    """Test of hyperparam_opt with singletask_to_multitask."""
@@ -114,13 +119,15 @@ class TestHyperparamOptAPI(TestAPI): 
                                verbosity=None):

      return SingletaskToMultitask(tasks, task_types, params_dict,

                                   self.model_dir, model_builder)

    self._hyperparam_opt(multitask_model_builder, params_dict, train_dataset,

                         valid_dataset, output_transformers, tasks, task_types,

                         classification_metric)



    optimizer = HyperparamOpt(multitask_model_builder, tasks, task_types,

                              verbosity="low")

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, output_transformers,

      classification_metric, logdir=None)



  def test_multitask_keras_mlp_ECFP_classification_hyperparam_opt(self):

    """Straightforward test of Keras multitask deepchem classification API."""

    from deepchem.models.keras_models.fcnet import MultiTaskDNN

    splittype = "scaffold"

    output_transformers = []

    input_transformers = []
@@ -154,6 +161,62 @@ class TestHyperparamOptAPI(TestAPI): 
                  "batchnorm": [False],

                  "data_shape": [train_dataset.get_data_shape()]}

    

    self._hyperparam_opt(MultiTaskDNN, params_dict, train_dataset,

                         valid_dataset, output_transformers, tasks, task_types,

                         metric)
 
    optimizer = HyperparamOpt(MultiTaskDNN, tasks, task_types,

                              verbosity="low")

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, output_transformers,

      metric, logdir=None)



  def test_multitask_tf_mlp_ECFP_classification_hyperparam_opt(self):

    """Straightforward test of Tensorflow multitask deepchem classification API."""

    splittype = "scaffold"

    output_transformers = []

    input_transformers = []

    task_type = "classification"



    input_file = os.path.join(self.current_dir, "multitask_example.csv")

    tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",

             "task7", "task8", "task9", "task10", "task11", "task12",

             "task13", "task14", "task15", "task16"]

    task_types = {task: task_type for task in tasks}



    compound_featurizers = [CircularFingerprint(size=1024)]

    complex_featurizers = []



    train_dataset, valid_dataset, _, transformers = self._featurize_train_test_split(

        splittype, compound_featurizers, 

        complex_featurizers, input_transformers,

        output_transformers, input_file, tasks)

    metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification")

    params_dict = {"activation": ["relu"],

                    "momentum": [.9],

                    "batch_size": [50],

                    "init": ["glorot_uniform"],

                    "data_shape": [train_dataset.get_data_shape()],

                    "learning_rate": [1e-3],

                    "decay": [1e-6],

                    "nb_hidden": [1000], 

                    "nb_epoch": [1],

                    "nesterov": [False],

                    "dropouts": [(.5,)],

                    "nb_layers": [1],

                    "batchnorm": [False],

                    "layer_sizes": [(1000,)],

                    "weight_init_stddevs": [(.1,)],

                    "bias_init_consts": [(1.,)],

                    "num_classes": [2],

                    "penalty": [0.], 

                    "optimizer": ["sgd"],

                    "num_classification_tasks": [len(task_types)]

                  }



    def model_builder(tasks, task_types, params_dict, logdir, verbosity=None):

        return TensorflowModel(

            tasks, task_types, params_dict, logdir, 

            tf_class=TensorflowMultiTaskClassifier,

            verbosity=verbosity)

    optimizer = HyperparamOpt(model_builder, tasks, task_types,

                              verbosity="low")

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, output_transformers,

      metric, logdir=None)

deepchem/hyperparameters/tests/test_tf_hyperparam_opt.py

deleted100644 → 0
+0 −76
Original line number Diff line number Diff line 
"""

Integration tests for hyperparam optimization.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "LGPL"





import os

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.models.test import TestAPI

from deepchem.metrics import Metric

from deepchem import metrics

import numpy as np

from deepchem.models.tensorflow_models import TensorflowModel

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier



class TestTFHyperparamOptAPI(TestAPI):

  """

  Test hyperparameter optimization API.

  """

  def test_multitask_tf_mlp_ECFP_classification_hyperparam_opt(self):

    """Straightforward test of Tensorflow multitask deepchem classification API."""

    splittype = "scaffold"

    output_transformers = []

    input_transformers = []

    task_type = "classification"



    input_file = os.path.join(self.current_dir, "multitask_example.csv")

    tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6",

             "task7", "task8", "task9", "task10", "task11", "task12",

             "task13", "task14", "task15", "task16"]

    task_types = {task: task_type for task in tasks}



    compound_featurizers = [CircularFingerprint(size=1024)]

    complex_featurizers = []



    train_dataset, valid_dataset, _, transformers = self._featurize_train_test_split(

        splittype, compound_featurizers, 

        complex_featurizers, input_transformers,

        output_transformers, input_file, tasks)

    metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification")

    params_dict = {"activation": ["relu"],

                    "momentum": [.9],

                    "batch_size": [50],

                    "init": ["glorot_uniform"],

                    "data_shape": [train_dataset.get_data_shape()],

                    "learning_rate": [1e-3],

                    "decay": [1e-6],

                    "nb_hidden": [1000], 

                    "nb_epoch": [1],

                    "nesterov": [False],

                    "dropouts": [(.5,)],

                    "nb_layers": [1],

                    "batchnorm": [False],

                    "layer_sizes": [(1000,)],

                    "weight_init_stddevs": [(.1,)],

                    "bias_init_consts": [(1.,)],

                    "num_classes": [2],

                    "penalty": [0.], 

                    "optimizer": ["sgd"],

                    "num_classification_tasks": [len(task_types)]

                  }



    def model_builder(tasks, task_types, params_dict, logdir, verbosity=None):

        return TensorflowModel(

            tasks, task_types, params_dict, logdir, 

            tf_class=TensorflowMultiTaskClassifier,

            verbosity=verbosity)

    self._hyperparam_opt(model_builder, params_dict, train_dataset,

                         valid_dataset, output_transformers, tasks, task_types,

                         metric, logdir=self.model_dir)

deepchem/models/test/test_tf_api.py

deleted100644 → 0
+0 −87
Original line number Diff line number Diff line 
"""

Integration tests for singletask vector feature models.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "LGPL"



import os

import unittest

import tempfile

import shutil

from deepchem.featurizers.featurize import DataFeaturizer

from deepchem.featurizers.featurize import FeaturizedSamples

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.featurizers.basic import RDKitDescriptors

from deepchem.featurizers.nnscore import NNScoreComplexFeaturizer

from deepchem.featurizers.grid_featurizer import GridFeaturizer

from deepchem.datasets import Dataset

from deepchem.utils.evaluate import Evaluator

from deepchem.models import Model

from deepchem.models.sklearn_models import SklearnModel

from deepchem.models.tensorflow_models import TensorflowModel

from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier

from deepchem.transformers import NormalizationTransformer

from deepchem.transformers import LogTransformer

from deepchem.transformers import ClippingTransformer

from deepchem.models.test import TestAPI

from deepchem import metrics

from deepchem.metrics import Metric

from sklearn.ensemble import RandomForestRegressor



class TestTensorflowAPI(TestAPI):

  """

  Test top-level API for ML models."

  """



  def test_singletask_tf_mlp_ECFP_classification_API(self):

    """Straightforward test of Tensorflow singletask deepchem classification API."""

    splittype = "scaffold"

    output_transformers = []

    input_transformers = []

    task_type = "classification"



    compound_featurizers = [CircularFingerprint(size=1024)]

    complex_featurizers = []



    tasks = ["outcome"]

    task_type = "classification"

    task_types = {task: task_type for task in tasks}

    input_file = "example_classification.csv"

    input_transformers = []

    output_transformers = [NormalizationTransformer]



    train_dataset, test_dataset, _, transformers = self._featurize_train_test_split(

        splittype, compound_featurizers, 

        complex_featurizers, input_transformers,

        output_transformers, input_file, tasks)



    model_params = {

      "batch_size": 2,

      "num_classification_tasks": 1,

      "num_features": 1024,

      "layer_sizes": [1024],

      "weight_init_stddevs": [1.],

      "bias_init_consts": [0.],

      "dropouts": [.5],

      "num_classes": 2,

      "nb_epoch": 1,

      "penalty": 0.0,

      "optimizer": "adam",

      "learning_rate": .001,

      "data_shape": train_dataset.get_data_shape()

    }

    classification_metrics = [Metric(metrics.roc_auc_score),

                              Metric(metrics.matthews_corrcoef),

                              Metric(metrics.recall_score),

                              Metric(metrics.accuracy_score)]



    model = TensorflowModel(

        tasks, task_types, model_params, self.model_dir,

        tf_class=TensorflowMultiTaskClassifier)

    self._create_model(train_dataset, test_dataset, model, transformers,

                       classification_metrics)

deepchem/models/test/__init__.py→deepchem/models/tests/__init__.py

+24 −30
Original line number Diff line number Diff line
@@ -58,36 +58,30 @@ class TestAPI(unittest.TestCase): 
    # debug.

    #shutil.rmtree(self.model_dir)



  def _hyperparam_opt(self, model_builder, params_dict, train_dataset,

                      valid_dataset, output_transformers, tasks, task_types, metric,

                      logdir=None):



    optimizer = HyperparamOpt(model_builder, tasks, task_types, verbosity="low")

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, output_transformers,

      metric, logdir=logdir)



  def _create_model(self, train_dataset, test_dataset, model, transformers,

                    metrics):

    """Helper method to create model for test."""



    # Fit trained model

    model.fit(train_dataset)

    model.save()



    # Eval model on train

    evaluator = Evaluator(model, train_dataset, transformers, verbosity=True)

    with tempfile.NamedTemporaryFile() as train_csv_out:

      with tempfile.NamedTemporaryFile() as train_stats_out:

        _ = evaluator.compute_model_performance(

            metrics, train_csv_out.name, train_stats_out)



    # Eval model on test

    evaluator = Evaluator(model, test_dataset, transformers, verbosity=True)

    with tempfile.NamedTemporaryFile() as test_csv_out:

      with tempfile.NamedTemporaryFile() as test_stats_out:

        _ = evaluator.compute_model_performance(

            metrics, test_csv_out.name, test_stats_out)

  #def _hyperparam_opt(self, model_builder, params_dict, train_dataset,

  #                    valid_dataset, output_transformers, tasks, task_types, metric,

  #                    logdir=None):



  #  optimizer = HyperparamOpt(model_builder, tasks, task_types, verbosity="low")

  #  best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

  #    params_dict, train_dataset, valid_dataset, output_transformers,

  #    metric, logdir=logdir)



  #def _create_model(self, train_dataset, test_dataset, model, transformers,

  #                  metrics):

  #  """Helper method to create model for test."""



  #  # Fit trained model

  #  model.fit(train_dataset)

  #  model.save()



  #  # Eval model on train

  #  evaluator = Evaluator(model, train_dataset, transformers, verbosity=True)

  #  _ = evaluator.compute_model_performance(metrics)



  #  # Eval model on test

  #  evaluator = Evaluator(model, test_dataset, transformers, verbosity=True)

  #  _ = evaluator.compute_model_performance(metrics)



  def _featurize_train_test_split(self, splittype, compound_featurizers, 

                                  complex_featurizers,

CRA Git | Maintained and supported by SUSTech CRA and CCSE