Loading deepchem/hyperparameters/tests/test_hyperparam_opt.py +73 −10 Original line number Diff line number Diff line Loading @@ -24,6 +24,10 @@ from deepchem.models.multitask import SingletaskToMultitask from sklearn.linear_model import LogisticRegression from sklearn.ensemble import RandomForestRegressor from deepchem.datasets import Dataset from deepchem.hyperparameters import HyperparamOpt from deepchem.models.keras_models.fcnet import MultiTaskDNN from deepchem.models.tensorflow_models import TensorflowModel from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier def rf_model_builder(tasks, task_types, params_dict, model_dir, verbosity=None): """Builds random forests given hyperparameters. Loading Loading @@ -65,9 +69,10 @@ class TestHyperparamOptAPI(TestAPI): } metric = Metric(metrics.r2_score) self._hyperparam_opt(rf_model_builder, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, metric) optimizer = HyperparamOpt(rf_model_builder, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, metric, logdir=None) def test_singletask_to_multitask_sklearn_hyperparam_opt(self): """Test of hyperparam_opt with singletask_to_multitask.""" Loading Loading @@ -114,13 +119,15 @@ class TestHyperparamOptAPI(TestAPI): verbosity=None): return SingletaskToMultitask(tasks, task_types, params_dict, self.model_dir, model_builder) self._hyperparam_opt(multitask_model_builder, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, classification_metric) optimizer = HyperparamOpt(multitask_model_builder, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, classification_metric, logdir=None) def test_multitask_keras_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Keras multitask deepchem classification API.""" from deepchem.models.keras_models.fcnet import MultiTaskDNN splittype = "scaffold" output_transformers = [] input_transformers = [] Loading Loading @@ -154,6 +161,62 @@ class TestHyperparamOptAPI(TestAPI): "batchnorm": [False], "data_shape": [train_dataset.get_data_shape()]} self._hyperparam_opt(MultiTaskDNN, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, metric) optimizer = HyperparamOpt(MultiTaskDNN, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, metric, logdir=None) def test_multitask_tf_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Tensorflow multitask deepchem classification API.""" splittype = "scaffold" output_transformers = [] input_transformers = [] task_type = "classification" input_file = os.path.join(self.current_dir, "multitask_example.csv") tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] task_types = {task: task_type for task in tasks} compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] train_dataset, valid_dataset, _, transformers = self._featurize_train_test_split( splittype, compound_featurizers, complex_featurizers, input_transformers, output_transformers, input_file, tasks) metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification") params_dict = {"activation": ["relu"], "momentum": [.9], "batch_size": [50], "init": ["glorot_uniform"], "data_shape": [train_dataset.get_data_shape()], "learning_rate": [1e-3], "decay": [1e-6], "nb_hidden": [1000], "nb_epoch": [1], "nesterov": [False], "dropouts": [(.5,)], "nb_layers": [1], "batchnorm": [False], "layer_sizes": [(1000,)], "weight_init_stddevs": [(.1,)], "bias_init_consts": [(1.,)], "num_classes": [2], "penalty": [0.], "optimizer": ["sgd"], "num_classification_tasks": [len(task_types)] } def model_builder(tasks, task_types, params_dict, logdir, verbosity=None): return TensorflowModel( tasks, task_types, params_dict, logdir, tf_class=TensorflowMultiTaskClassifier, verbosity=verbosity) optimizer = HyperparamOpt(model_builder, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, metric, logdir=None) deepchem/hyperparameters/tests/test_tf_hyperparam_opt.py +0 −76 Original line number Diff line number Diff line """ Integration tests for hyperparam optimization. """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals __author__ = "Bharath Ramsundar" __copyright__ = "Copyright 2016, Stanford University" __license__ = "LGPL" import os from deepchem.featurizers.fingerprints import CircularFingerprint from deepchem.models.test import TestAPI from deepchem.metrics import Metric from deepchem import metrics import numpy as np from deepchem.models.tensorflow_models import TensorflowModel from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier class TestTFHyperparamOptAPI(TestAPI): """ Test hyperparameter optimization API. """ def test_multitask_tf_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Tensorflow multitask deepchem classification API.""" splittype = "scaffold" output_transformers = [] input_transformers = [] task_type = "classification" input_file = os.path.join(self.current_dir, "multitask_example.csv") tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] task_types = {task: task_type for task in tasks} compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] train_dataset, valid_dataset, _, transformers = self._featurize_train_test_split( splittype, compound_featurizers, complex_featurizers, input_transformers, output_transformers, input_file, tasks) metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification") params_dict = {"activation": ["relu"], "momentum": [.9], "batch_size": [50], "init": ["glorot_uniform"], "data_shape": [train_dataset.get_data_shape()], "learning_rate": [1e-3], "decay": [1e-6], "nb_hidden": [1000], "nb_epoch": [1], "nesterov": [False], "dropouts": [(.5,)], "nb_layers": [1], "batchnorm": [False], "layer_sizes": [(1000,)], "weight_init_stddevs": [(.1,)], "bias_init_consts": [(1.,)], "num_classes": [2], "penalty": [0.], "optimizer": ["sgd"], "num_classification_tasks": [len(task_types)] } def model_builder(tasks, task_types, params_dict, logdir, verbosity=None): return TensorflowModel( tasks, task_types, params_dict, logdir, tf_class=TensorflowMultiTaskClassifier, verbosity=verbosity) self._hyperparam_opt(model_builder, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, metric, logdir=self.model_dir) deepchem/models/test/__init__.py +24 −30 Original line number Diff line number Diff line Loading @@ -58,36 +58,30 @@ class TestAPI(unittest.TestCase): # debug. #shutil.rmtree(self.model_dir) def _hyperparam_opt(self, model_builder, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, metric, logdir=None): optimizer = HyperparamOpt(model_builder, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, metric, logdir=logdir) def _create_model(self, train_dataset, test_dataset, model, transformers, metrics): """Helper method to create model for test.""" # Fit trained model model.fit(train_dataset) model.save() # Eval model on train evaluator = Evaluator(model, train_dataset, transformers, verbosity=True) with tempfile.NamedTemporaryFile() as train_csv_out: with tempfile.NamedTemporaryFile() as train_stats_out: _ = evaluator.compute_model_performance( metrics, train_csv_out.name, train_stats_out) # Eval model on test evaluator = Evaluator(model, test_dataset, transformers, verbosity=True) with tempfile.NamedTemporaryFile() as test_csv_out: with tempfile.NamedTemporaryFile() as test_stats_out: _ = evaluator.compute_model_performance( metrics, test_csv_out.name, test_stats_out) #def _hyperparam_opt(self, model_builder, params_dict, train_dataset, # valid_dataset, output_transformers, tasks, task_types, metric, # logdir=None): # optimizer = HyperparamOpt(model_builder, tasks, task_types, verbosity="low") # best_model, best_hyperparams, all_results = optimizer.hyperparam_search( # params_dict, train_dataset, valid_dataset, output_transformers, # metric, logdir=logdir) #def _create_model(self, train_dataset, test_dataset, model, transformers, # metrics): # """Helper method to create model for test.""" # # Fit trained model # model.fit(train_dataset) # model.save() # # Eval model on train # evaluator = Evaluator(model, train_dataset, transformers, verbosity=True) # _ = evaluator.compute_model_performance(metrics) # # Eval model on test # evaluator = Evaluator(model, test_dataset, transformers, verbosity=True) # _ = evaluator.compute_model_performance(metrics) def _featurize_train_test_split(self, splittype, compound_featurizers, complex_featurizers, Loading deepchem/models/test/test_api.py +199 −82 File changed.Preview size limit exceeded, changes collapsed. Show changes deepchem/models/test/test_singletask_to_multitask.py +15 −2 Original line number Diff line number Diff line Loading @@ -18,6 +18,7 @@ from deepchem.featurizers.fingerprints import CircularFingerprint from deepchem.models.multitask import SingletaskToMultitask from deepchem.models.sklearn_models import SklearnModel from sklearn.linear_model import LogisticRegression from deepchem.utils.evaluate import Evaluator class TestSingletasktoMultitaskAPI(TestAPI): """ Loading Loading @@ -64,5 +65,17 @@ class TestSingletasktoMultitaskAPI(TestAPI): model_instance=LogisticRegression()) multitask_model = SingletaskToMultitask(tasks, task_types, params_dict, self.model_dir, model_builder) self._create_model(train_dataset, test_dataset, multitask_model, output_transformers, classification_metrics) # Fit trained model multitask_model.fit(train_dataset) multitask_model.save() # Eval multitask_model on train evaluator = Evaluator(multitask_model, train_dataset, output_transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics) # Eval multitask_model on test evaluator = Evaluator(multitask_model, test_dataset, output_transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics) Loading
deepchem/hyperparameters/tests/test_hyperparam_opt.py +73 −10 Original line number Diff line number Diff line Loading @@ -24,6 +24,10 @@ from deepchem.models.multitask import SingletaskToMultitask from sklearn.linear_model import LogisticRegression from sklearn.ensemble import RandomForestRegressor from deepchem.datasets import Dataset from deepchem.hyperparameters import HyperparamOpt from deepchem.models.keras_models.fcnet import MultiTaskDNN from deepchem.models.tensorflow_models import TensorflowModel from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier def rf_model_builder(tasks, task_types, params_dict, model_dir, verbosity=None): """Builds random forests given hyperparameters. Loading Loading @@ -65,9 +69,10 @@ class TestHyperparamOptAPI(TestAPI): } metric = Metric(metrics.r2_score) self._hyperparam_opt(rf_model_builder, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, metric) optimizer = HyperparamOpt(rf_model_builder, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, metric, logdir=None) def test_singletask_to_multitask_sklearn_hyperparam_opt(self): """Test of hyperparam_opt with singletask_to_multitask.""" Loading Loading @@ -114,13 +119,15 @@ class TestHyperparamOptAPI(TestAPI): verbosity=None): return SingletaskToMultitask(tasks, task_types, params_dict, self.model_dir, model_builder) self._hyperparam_opt(multitask_model_builder, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, classification_metric) optimizer = HyperparamOpt(multitask_model_builder, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, classification_metric, logdir=None) def test_multitask_keras_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Keras multitask deepchem classification API.""" from deepchem.models.keras_models.fcnet import MultiTaskDNN splittype = "scaffold" output_transformers = [] input_transformers = [] Loading Loading @@ -154,6 +161,62 @@ class TestHyperparamOptAPI(TestAPI): "batchnorm": [False], "data_shape": [train_dataset.get_data_shape()]} self._hyperparam_opt(MultiTaskDNN, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, metric) optimizer = HyperparamOpt(MultiTaskDNN, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, metric, logdir=None) def test_multitask_tf_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Tensorflow multitask deepchem classification API.""" splittype = "scaffold" output_transformers = [] input_transformers = [] task_type = "classification" input_file = os.path.join(self.current_dir, "multitask_example.csv") tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] task_types = {task: task_type for task in tasks} compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] train_dataset, valid_dataset, _, transformers = self._featurize_train_test_split( splittype, compound_featurizers, complex_featurizers, input_transformers, output_transformers, input_file, tasks) metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification") params_dict = {"activation": ["relu"], "momentum": [.9], "batch_size": [50], "init": ["glorot_uniform"], "data_shape": [train_dataset.get_data_shape()], "learning_rate": [1e-3], "decay": [1e-6], "nb_hidden": [1000], "nb_epoch": [1], "nesterov": [False], "dropouts": [(.5,)], "nb_layers": [1], "batchnorm": [False], "layer_sizes": [(1000,)], "weight_init_stddevs": [(.1,)], "bias_init_consts": [(1.,)], "num_classes": [2], "penalty": [0.], "optimizer": ["sgd"], "num_classification_tasks": [len(task_types)] } def model_builder(tasks, task_types, params_dict, logdir, verbosity=None): return TensorflowModel( tasks, task_types, params_dict, logdir, tf_class=TensorflowMultiTaskClassifier, verbosity=verbosity) optimizer = HyperparamOpt(model_builder, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, metric, logdir=None)
deepchem/hyperparameters/tests/test_tf_hyperparam_opt.py +0 −76 Original line number Diff line number Diff line """ Integration tests for hyperparam optimization. """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals __author__ = "Bharath Ramsundar" __copyright__ = "Copyright 2016, Stanford University" __license__ = "LGPL" import os from deepchem.featurizers.fingerprints import CircularFingerprint from deepchem.models.test import TestAPI from deepchem.metrics import Metric from deepchem import metrics import numpy as np from deepchem.models.tensorflow_models import TensorflowModel from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier class TestTFHyperparamOptAPI(TestAPI): """ Test hyperparameter optimization API. """ def test_multitask_tf_mlp_ECFP_classification_hyperparam_opt(self): """Straightforward test of Tensorflow multitask deepchem classification API.""" splittype = "scaffold" output_transformers = [] input_transformers = [] task_type = "classification" input_file = os.path.join(self.current_dir, "multitask_example.csv") tasks = ["task0", "task1", "task2", "task3", "task4", "task5", "task6", "task7", "task8", "task9", "task10", "task11", "task12", "task13", "task14", "task15", "task16"] task_types = {task: task_type for task in tasks} compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] train_dataset, valid_dataset, _, transformers = self._featurize_train_test_split( splittype, compound_featurizers, complex_featurizers, input_transformers, output_transformers, input_file, tasks) metric = Metric(metrics.matthews_corrcoef, np.mean, mode="classification") params_dict = {"activation": ["relu"], "momentum": [.9], "batch_size": [50], "init": ["glorot_uniform"], "data_shape": [train_dataset.get_data_shape()], "learning_rate": [1e-3], "decay": [1e-6], "nb_hidden": [1000], "nb_epoch": [1], "nesterov": [False], "dropouts": [(.5,)], "nb_layers": [1], "batchnorm": [False], "layer_sizes": [(1000,)], "weight_init_stddevs": [(.1,)], "bias_init_consts": [(1.,)], "num_classes": [2], "penalty": [0.], "optimizer": ["sgd"], "num_classification_tasks": [len(task_types)] } def model_builder(tasks, task_types, params_dict, logdir, verbosity=None): return TensorflowModel( tasks, task_types, params_dict, logdir, tf_class=TensorflowMultiTaskClassifier, verbosity=verbosity) self._hyperparam_opt(model_builder, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, metric, logdir=self.model_dir)
deepchem/models/test/__init__.py +24 −30 Original line number Diff line number Diff line Loading @@ -58,36 +58,30 @@ class TestAPI(unittest.TestCase): # debug. #shutil.rmtree(self.model_dir) def _hyperparam_opt(self, model_builder, params_dict, train_dataset, valid_dataset, output_transformers, tasks, task_types, metric, logdir=None): optimizer = HyperparamOpt(model_builder, tasks, task_types, verbosity="low") best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, output_transformers, metric, logdir=logdir) def _create_model(self, train_dataset, test_dataset, model, transformers, metrics): """Helper method to create model for test.""" # Fit trained model model.fit(train_dataset) model.save() # Eval model on train evaluator = Evaluator(model, train_dataset, transformers, verbosity=True) with tempfile.NamedTemporaryFile() as train_csv_out: with tempfile.NamedTemporaryFile() as train_stats_out: _ = evaluator.compute_model_performance( metrics, train_csv_out.name, train_stats_out) # Eval model on test evaluator = Evaluator(model, test_dataset, transformers, verbosity=True) with tempfile.NamedTemporaryFile() as test_csv_out: with tempfile.NamedTemporaryFile() as test_stats_out: _ = evaluator.compute_model_performance( metrics, test_csv_out.name, test_stats_out) #def _hyperparam_opt(self, model_builder, params_dict, train_dataset, # valid_dataset, output_transformers, tasks, task_types, metric, # logdir=None): # optimizer = HyperparamOpt(model_builder, tasks, task_types, verbosity="low") # best_model, best_hyperparams, all_results = optimizer.hyperparam_search( # params_dict, train_dataset, valid_dataset, output_transformers, # metric, logdir=logdir) #def _create_model(self, train_dataset, test_dataset, model, transformers, # metrics): # """Helper method to create model for test.""" # # Fit trained model # model.fit(train_dataset) # model.save() # # Eval model on train # evaluator = Evaluator(model, train_dataset, transformers, verbosity=True) # _ = evaluator.compute_model_performance(metrics) # # Eval model on test # evaluator = Evaluator(model, test_dataset, transformers, verbosity=True) # _ = evaluator.compute_model_performance(metrics) def _featurize_train_test_split(self, splittype, compound_featurizers, complex_featurizers, Loading
deepchem/models/test/test_api.py +199 −82 File changed.Preview size limit exceeded, changes collapsed. Show changes
deepchem/models/test/test_singletask_to_multitask.py +15 −2 Original line number Diff line number Diff line Loading @@ -18,6 +18,7 @@ from deepchem.featurizers.fingerprints import CircularFingerprint from deepchem.models.multitask import SingletaskToMultitask from deepchem.models.sklearn_models import SklearnModel from sklearn.linear_model import LogisticRegression from deepchem.utils.evaluate import Evaluator class TestSingletasktoMultitaskAPI(TestAPI): """ Loading Loading @@ -64,5 +65,17 @@ class TestSingletasktoMultitaskAPI(TestAPI): model_instance=LogisticRegression()) multitask_model = SingletaskToMultitask(tasks, task_types, params_dict, self.model_dir, model_builder) self._create_model(train_dataset, test_dataset, multitask_model, output_transformers, classification_metrics) # Fit trained model multitask_model.fit(train_dataset) multitask_model.save() # Eval multitask_model on train evaluator = Evaluator(multitask_model, train_dataset, output_transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics) # Eval multitask_model on test evaluator = Evaluator(multitask_model, test_dataset, output_transformers, verbosity=True) _ = evaluator.compute_model_performance(classification_metrics)
