Loading deepchem/hyper/base_classes.py +3 −0 Original line number Diff line number Diff line Loading @@ -80,6 +80,7 @@ class HyperparamOpt(object): valid_dataset: Dataset, metric: Metric, output_transformers: List[Transformer] = [], nb_epoch: int = 10, use_max: bool = True, logdir: Optional[str] = None, **kwargs) -> Tuple[Model, Dict, Dict]: Loading Loading @@ -110,6 +111,8 @@ class HyperparamOpt(object): `train_dataset` and `valid_dataset` may have been transformed for learning and need the transform to be inverted before the metric can be evaluated on a model. nb_epoch: int, (default 10) Specifies the number of training epochs during each iteration of optimization. use_max: bool, optional If True, return the model with the highest score. Else return model with the minimum score. Loading deepchem/hyper/gaussian_process.py +35 −4 Original line number Diff line number Diff line Loading @@ -132,6 +132,7 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): valid_dataset: Dataset, metric: Metric, output_transformers: List[Transformer] = [], nb_epoch: int = 10, use_max: bool = True, logdir: Optional[str] = None, max_iter: int = 20, Loading Loading @@ -160,6 +161,9 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): `train_dataset` and `valid_dataset` may have been transformed for learning and need the transform to be inverted before the metric can be evaluated on a model. nb_epoch: int, (default 10) Specifies the number of training epochs during each iteration of optimization. Not used by all model types. use_max: bool, (default True) Specifies whether to maximize or minimize `metric`. maximization(True) or minimization(False) Loading Loading @@ -230,14 +234,17 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): # Stores all model locations model_locations = {} # Demarcating internal function for readability def optimizing_function(**placeholders): def _optimize(nb_epoch, **placeholders): """Private Optimizing function Take in hyper parameter values and return valid set performances Take in hyper parameter values and number of training epochs. Return valid set performances. Parameters ---------- nb_epoch: int Number of epochs to train model being optimized during each iteration. Not used by all model types. placeholders: keyword arguments Should be various hyperparameters as specified in `param_keys` above. Loading Loading @@ -277,6 +284,10 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): # Add it on to the information needed for the constructor hyper_parameters["model_dir"] = model_dir model = self.model_builder(**hyper_parameters) try: model.fit(train_dataset, nb_epoch=nb_epoch) # Not all models have nb_epoch except TypeError: model.fit(train_dataset) try: model.save() Loading Loading @@ -305,6 +316,26 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): else: return -score # Demarcating internal function for readability def optimizing_function(**placeholders): """Wrapper function Take in hyper parameter values. Calls a private optimize function (_optimize) with number of epochs. Returns valid set performances. Parameters ---------- placeholders: keyword arguments Should be various hyperparameters as specified in `param_keys` above. Returns: -------- valid_scores: float valid set performances """ return _optimize(nb_epoch=nb_epoch, **placeholders) # execute GPGO cov = matern32() gp = GaussianProcess(cov) Loading deepchem/hyper/grid_search.py +10 −1 Original line number Diff line number Diff line Loading @@ -66,6 +66,7 @@ class GridHyperparamOpt(HyperparamOpt): valid_dataset: Dataset, metric: Metric, output_transformers: List[Transformer] = [], nb_epoch: int = 10, use_max: bool = True, logdir: Optional[str] = None, **kwargs, Loading @@ -91,6 +92,9 @@ class GridHyperparamOpt(HyperparamOpt): `train_dataset` and `valid_dataset` may have been transformed for learning and need the transform to be inverted before the metric can be evaluated on a model. nb_epoch: int, (default 10) Specifies the number of training epochs during each iteration of optimization. Not used by all model types. use_max: bool, optional If True, return the model with the highest score. Else return model with the minimum score. Loading Loading @@ -144,6 +148,11 @@ class GridHyperparamOpt(HyperparamOpt): model_dir = tempfile.mkdtemp() model_params['model_dir'] = model_dir model = self.model_builder(**model_params) # mypy test throws error, so ignoring it in try try: model.fit(train_dataset, nb_epoch=nb_epoch) # type: ignore # Not all models have nb_epoch except TypeError: model.fit(train_dataset) try: model.save() Loading deepchem/hyper/tests/test_gaussian_hyperparam_opt.py +37 −0 Original line number Diff line number Diff line Loading @@ -7,6 +7,7 @@ valuable test suite so leaving it in despite the flakiness. """ import numpy as np import sklearn import sklearn.ensemble import deepchem as dc import unittest import tempfile Loading Loading @@ -171,3 +172,39 @@ class TestGaussianHyperparamOpt(unittest.TestCase): assert "learning_rate" in hp_str assert valid_score["mean-mean_squared_error"] == min(all_results.values()) assert valid_score["mean-mean_squared_error"] > 0 @flaky def test_multitask_example_nb_epoch(self): """Test a simple example of optimizing a multitask model with a gaussian process search with a different number of training epochs.""" # Generate dummy dataset np.random.seed(123) train_dataset = dc.data.NumpyDataset( np.random.rand(10, 3), np.zeros((10, 2)), np.ones((10, 2)), np.arange(10)) valid_dataset = dc.data.NumpyDataset( np.random.rand(5, 3), np.zeros((5, 2)), np.ones((5, 2)), np.arange(5)) transformers = [] optimizer = dc.hyper.GaussianProcessHyperparamOpt( lambda **params: dc.models.MultitaskRegressor(n_tasks=2, n_features=3, dropouts=[0.], weight_init_stddevs=[np.sqrt(6) / np.sqrt(1000)], learning_rate=0.003, **params)) params_dict = {"batch_size": 10} metric = dc.metrics.Metric( dc.metrics.mean_squared_error, task_averager=np.mean) best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, metric, transformers, nb_epoch=3, max_iter=1, use_max=False) valid_score = best_model.evaluate(valid_dataset, [metric], transformers) assert valid_score["mean-mean_squared_error"] == min(all_results.values()) assert valid_score["mean-mean_squared_error"] > 0 deepchem/hyper/tests/test_grid_hyperparam_opt.py +36 −0 Original line number Diff line number Diff line Loading @@ -6,6 +6,7 @@ import tempfile import numpy as np import deepchem as dc import sklearn import sklearn.ensemble class TestGridHyperparamOpt(unittest.TestCase): Loading Loading @@ -158,3 +159,38 @@ class TestGridHyperparamOpt(unittest.TestCase): assert valid_score["mean-mean_squared_error"] == min(all_results.values()) assert valid_score["mean-mean_squared_error"] > 0 def test_multitask_nb_epoch(self): """Test a simple example of optimizing a multitask model with a grid search with a different number of training epochs.""" # Generate dummy dataset np.random.seed(123) train_dataset = dc.data.NumpyDataset( np.random.rand(10, 3), np.zeros((10, 2)), np.ones((10, 2)), np.arange(10)) valid_dataset = dc.data.NumpyDataset( np.random.rand(5, 3), np.zeros((5, 2)), np.ones((5, 2)), np.arange(5)) optimizer = dc.hyper.GridHyperparamOpt( lambda **params: dc.models.MultitaskRegressor(n_tasks=2, n_features=3, dropouts=[0.], weight_init_stddevs=[np.sqrt(6) / np.sqrt(1000)], learning_rate=0.003, **params)) params_dict = {"batch_size": [10, 20]} transformers = [] metric = dc.metrics.Metric( dc.metrics.mean_squared_error, task_averager=np.mean) # Define nb_epoch in hyperparam_search function call best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, metric, transformers, nb_epoch=3, use_max=False) valid_score = best_model.evaluate(valid_dataset, [metric]) assert valid_score["mean-mean_squared_error"] == min(all_results.values()) assert valid_score["mean-mean_squared_error"] > 0 Loading
deepchem/hyper/base_classes.py +3 −0 Original line number Diff line number Diff line Loading @@ -80,6 +80,7 @@ class HyperparamOpt(object): valid_dataset: Dataset, metric: Metric, output_transformers: List[Transformer] = [], nb_epoch: int = 10, use_max: bool = True, logdir: Optional[str] = None, **kwargs) -> Tuple[Model, Dict, Dict]: Loading Loading @@ -110,6 +111,8 @@ class HyperparamOpt(object): `train_dataset` and `valid_dataset` may have been transformed for learning and need the transform to be inverted before the metric can be evaluated on a model. nb_epoch: int, (default 10) Specifies the number of training epochs during each iteration of optimization. use_max: bool, optional If True, return the model with the highest score. Else return model with the minimum score. Loading
deepchem/hyper/gaussian_process.py +35 −4 Original line number Diff line number Diff line Loading @@ -132,6 +132,7 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): valid_dataset: Dataset, metric: Metric, output_transformers: List[Transformer] = [], nb_epoch: int = 10, use_max: bool = True, logdir: Optional[str] = None, max_iter: int = 20, Loading Loading @@ -160,6 +161,9 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): `train_dataset` and `valid_dataset` may have been transformed for learning and need the transform to be inverted before the metric can be evaluated on a model. nb_epoch: int, (default 10) Specifies the number of training epochs during each iteration of optimization. Not used by all model types. use_max: bool, (default True) Specifies whether to maximize or minimize `metric`. maximization(True) or minimization(False) Loading Loading @@ -230,14 +234,17 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): # Stores all model locations model_locations = {} # Demarcating internal function for readability def optimizing_function(**placeholders): def _optimize(nb_epoch, **placeholders): """Private Optimizing function Take in hyper parameter values and return valid set performances Take in hyper parameter values and number of training epochs. Return valid set performances. Parameters ---------- nb_epoch: int Number of epochs to train model being optimized during each iteration. Not used by all model types. placeholders: keyword arguments Should be various hyperparameters as specified in `param_keys` above. Loading Loading @@ -277,6 +284,10 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): # Add it on to the information needed for the constructor hyper_parameters["model_dir"] = model_dir model = self.model_builder(**hyper_parameters) try: model.fit(train_dataset, nb_epoch=nb_epoch) # Not all models have nb_epoch except TypeError: model.fit(train_dataset) try: model.save() Loading Loading @@ -305,6 +316,26 @@ class GaussianProcessHyperparamOpt(HyperparamOpt): else: return -score # Demarcating internal function for readability def optimizing_function(**placeholders): """Wrapper function Take in hyper parameter values. Calls a private optimize function (_optimize) with number of epochs. Returns valid set performances. Parameters ---------- placeholders: keyword arguments Should be various hyperparameters as specified in `param_keys` above. Returns: -------- valid_scores: float valid set performances """ return _optimize(nb_epoch=nb_epoch, **placeholders) # execute GPGO cov = matern32() gp = GaussianProcess(cov) Loading
deepchem/hyper/grid_search.py +10 −1 Original line number Diff line number Diff line Loading @@ -66,6 +66,7 @@ class GridHyperparamOpt(HyperparamOpt): valid_dataset: Dataset, metric: Metric, output_transformers: List[Transformer] = [], nb_epoch: int = 10, use_max: bool = True, logdir: Optional[str] = None, **kwargs, Loading @@ -91,6 +92,9 @@ class GridHyperparamOpt(HyperparamOpt): `train_dataset` and `valid_dataset` may have been transformed for learning and need the transform to be inverted before the metric can be evaluated on a model. nb_epoch: int, (default 10) Specifies the number of training epochs during each iteration of optimization. Not used by all model types. use_max: bool, optional If True, return the model with the highest score. Else return model with the minimum score. Loading Loading @@ -144,6 +148,11 @@ class GridHyperparamOpt(HyperparamOpt): model_dir = tempfile.mkdtemp() model_params['model_dir'] = model_dir model = self.model_builder(**model_params) # mypy test throws error, so ignoring it in try try: model.fit(train_dataset, nb_epoch=nb_epoch) # type: ignore # Not all models have nb_epoch except TypeError: model.fit(train_dataset) try: model.save() Loading
deepchem/hyper/tests/test_gaussian_hyperparam_opt.py +37 −0 Original line number Diff line number Diff line Loading @@ -7,6 +7,7 @@ valuable test suite so leaving it in despite the flakiness. """ import numpy as np import sklearn import sklearn.ensemble import deepchem as dc import unittest import tempfile Loading Loading @@ -171,3 +172,39 @@ class TestGaussianHyperparamOpt(unittest.TestCase): assert "learning_rate" in hp_str assert valid_score["mean-mean_squared_error"] == min(all_results.values()) assert valid_score["mean-mean_squared_error"] > 0 @flaky def test_multitask_example_nb_epoch(self): """Test a simple example of optimizing a multitask model with a gaussian process search with a different number of training epochs.""" # Generate dummy dataset np.random.seed(123) train_dataset = dc.data.NumpyDataset( np.random.rand(10, 3), np.zeros((10, 2)), np.ones((10, 2)), np.arange(10)) valid_dataset = dc.data.NumpyDataset( np.random.rand(5, 3), np.zeros((5, 2)), np.ones((5, 2)), np.arange(5)) transformers = [] optimizer = dc.hyper.GaussianProcessHyperparamOpt( lambda **params: dc.models.MultitaskRegressor(n_tasks=2, n_features=3, dropouts=[0.], weight_init_stddevs=[np.sqrt(6) / np.sqrt(1000)], learning_rate=0.003, **params)) params_dict = {"batch_size": 10} metric = dc.metrics.Metric( dc.metrics.mean_squared_error, task_averager=np.mean) best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, metric, transformers, nb_epoch=3, max_iter=1, use_max=False) valid_score = best_model.evaluate(valid_dataset, [metric], transformers) assert valid_score["mean-mean_squared_error"] == min(all_results.values()) assert valid_score["mean-mean_squared_error"] > 0
deepchem/hyper/tests/test_grid_hyperparam_opt.py +36 −0 Original line number Diff line number Diff line Loading @@ -6,6 +6,7 @@ import tempfile import numpy as np import deepchem as dc import sklearn import sklearn.ensemble class TestGridHyperparamOpt(unittest.TestCase): Loading Loading @@ -158,3 +159,38 @@ class TestGridHyperparamOpt(unittest.TestCase): assert valid_score["mean-mean_squared_error"] == min(all_results.values()) assert valid_score["mean-mean_squared_error"] > 0 def test_multitask_nb_epoch(self): """Test a simple example of optimizing a multitask model with a grid search with a different number of training epochs.""" # Generate dummy dataset np.random.seed(123) train_dataset = dc.data.NumpyDataset( np.random.rand(10, 3), np.zeros((10, 2)), np.ones((10, 2)), np.arange(10)) valid_dataset = dc.data.NumpyDataset( np.random.rand(5, 3), np.zeros((5, 2)), np.ones((5, 2)), np.arange(5)) optimizer = dc.hyper.GridHyperparamOpt( lambda **params: dc.models.MultitaskRegressor(n_tasks=2, n_features=3, dropouts=[0.], weight_init_stddevs=[np.sqrt(6) / np.sqrt(1000)], learning_rate=0.003, **params)) params_dict = {"batch_size": [10, 20]} transformers = [] metric = dc.metrics.Metric( dc.metrics.mean_squared_error, task_averager=np.mean) # Define nb_epoch in hyperparam_search function call best_model, best_hyperparams, all_results = optimizer.hyperparam_search( params_dict, train_dataset, valid_dataset, metric, transformers, nb_epoch=3, use_max=False) valid_score = best_model.evaluate(valid_dataset, [metric]) assert valid_score["mean-mean_squared_error"] == min(all_results.values()) assert valid_score["mean-mean_squared_error"] > 0
