Merge pull request #1878 from deepchem/hyperparam

from deepchem.hyper.grid_search import HyperparamOpt

from deepchem.hyper.base_classes import HyperparamOpt

from deepchem.hyper.grid_search import GridHyperparamOpt

from deepchem.hyper.gaussian_process import GaussianProcessHyperparamOpt

import logging

logger = logging.getLogger(__name__)

def _convert_hyperparam_dict_to_filename(hyper_params):

  """Helper function that converts a dictionary of hyperparameters to a string that can be a filename.

  Parameters

  ----------

  hyper_params: dict

    Maps string of hyperparameter name to int/float.

  Returns

  -------

  filename: str

    A filename of form "_key1_value1_value2_..._key2..."

  """

  filename = ""

  keys = sorted(hyper_params.keys())

  for key in keys:

    filename += "_%s" % str(key)

    value = hyper_params[key]

    if isinstance(value, int):

      filename += "_%s" % str(value)

    elif isinstance(value, float):

      filename += "_%.2f" % value

    else:

      filename += "%s" % str(value)

  return filename

class HyperparamOpt(object):

  """Abstract superclass for hyperparameter search classes.

  This class is an abstract base class for hyperparameter search

  classes in DeepChem. Hyperparameter search is performed on

  `dc.models.Model` classes. Each hyperparameter object accepts a

  `dc.models.Model` class upon construct. When the `hyperparam_search`

  class is invoked, this class is used to construct many different

  concrete models which are trained on the specified training set and

  evaluated on a given validation set.

  Different subclasses of `HyperparamOpt` differ in the choice of

  strategy for searching the hyperparameter evaluation space. This

  class itself is an abstract superclass and should never be directly

  instantiated.

  """

  def __init__(self, model_builder):

    """Initialize Hyperparameter Optimizer.

    Note this is an abstract constructor which should only be used by

    subclasses.

    Parameters

    ----------

    model_builder: constructor function.

      This parameter must be constructor function which returns an

      object which is an instance of `dc.models.Model`. This function

      must accept two arguments, `model_params` of type `dict` and

      `model_dir`, a string specifying a path to a model directory.

      See the example.

    """

    if self.__class__.__name__ == "HyperparamOpt":

      raise ValueError(

          "HyperparamOpt is an abstract superclass and cannot be directly instantiated. You probably want to instantiate a concrete subclass instead."

      )

    self.model_builder = model_builder

  def hyperparam_search(self,

                        params_dict,

                        train_dataset,

                        valid_dataset,

                        transformers,

                        metric,

                        use_max=True,

                        logdir=None):

    """Conduct Hyperparameter search.

    This method defines the common API shared by all hyperparameter

    optimization subclasses. Different classes will implement

    different search methods but they must all follow this common API.

    Parameters

    ----------

    params_dict: dict

      Dictionary mapping strings to values. Note that the

      precise semantics of `params_dict` will change depending on the

      optimizer that you're using. Depending on the type of

      hyperparameter optimization, these values can be

      ints/floats/strings/lists/etc. Read the documentation for the

      concrete hyperparameter optimization subclass you're using to

      learn more about what's expected.

    train_dataset: `dc.data.Dataset`

      dataset used for training

    valid_dataset: `dc.data.Dataset`

      dataset used for validation(optimization on valid scores)

    output_transformers: list[dc.trans.Transformer]

      Transformers for evaluation. This argument is needed since

      `train_dataset` and `valid_dataset` may have been transformed

      for learning and need the transform to be inverted before

      the metric can be evaluated on a model.

    use_max: bool, optional

      If True, return the model with the highest score. Else return

      model with the minimum score.

    logdir: str, optional

      The directory in which to store created models. If not set, will

      use a temporary directory.

    Returns

    -------

    `(best_model, best_hyperparams, all_scores)` where `best_model` is

    an instance of `dc.models.Models`, `best_hyperparams` is a

    dictionary of parameters, and `all_scores` is a dictionary mapping

    string representations of hyperparameter sets to validation

    scores.

    """

    raise NotImplementedError

#!/usr/bin/env python2

# -*- coding: utf-8 -*-

"""

Contains basic hyperparameter optimizations.

"""

import tempfile

import shutil

import collections

import logging

from functools import reduce

from operator import mul

from deepchem.utils.evaluate import Evaluator

from deepchem.utils.save import log

from deepchem.hyper.base_classes import HyperparamOpt

from deepchem.hyper.base_classes import _convert_hyperparam_dict_to_filename

logger = logging.getLogger(__name__)

class HyperparamOpt(object):

  """

  Provides simple hyperparameter search capabilities.

class GridHyperparamOpt(HyperparamOpt):

  """

  Provides simple grid hyperparameter search capabilities.

  This class performs a grid hyperparameter search over the specified

  hyperparameter space. This implementation is simple and simply does

  a direct iteration over all possible hyperparameters and doesn't use

  parallelization to speed up the search.

  Example

  -------

  This example shows the type of constructor function expected. 

  >>> import sklearn

  >>> import deepchem as dc

  >>> optimizer = dc.hyper.GridHyperparamOpt(lambda **p: dc.models.GraphConvModel(**p))

  Here's a more sophisticated example that shows how to optimize only

  some parameters of a model. In this case, we have some parameters we

  want to optimize, and others which we don't. To handle this type of

  search, we create a `model_builder` which hard codes some arguments

  (in this case, `n_tasks` and `n_features` which are properties of a

  dataset and not hyperparameters to search over.)

  def __init__(self, model_class, verbose=True):

    self.model_class = model_class

    self.verbose = verbose

  >>> def model_builder(**model_params):

  ...   n_layers = model_params['layers']

  ...   layer_width = model_params['width']

  ...   dropout = model_params['dropout']

  ...   return dc.models.MultitaskClassifier(

  ...     n_tasks=5,

  ...     n_features=100,

  ...     layer_sizes=[layer_width]*n_layers,

  ...     dropouts=dropout

  ...   )

  >>> optimizer = dc.hyper.GridHyperparamOpt(model_builder)

  """

  # TODO(rbharath): This function is complicated and monolithic. Is there a nice

  # way to refactor this?

  def hyperparam_search(self,

                        params_dict,

                        train_dataset,

                        logdir=None):

    """Perform hyperparams search according to params_dict.

    Each key to hyperparams_dict is a model_param. The values should be a list

    of potential values for that hyperparam.

    Each key to hyperparams_dict is a model_param. The values should

    be a list of potential values for that hyperparam.

    Parameters

    ----------

    params_dict: Dict[str, list]

      Maps hyperparameter names (strings) to lists of possible

      parameter values.

    train_dataset: `dc.data.Dataset`

      dataset used for training

    valid_dataset: `dc.data.Dataset`

      dataset used for validation(optimization on valid scores)

    output_transformers: list[dc.trans.Transformer]

      transformers for evaluation

    metric: dc.metrics.Metric

      metric used for evaluation

    use_max: bool, optional

      If True, return the model with the highest score. Else return

      model with the minimum score.

    logdir: str, optional

      The directory in which to store created models. If not set, will

      use a temporary directory.

    TODO(rbharath): This shouldn't be stored in a temporary directory.

    Returns

    -------

    `(best_model, best_hyperparams, all_scores)` where `best_model` is

    an instance of `dc.model.Models`, `best_hyperparams` is a

    dictionary of parameters, and `all_scores` is a dictionary mapping

    string representations of hyperparameter sets to validation

    scores.

    """

    hyperparams = params_dict.keys()

    hyperparam_vals = params_dict.values()

    for ind, hyperparameter_tuple in enumerate(

        itertools.product(*hyperparam_vals)):

      model_params = {}

      log("Fitting model %d/%d" % (ind + 1, number_combinations), self.verbose)

      logger.info("Fitting model %d/%d" % (ind + 1, number_combinations))

      # Construction dictionary mapping hyperparameter names to values

      hyper_params = dict(zip(hyperparams, hyperparameter_tuple))

      for hyperparam, hyperparam_val in zip(hyperparams, hyperparameter_tuple):

        model_params[hyperparam] = hyperparam_val

      log("hyperparameters: %s" % str(model_params), self.verbose)

      logger.info("hyperparameters: %s" % str(model_params))

      if logdir is not None:

        model_dir = os.path.join(logdir, str(ind))

        log("model_dir is %s" % model_dir, self.verbose)

        logger.info("model_dir is %s" % model_dir)

        try:

          os.makedirs(model_dir)

        except OSError:

          if not os.path.isdir(model_dir):

            log("Error creating model_dir, using tempfile directory",

                self.verbose)

            logger.info("Error creating model_dir, using tempfile directory")

            model_dir = tempfile.mkdtemp()

      else:

        model_dir = tempfile.mkdtemp()

      model = self.model_class(model_params, model_dir)

      model_params['model_dir'] = model_dir

      model = self.model_builder(**model_params)

      model.fit(train_dataset)

      try:

        model.save()

      # Some models autosave

      except NotImplementedError:

        pass

      evaluator = Evaluator(model, valid_dataset, output_transformers)

      multitask_scores = evaluator.compute_model_performance([metric])

      valid_score = multitask_scores[metric.name]

      all_scores[str(hyperparameter_tuple)] = valid_score

      hp_str = _convert_hyperparam_dict_to_filename(hyper_params)

      all_scores[hp_str] = valid_score

      if (use_max and valid_score >= best_validation_score) or (

          not use_max and valid_score <= best_validation_score):

      else:

        shutil.rmtree(model_dir)

      log(

          "Model %d/%d, Metric %s, Validation set %s: %f" %

          (ind + 1, number_combinations, metric.name, ind, valid_score),

          self.verbose)

      log("\tbest_validation_score so far: %f" % best_validation_score,

          self.verbose)

      logger.info("Model %d/%d, Metric %s, Validation set %s: %f" %

                  (ind + 1, number_combinations, metric.name, ind, valid_score))

      logger.info("\tbest_validation_score so far: %f" % best_validation_score)

    if best_model is None:

      log("No models trained correctly.", self.verbose)

      logger.info("No models trained correctly.")

      # arbitrarily return last model

      best_model, best_hyperparams = model, hyperparameter_tuple

      return best_model, best_hyperparams, all_scores

    train_evaluator = Evaluator(best_model, train_dataset, output_transformers)

    multitask_scores = train_evaluator.compute_model_performance([metric])

    train_score = multitask_scores[metric.name]

    log("Best hyperparameters: %s" % str(best_hyperparams), self.verbose)

    log("train_score: %f" % train_score, self.verbose)

    log("validation_score: %f" % best_validation_score, self.verbose)

    logger.info("Best hyperparameters: %s" % str(best_hyperparams))

    logger.info("train_score: %f" % train_score)

    logger.info("validation_score: %f" % best_validation_score)

    return best_model, best_hyperparams, all_scores

"""

Tests for Gaussian Process Hyperparameter Optimization.

These tests fails every so often. I think it's when the Gaussian

process optimizer doesn't find an optimal point. This is still a

valuable test suite so leaving it in despite the flakiness.

"""

import os

import numpy as np

import sklearn

import deepchem as dc

import unittest

import tempfile

from flaky import flaky

class TestGaussianHyperparamOpt(unittest.TestCase):

  """

  Test Gaussian Hyperparameter Optimization.

  """

  def setUp(self):

    """Set up common resources."""

    def rf_model_builder(**model_params):

      rf_params = {k: v for (k, v) in model_params.items() if k != 'model_dir'}

      model_dir = model_params['model_dir']

      sklearn_model = sklearn.ensemble.RandomForestRegressor(**rf_params)

      return dc.models.SklearnModel(sklearn_model, model_dir)

    self.rf_model_builder = rf_model_builder

    self.train_dataset = dc.data.NumpyDataset(

        X=np.random.rand(50, 5), y=np.random.rand(50, 1))

    self.valid_dataset = dc.data.NumpyDataset(

        X=np.random.rand(20, 5), y=np.random.rand(20, 1))

  def test_rf_example(self):

    """Test a simple example of optimizing a RF model with a gaussian process."""

    optimizer = dc.hyper.GaussianProcessHyperparamOpt(self.rf_model_builder)

    params_dict = {"n_estimators": 10}

    transformers = []

    metric = dc.metrics.Metric(dc.metrics.pearson_r2_score)

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

        params_dict,

        self.train_dataset,

        self.valid_dataset,

        transformers,

        metric,

        max_iter=2)

    valid_score = best_model.evaluate(self.valid_dataset, [metric],

                                      transformers)

    assert valid_score["pearson_r2_score"] == max(all_results.values())

    assert valid_score["pearson_r2_score"] > 0

  def test_rf_example_min(self):

    """Test a simple example of optimizing a RF model with a gaussian process looking for minimum score."""

    optimizer = dc.hyper.GaussianProcessHyperparamOpt(self.rf_model_builder)

    params_dict = {"n_estimators": 10}

    transformers = []

    metric = dc.metrics.Metric(dc.metrics.pearson_r2_score)

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

        params_dict,

        self.train_dataset,

        self.valid_dataset,

        transformers,

        metric,

        use_max=False,

        max_iter=2)

    valid_score = best_model.evaluate(self.valid_dataset, [metric],

                                      transformers)

    assert valid_score["pearson_r2_score"] == min(all_results.values())

    assert valid_score["pearson_r2_score"] > 0

  def test_rf_with_logdir(self):

    """Test that using a logdir can work correctly."""

    optimizer = dc.hyper.GaussianProcessHyperparamOpt(self.rf_model_builder)

    params_dict = {"n_estimators": 10}

    transformers = []

    metric = dc.metrics.Metric(dc.metrics.pearson_r2_score)

    with tempfile.TemporaryDirectory() as tmpdirname:

      best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

          params_dict,

          self.train_dataset,

          self.valid_dataset,

          transformers,

          metric,

          logdir=tmpdirname,

          max_iter=2)

    valid_score = best_model.evaluate(self.valid_dataset, [metric],

                                      transformers)

    assert valid_score["pearson_r2_score"] == max(all_results.values())

    assert valid_score["pearson_r2_score"] > 0

  @flaky

  def test_multitask_example(self):

    """Test a simple example of optimizing a multitask model with a gaussian process search."""

    # Generate dummy dataset

    np.random.seed(123)

    train_dataset = dc.data.NumpyDataset(

        np.random.rand(10, 3), np.zeros((10, 2)), np.ones((10, 2)),

        np.arange(10))

    valid_dataset = dc.data.NumpyDataset(

        np.random.rand(5, 3), np.zeros((5, 2)), np.ones((5, 2)), np.arange(5))

    optimizer = dc.hyper.GaussianProcessHyperparamOpt(

        lambda **p: dc.models.MultitaskRegressor(n_tasks=2,

             n_features=3, dropouts=[0.],

             weight_init_stddevs=[np.sqrt(6)/np.sqrt(1000)],

             learning_rate=0.003, **p))

    params_dict = {"batch_size": 10}

    transformers = []

    metric = dc.metrics.Metric(

        dc.metrics.mean_squared_error, task_averager=np.mean)

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

        params_dict,

        train_dataset,

        valid_dataset,

        transformers,

        metric,

        max_iter=1,

        use_max=False)

    valid_score = best_model.evaluate(valid_dataset, [metric])

    assert valid_score["mean-mean_squared_error"] == min(all_results.values())

    assert valid_score["mean-mean_squared_error"] > 0

  @flaky

  def test_multitask_example_different_search_range(self):

    """Test a simple example of optimizing a multitask model with a gaussian process search with per-parameter search range."""

    # Generate dummy dataset

    np.random.seed(123)

    train_dataset = dc.data.NumpyDataset(

        np.random.rand(10, 3), np.zeros((10, 2)), np.ones((10, 2)),

        np.arange(10))

    valid_dataset = dc.data.NumpyDataset(

        np.random.rand(5, 3), np.zeros((5, 2)), np.ones((5, 2)), np.arange(5))

    optimizer = dc.hyper.GaussianProcessHyperparamOpt(

        lambda **p: dc.models.MultitaskRegressor(

            n_tasks=2,

            n_features=3,

            dropouts=[0.],

            weight_init_stddevs=[np.sqrt(6) / np.sqrt(1000)],

            **p))

    params_dict = {"learning_rate": 0.003, "batch_size": 10}

    # These are per-example multiplier

    search_range = {"learning_rate": 10, "batch_size": 4}

    transformers = []

    metric = dc.metrics.Metric(

        dc.metrics.mean_squared_error, task_averager=np.mean)

    with tempfile.TemporaryDirectory() as tmpdirname:

      best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

          params_dict,

          train_dataset,

          valid_dataset,

          transformers,

          metric,

          max_iter=2,

          logdir=tmpdirname,

          search_range=search_range,

          use_max=False)

      valid_score = best_model.evaluate(valid_dataset, [metric])

    # Test that 2 parameters were optimized

    for hp_str in all_results.keys():

      # Recall that the key is a string of the form _batch_size_39_learning_rate_0.01 for example

      assert "batch_size" in hp_str

      assert "learning_rate" in hp_str

    assert valid_score["mean-mean_squared_error"] == min(all_results.values())

    assert valid_score["mean-mean_squared_error"] > 0