Merge pull request #125 from rbharath/hyperparam_tests

import collections

from operator import mul

from deepchem.utils.evaluate import Evaluator

from deepchem.utils.save import log

class HyperparamOpt(object):

  """

  Provides simple hyperparameter search capabilities.

  """

  def __init__(self, model_class, task_types):

  def __init__(self, model_class, task_types, verbosity=None):

    self.model_class = model_class

    self.task_types = task_types

    assert verbosity in [None, "low", "high"]

    self.verbosity = verbosity

  def hyperparam_search(self, params_dict, train_dataset, valid_dataset,

                        output_transformers, metric, use_max=True,

                        verbosity=None, logdir=None):

                        logdir=None):

    """Perform hyperparams search according to params_dict.

    Each key to hyperparams_dict is a model_param. The values should be a list

      evaluator = Evaluator(model, valid_dataset, output_transformers)

      df, score = evaluator.compute_model_performance(

          valid_csv_out, valid_stats_out)

      valid_score = score.iloc[0][metric]

          [metric], valid_csv_out, valid_stats_out)

      valid_score = score.iloc[0][metric.name]

      all_scores[hyperparameter_tuple] = valid_score

      if (use_max and valid_score > best_validation_score) or (

      else:

        shutil.rmtree(model_dir)

      print("Model %d/%d, Metric %s, Validation set %s: %f" %

            (ind, number_combinations, metric, ind, valid_score))

      print("\tbest_validation_score so  far: %f" % best_validation_score)

      log("Model %d/%d, Metric %s, Validation set %s: %f" %

          (ind, number_combinations, metric.name, ind, valid_score),

          self.verbosity)

      log("\tbest_validation_score so  far: %f" % best_validation_score,

          self.verbosity)

    train_csv_out = tempfile.NamedTemporaryFile()

    train_stats_out = tempfile.NamedTemporaryFile()

    train_evaluator = Evaluator(best_model, train_dataset, output_transformers)

    train_df, train_score = train_evaluator.compute_model_performance(

        train_csv_out, train_stats_out)

    train_score = train_score.iloc[0][metric]

    print("Best hyperparameters: %s" % str(zip(hyperparams, best_hyperparams)))

    print("train_score: %f" % train_score)

    print("validation_score: %f" % best_validation_score)

        [metric], train_csv_out, train_stats_out)

    train_score = train_score.iloc[0][metric.name]

    log("Best hyperparameters: %s" % str(zip(hyperparams, best_hyperparams)),

        self.verbosity)

    log("train_score: %f" % train_score, self.verbosity)

    log("validation_score: %f" % best_validation_score, self.verbosity)

    return best_model, best_hyperparams, all_scores

"""

Integration tests for hyperparam optimization.

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals

__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "LGPL"

import os

import unittest

import tempfile

import shutil

from deepchem.models.test import TestAPI

from deepchem.models.sklearn_models import SklearnModel

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.transformers import NormalizationTransformer

from deepchem import metrics

from deepchem.metrics import Metric

from sklearn.ensemble import RandomForestRegressor

def rf_model_builder(task_types, params_dict, logdir=None,

                     train=True):

    """Builds random forests given hyperparameters.

    Last two arguments only for tensorflow models and ignored.

    """

    n_estimators = params_dict["n_estimators"]

    max_features = params_dict["max_features"]

    return SklearnModel(

        task_types, params_dict,

        model_instance=RandomForestRegressor(n_estimators=n_estimators,

                                             max_features=max_features))

class TestHyperparamOptAPI(TestAPI):

  """

  Test hyperparameter optimization API.

  """

  def test_singletask_sklearn_rf_ECFP_regression_hyperparam_opt(self):

    """Test of singletask RF ECFP regression API."""

    splittype = "scaffold"

    compound_featurizers = [CircularFingerprint(size=1024)]

    complex_featurizers = []

    input_transformer_classes = []

    output_transformer_classes = [NormalizationTransformer]

    task_types = {"log-solubility": "regression"}

    input_file = "example.csv"

    train_dataset, valid_dataset, _, output_transformers, = \

        self._featurize_train_test_split(

            splittype, compound_featurizers, 

            complex_featurizers, input_transformer_classes,

            output_transformer_classes, input_file, task_types.keys())

    params_dict = {

      "n_estimators": [10, 100],

      "max_features": ["auto"],

      "data_shape": train_dataset.get_data_shape()

    }

    metric = Metric(metrics.r2_score)

    self._hyperparam_opt(rf_model_builder, params_dict, train_dataset,

                         valid_dataset, output_transformers, task_types,

                         metric)

from deepchem.transformers import NormalizationTransformer

from deepchem.transformers import LogTransformer

from deepchem.transformers import ClippingTransformer

from deepchem.hyperparameters import HyperparamOpt

from sklearn.ensemble import RandomForestRegressor

class TestAPI(unittest.TestCase):

    # debug.

    #shutil.rmtree(self.model_dir)

  def _hyperparam_opt(self, model_builder, params_dict, train_dataset,

                      valid_dataset, output_transformers, task_types, metric):

    optimizer = HyperparamOpt(model_builder, task_types, verbosity="low")

    best_model, best_hyperparams, all_results = optimizer.hyperparam_search(

      params_dict, train_dataset, valid_dataset, output_transformers,

      metric)

  def _create_model(self, train_dataset, test_dataset, model, transformers,

                    metrics):

    """Helper method to create model for test."""

class TestKerasSklearnAPI(TestAPI):

  """

  Test top-level API for ML models."

  Test top-level API for ML models.

  """

  def test_singletask_sklearn_rf_ECFP_regression_API(self):

    """Test of singletask RF ECFP regression API."""