Commit 1d02383a authored by miaecle's avatar miaecle
Browse files

Merge remote-tracking branch 'remotes/mine/GPGO' into DAG

parents a91bd758 653cd870

deepchem/hyper/GP_hyperparameter_optimization.py

0 → 100644
+153 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python2

# -*- coding: utf-8 -*-

"""

Created on Fri Jul 28 17:30:59 2017



@author: zqwu

"""



from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals



import numpy as np

import deepchem

from deepchem.molnet.preset_hyper_parameters import hps

import csv

import copy

from deepchem.utils.dependencies import pyGPGO_covfunc, pyGPGO_acquisition, \

    pyGPGO_surrogates_GaussianProcess, pyGPGO_GPGO





def GaussianProcessHyperparamOpt(dataset='tox21',

                                 model='tf',

                                 split='random',

                                 max_iter=20,

                                 search_range=4,

                                 hp_invalid_list=[

                                     'seed', 'nb_epoch', 'penalty_type',

                                     'dropouts', 'bypass_dropouts',

                                     'n_pair_feat'

                                 ]):

  hyper_parameters = copy.deepcopy(hps[model])

  # Extract parameters

  hp_list = hyper_parameters.keys()

  for hp in hp_invalid_list:

    if hp in hp_list:

      hp_list.remove(hp)



  hp_list_class = [hyper_parameters[hp].__class__ for hp in hp_list]

  assert set(hp_list_class) <= set([list, int, float])

  # Float or int hyper parameters(ex. batch_size, learning_rate)

  hp_list1 = [

      hp_list[i] for i in range(len(hp_list)) if not hp_list_class[i] is list

  ]

  # List of float or int hyper parameters(ex. layer_sizes)

  hp_list2 = [(hp_list[i], len(hyper_parameters[hp_list[i]]))

              for i in range(len(hp_list)) if hp_list_class[i] is list]



  # Number of parameters

  n_param = len(hp_list1) + sum([hp[1] for hp in hp_list2])

  # Range of optimization

  param_range = []

  for hp in hp_list1:

    if hyper_parameters[hp].__class__ is int:

      param_range.append((('int'), [

          hyper_parameters[hp] / search_range,

          hyper_parameters[hp] * search_range

      ]))

    else:

      param_range.append((('cont'), [

          hyper_parameters[hp] / search_range,

          hyper_parameters[hp] * search_range

      ]))

  for hp in hp_list2:

    if hyper_parameters[hp[0]][0].__class__ is int:

      param_range.extend([(('int'), [

          hyper_parameters[hp[0]][i] / search_range,

          hyper_parameters[hp[0]][i] * search_range

      ]) for i in range(hp[1])])

    else:

      param_range.extend([(('cont'), [

          hyper_parameters[hp[0]][i] / search_range,

          hyper_parameters[hp[0]][i] * search_range

      ]) for i in range(hp[1])])



  # Dummy names

  param_name = ['l' + format(i, '02d') for i in range(20)]

  param = dict(zip(param_name[:n_param], param_range))



  def f(l00=0,

        l01=0,

        l02=0,

        l03=0,

        l04=0,

        l05=0,

        l06=0,

        l07=0,

        l08=0,

        l09=0,

        l10=0,

        l11=0,

        l12=0,

        l13=0,

        l14=0,

        l15=0,

        l16=0,

        l17=0,

        l18=0,

        l19=0):

    args = locals()

    # Input hyper parameters

    i = 0

    for hp in hp_list1:

      hyper_parameters[hp] = float(args[param_name[i]])

      if param_range[i][0] == 'int':

        hyper_parameters[hp] = int(hyper_parameters[hp])

      i = i + 1

    for hp in hp_list2:

      hyper_parameters[hp[0]] = [

          float(args[param_name[j]]) for j in range(i, i + hp[1])

      ]

      if param_range[i][0] == 'int':

        hyper_parameters[hp[0]] = map(int, hyper_parameters[hp[0]])

      i = i + hp[1]



    print(hyper_parameters)

    # Run benchmark

    deepchem.molnet.run_benchmark(

        [dataset],

        str(model),

        split=str(split),

        hyper_parameters=hyper_parameters,

        out_path='/tmp')

    # Read results

    with open('/tmp/results.csv', 'r') as f:

      reader = csv.reader(f)

      # Return valid set performances

      return float(list(reader)[-1][8])



  cov = pyGPGO_covfunc.matern32()

  gp = pyGPGO_surrogates_GaussianProcess.GaussianProcess(cov)

  acq = pyGPGO_acquisition.Acquisition(mode='ExpectedImprovement')

  gpgo = pyGPGO_GPGO.GPGO(gp, acq, f, param)

  gpgo.run(max_iter=max_iter)



  hp_opt, valid_performance_opt = gpgo.getResult()



  # Readout best hyper parameters

  i = 0

  for hp in hp_list1:

    hyper_parameters[hp] = float(hp_opt[param_name[i]])

    if param_range[i][0] == 'int':

      hyper_parameters[hp] = int(hyper_parameters[hp])

    i = i + 1

  for hp in hp_list2:

    hyper_parameters[hp[0]] = [

        float(hp_opt[param_name[j]]) for j in range(i, i + hp[1])

    ]

    if param_range[i][0] == 'int':

      hyper_parameters[hp[0]] = map(int, hyper_parameters[hp[0]])

    i = i + hp[1]



  return hyper_parameters, valid_performance_opt

deepchem/hyper/__init__.py

+1 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ from functools import reduce 
from operator import mul

from deepchem.utils.evaluate import Evaluator

from deepchem.utils.save import log

from deepchem.hyper.GP_hyperparameter_optimization import GaussianProcessHyperparamOpt





class HyperparamOpt(object):

deepchem/utils/dependencies.py

+5 −0
Original line number Diff line number Diff line
@@ -18,3 +18,8 @@ xgboost = import_lazy("xgboost") 
pdbfixer = import_lazy("pdbfixer")

simtk = import_lazy("simtk")

mdtraj = import_lazy("mdtraj")

pyGPGO_covfunc = import_lazy("pyGPGO.covfunc")

pyGPGO_acquisition = import_lazy("pyGPGO.acquisition")

pyGPGO_surrogates_GaussianProcess = import_lazy(

    "pyGPGO.surrogates.GaussianProcess")

pyGPGO_GPGO = import_lazy("pyGPGO.GPGO")

examples/benchmark_hyperparam_search.py

deleted100644 → 0
+0 −119
Original line number Diff line number Diff line 
# -*- coding: utf-8 -*-

"""

Created on Mon Nov 07 22:41:34 2016



@author: Zhenqin Wu

"""

import numpy as np

import deepchem as dc

import os

import time

import sys

from scipy.stats import truncnorm



# main layer

layer_sizes_0 = [1200]

weight_init_stddevs_0 = [0.02]

bias_init_consts_0 = [1.]

dropouts_0 = [0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7]



#bypass layer

bypass_layer_sizes_0 = truncnorm(-1, 1, loc=200, scale=150)

bypass_weight_init_stddevs_0 = [.02]

bypass_bias_init_consts_0 = [1.]

bypass_dropouts_0 = [0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7]



#penalty

penalty_0 = truncnorm(-1, 1, loc=0.3, scale=0.2)

penalty_type_0 = ['l2']



#general figure

batch_size_0 = [50]

nb_epoch_0 = [12]



#learning rate

learning_rate_0 = truncnorm(-1, 1, loc=-3.2, scale=1.2)



#for graph-conv and random forest

n_filters_0 = [64, 96, 128]

n_fully_connected_nodes_0 = [100, 120, 140, 160, 200, 240, 300]

n_estimators_0 = [500]

seed = None



out_path = '.'



dname = sys.argv[1]

model = sys.argv[2]



parameters_printed = {

    'tf': [

        'layer_sizes', 'weight_init_stddevs', 'bias_init_consts', 'dropouts',

        'penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'

    ],

    'tf_robust': [

        'layer_sizes', 'weight_init_stddevs', 'bias_init_consts', 'dropouts',

        'bypass_layer_sizes', 'bypass_weight_init_stddevs',

        'bypass_bias_init_consts', 'bypass_dropouts', 'penalty', 'penalty_type',

        'batch_size', 'nb_epoch', 'learning_rate'

    ],

    'logreg':

    ['penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'],

    'graphconv': [

        'batch_size', 'nb_epoch', 'learning_rate', 'n_filters',

        'n_fully_connected_nodes'

    ],

    'rf': ['n_estimators']

}

hps = {}

for i in range(int(sys.argv[3])):

  layer_sizes = layer_sizes_0

  weight_init_stddevs = weight_init_stddevs_0

  bias_init_consts = bias_init_consts_0

  dropouts = [np.random.choice(dropouts_0)]



  bypass_layer_sizes = [int(bypass_layer_sizes_0.rvs())]

  bypass_weight_init_stddevs = bypass_weight_init_stddevs_0

  bypass_bias_init_consts = bypass_bias_init_consts_0

  bypass_dropouts = [np.random.choice(bypass_dropouts_0)]



  penalty = penalty_0.rvs()

  penalty_type = np.random.choice(penalty_type_0)



  batch_size = np.random.choice(batch_size_0)

  nb_epoch = np.random.choice(nb_epoch_0)



  learning_rate = 10**(learning_rate_0.rvs())



  n_filters = np.random.choice(n_filters_0)

  n_fully_connected_nodes = np.random.choice(n_fully_connected_nodes_0)

  n_estimators = np.random.choice(n_estimators_0)



  hps[model] = {

      'layer_sizes': layer_sizes,

      'weight_init_stddevs': weight_init_stddevs,

      'bias_init_consts': bias_init_consts,

      'dropouts': dropouts,

      'bypass_layer_sizes': bypass_layer_sizes,

      'bypass_weight_init_stddevs': bypass_weight_init_stddevs,

      'bypass_bias_init_consts': bypass_bias_init_consts,

      'bypass_dropouts': bypass_dropouts,

      'penalty': penalty,

      'penalty_type': penalty_type,

      'batch_size': batch_size,

      'nb_epoch': nb_epoch,

      'learning_rate': learning_rate,

      'n_filters': n_filters,

      'n_fully_connected_nodes': n_fully_connected_nodes,

      'n_estimators': n_estimators,

      'seed': seed

  }



  with open(os.path.join(out_path, 'hps.csv'), 'a') as f:

    f.write('\n' + str(i) + ',' + dname + ',')

    for item in hps[model][i]:

      if item in parameters_printed[model]:

        f.write(item + ',')

        f.write(str(hps[model][i][item]) + ',')



  dc.molnet.run_benchmark(

      [dname], str(model), out_path=out_path, hyper_parameters=hps)

CRA Git | Maintained and supported by SUSTech CRA and CCSE