Merge pull request #738 from miaecle/GPGO (c657cb1d) · Commits · 钟慕尧 / deepchem

deepchem/hyper/init.py

+2 −113

Original line number	Diff line number	Diff line
		"""
		Contains basic hyperparameter optimizations.
		"""
		import numpy as np
		import os
		import itertools
		import tempfile
		import shutil
		import collections
		from functools import reduce
		from operator import mul
		from deepchem.utils.evaluate import Evaluator
		from deepchem.utils.save import log


		class HyperparamOpt(object):
		"""
		Provides simple hyperparameter search capabilities.
		"""

		def __init__(self, model_class, verbose=True):
		self.model_class = model_class
		self.verbose = verbose

		# TODO(rbharath): This function is complicated and monolithic. Is there a nice
		# way to refactor this?
		def hyperparam_search(self,
		params_dict,
		train_dataset,
		valid_dataset,
		output_transformers,
		metric,
		use_max=True,
		logdir=None):
		"""Perform hyperparams search according to params_dict.

		Each key to hyperparams_dict is a model_param. The values should be a list
		of potential values for that hyperparam.

		TODO(rbharath): This shouldn't be stored in a temporary directory.
		"""
		hyperparams = params_dict.keys()
		hyperparam_vals = params_dict.values()
		for hyperparam_list in params_dict.values():
		assert isinstance(hyperparam_list, collections.Iterable)

		number_combinations = reduce(mul, [len(vals) for vals in hyperparam_vals])

		if use_max:
		best_validation_score = -np.inf
		else:
		best_validation_score = np.inf
		best_hyperparams = None
		best_model, best_model_dir = None, None
		all_scores = {}
		for ind, hyperparameter_tuple in enumerate(
		itertools.product(*hyperparam_vals)):
		model_params = {}
		log("Fitting model %d/%d" % (ind + 1, number_combinations), self.verbose)
		for hyperparam, hyperparam_val in zip(hyperparams, hyperparameter_tuple):
		model_params[hyperparam] = hyperparam_val
		log("hyperparameters: %s" % str(model_params), self.verbose)

		if logdir is not None:
		model_dir = os.path.join(logdir, str(ind))
		log("model_dir is %s" % model_dir, self.verbose)
		try:
		os.makedirs(model_dir)
		except OSError:
		if not os.path.isdir(model_dir):
		log("Error creating model_dir, using tempfile directory",
		self.verbose)
		model_dir = tempfile.mkdtemp()
		else:
		model_dir = tempfile.mkdtemp()

		model = self.model_class(model_params, model_dir)
		model.fit(train_dataset, **model_params)
		model.save()

		evaluator = Evaluator(model, valid_dataset, output_transformers)
		multitask_scores = evaluator.compute_model_performance([metric])
		valid_score = multitask_scores[metric.name]
		all_scores[str(hyperparameter_tuple)] = valid_score

		if (use_max and valid_score >= best_validation_score) or (
		not use_max and valid_score <= best_validation_score):
		best_validation_score = valid_score
		best_hyperparams = hyperparameter_tuple
		if best_model_dir is not None:
		shutil.rmtree(best_model_dir)
		best_model_dir = model_dir
		best_model = model
		else:
		shutil.rmtree(model_dir)

		log("Model %d/%d, Metric %s, Validation set %s: %f" %
		(ind + 1, number_combinations, metric.name, ind,
		valid_score), self.verbose)
		log("\tbest_validation_score so far: %f" % best_validation_score,
		self.verbose)
		if best_model is None:
		log("No models trained correctly.", self.verbose)
		# arbitrarily return last model
		best_model, best_hyperparams = model, hyperparameter_tuple
		return best_model, best_hyperparams, all_scores
		train_evaluator = Evaluator(best_model, train_dataset, output_transformers)
		multitask_scores = train_evaluator.compute_model_performance([metric])
		train_score = multitask_scores[metric.name]
		log("Best hyperparameters: %s" % str(best_hyperparams), self.verbose)
		log("train_score: %f" % train_score, self.verbose)
		log("validation_score: %f" % best_validation_score, self.verbose)
		return best_model, best_hyperparams, all_scores
		from deepchem.hyper.grid_search import HyperparamOpt
		from deepchem.hyper.gaussian_process import GaussianProcessHyperparamOpt

deepchem/hyper/gaussian_process.py

0 → 100644

+237 −0

Original line number	Diff line number	Diff line
		"""
		Contains class for gaussian process hyperparameter optimizations.
		"""
		from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals

		import numpy as np
		import tempfile
		from deepchem.hyper.grid_search import HyperparamOpt
		from deepchem.utils.evaluate import Evaluator
		from deepchem.molnet.run_benchmark_models import benchmark_classification, benchmark_regression
		from deepchem.utils.dependencies import pyGPGO_covfunc, pyGPGO_acquisition, \
		pyGPGO_surrogates_GaussianProcess, pyGPGO_GPGO


		class GaussianProcessHyperparamOpt(HyperparamOpt):
		"""
		Gaussian Process Global Optimization(GPGO)
		"""

		def hyperparam_search(self,
		params_dict,
		train_dataset,
		valid_dataset,
		output_transformers,
		metric,
		n_features=1024,
		n_tasks=1,
		max_iter=20,
		search_range=4,
		hp_invalid_list=[
		'seed', 'nb_epoch', 'penalty_type', 'dropouts',
		'bypass_dropouts', 'n_pair_feat'
		],
		logdir=None):
		"""Perform hyperparams search using a gaussian process assumption

		params_dict include single-valued parameters being optimized,
		which should only contain int, float and list of int(float)

		parameters with names in hp_invalid_list will not be changed.

		For Molnet models, self.model_class is model name in string,
		params_dict = dc.molnet.preset_hyper_parameters.hps[self.model_class]

		Parameters
		----------
		params_dict: dict
		dict including parameters and their initial values
		parameters not suitable for optimization can be added to hp_invalid_list
		train_dataset: dc.data.Dataset struct
		dataset used for training
		valid_dataset: dc.data.Dataset struct
		dataset used for validation(optimization on valid scores)
		output_transformers: list of dc.trans.Transformer
		transformers for evaluation
		metric: list of dc.metrics.Metric
		metric used for evaluation
		n_features: int
		number of input features
		n_tasks: int
		number of tasks
		max_iter: int
		number of optimization trials
		search_range: int(float)
		optimization on [initial values / search_range,
		initial values * search_range]
		hp_invalid_list: list
		names of parameters that should not be optimized

		Returns
		-------
		hyper_parameters: dict
		params_dict with all optimized values
		valid_performance_opt: float
		best performance on valid dataset

		"""

		assert len(metric) == 1, 'Only use one metric'
		hyper_parameters = params_dict
		hp_list = hyper_parameters.keys()
		for hp in hp_invalid_list:
		if hp in hp_list:
		hp_list.remove(hp)

		hp_list_class = [hyper_parameters[hp].__class__ for hp in hp_list]
		assert set(hp_list_class) <= set([list, int, float])
		# Float or int hyper parameters(ex. batch_size, learning_rate)
		hp_list_single = [
		hp_list[i] for i in range(len(hp_list)) if not hp_list_class[i] is list
		]
		# List of float or int hyper parameters(ex. layer_sizes)
		hp_list_multiple = [(hp_list[i], len(hyper_parameters[hp_list[i]]))
		for i in range(len(hp_list))
		if hp_list_class[i] is list]

		# Number of parameters
		n_param = len(hp_list_single + sum([hp[1] for hp in hp_list_multiple]))
		# Range of optimization
		param_range = []
		for hp in hp_list_single:
		if hyper_parameters[hp].__class__ is int:
		param_range.append((('int'), [
		hyper_parameters[hp] // search_range,
		hyper_parameters[hp] * search_range
		]))
		else:
		param_range.append((('cont'), [
		hyper_parameters[hp] / search_range,
		hyper_parameters[hp] * search_range
		]))
		for hp in hp_list_multiple:
		if hyper_parameters[hp[0]][0].__class__ is int:
		param_range.extend([(('int'), [
		hyper_parameters[hp[0]][i] // search_range,
		hyper_parameters[hp[0]][i] * search_range
		]) for i in range(hp[1])])
		else:
		param_range.extend([(('cont'), [
		hyper_parameters[hp[0]][i] / search_range,
		hyper_parameters[hp[0]][i] * search_range
		]) for i in range(hp[1])])

		# Dummy names
		param_name = ['l' + format(i, '02d') for i in range(20)]
		param = dict(zip(param_name[:n_param], param_range))

		def f(l00=0,
		l01=0,
		l02=0,
		l03=0,
		l04=0,
		l05=0,
		l06=0,
		l07=0,
		l08=0,
		l09=0,
		l10=0,
		l11=0,
		l12=0,
		l13=0,
		l14=0,
		l15=0,
		l16=0,
		l17=0,
		l18=0,
		l19=0):
		""" Optimizing function
		Take in hyper parameter values and return valid set performances

		Parameters
		----------
		l00~l19: int or float
		placeholders for hyperparameters being optimized,
		hyper_parameters dict is rebuilt based on input values of placeholders

		Returns:
		--------
		valid_scores: float
		valid set performances
		"""
		args = locals()
		# Input hyper parameters
		i = 0
		for hp in hp_list_single:
		hyper_parameters[hp] = float(args[param_name[i]])
		if param_range[i][0] == 'int':
		hyper_parameters[hp] = int(hyper_parameters[hp])
		i = i + 1
		for hp in hp_list_multiple:
		hyper_parameters[hp[0]] = [
		float(args[param_name[j]]) for j in range(i, i + hp[1])
		]
		if param_range[i][0] == 'int':
		hyper_parameters[hp[0]] = map(int, hyper_parameters[hp[0]])
		i = i + hp[1]

		print(hyper_parameters)
		# Run benchmark
		if isinstance(self.model_class, str) or isinstance(
		self.model_class, unicode):
		try:
		train_scores, valid_scores, _ = benchmark_classification(
		train_dataset,
		valid_dataset,
		valid_dataset, ['task_placeholder'] * n_tasks,
		output_transformers,
		n_features,
		metric,
		self.model_class,
		hyper_parameters=hyper_parameters)
		except AssertionError:
		train_scores, valid_scores, _ = benchmark_regression(
		train_dataset,
		valid_dataset,
		valid_dataset, ['task_placeholder'] * n_tasks,
		output_transformers,
		n_features,
		metric,
		self.model_class,
		hyper_parameters=hyper_parameters)
		return valid_scores[self.model_class][metric[0].name]
		else:
		model_dir = tempfile.mkdtemp()
		model = self.model_class(hyper_parameters, model_dir)
		model.fit(train_dataset, **hyper_parameters)
		model.save()
		evaluator = Evaluator(model, valid_dataset, output_transformers)
		multitask_scores = evaluator.compute_model_performance([metric])
		return multitask_scores[metric.name]

		cov = pyGPGO_covfunc.matern32()
		gp = pyGPGO_surrogates_GaussianProcess.GaussianProcess(cov)
		acq = pyGPGO_acquisition.Acquisition(mode='ExpectedImprovement')
		gpgo = pyGPGO_GPGO.GPGO(gp, acq, f, param)
		gpgo.run(max_iter=max_iter)

		hp_opt, valid_performance_opt = gpgo.getResult()

		# Readout best hyper parameters
		i = 0
		for hp in hp_list_single:
		hyper_parameters[hp] = float(hp_opt[param_name[i]])
		if param_range[i][0] == 'int':
		hyper_parameters[hp] = int(hyper_parameters[hp])
		i = i + 1
		for hp in hp_list_multiple:
		hyper_parameters[hp[0]] = [
		float(hp_opt[param_name[j]]) for j in range(i, i + hp[1])
		]
		if param_range[i][0] == 'int':
		hyper_parameters[hp[0]] = map(int, hyper_parameters[hp[0]])
		i = i + hp[1]

		return hyper_parameters, valid_performance_opt

deepchem/hyper/grid_search.py

0 → 100644

+115 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python2
		# -- coding: utf-8 --
		"""
		Contains basic hyperparameter optimizations.
		"""
		import numpy as np
		import os
		import itertools
		import tempfile
		import shutil
		import collections
		from functools import reduce
		from operator import mul
		from deepchem.utils.evaluate import Evaluator
		from deepchem.utils.save import log


		class HyperparamOpt(object):
		"""
		Provides simple hyperparameter search capabilities.
		"""

		def __init__(self, model_class, verbose=True):
		self.model_class = model_class
		self.verbose = verbose

		# TODO(rbharath): This function is complicated and monolithic. Is there a nice
		# way to refactor this?
		def hyperparam_search(self,
		params_dict,
		train_dataset,
		valid_dataset,
		output_transformers,
		metric,
		use_max=True,
		logdir=None):
		"""Perform hyperparams search according to params_dict.

		Each key to hyperparams_dict is a model_param. The values should be a list
		of potential values for that hyperparam.

		TODO(rbharath): This shouldn't be stored in a temporary directory.
		"""
		hyperparams = params_dict.keys()
		hyperparam_vals = params_dict.values()
		for hyperparam_list in params_dict.values():
		assert isinstance(hyperparam_list, collections.Iterable)

		number_combinations = reduce(mul, [len(vals) for vals in hyperparam_vals])

		if use_max:
		best_validation_score = -np.inf
		else:
		best_validation_score = np.inf
		best_hyperparams = None
		best_model, best_model_dir = None, None
		all_scores = {}
		for ind, hyperparameter_tuple in enumerate(
		itertools.product(*hyperparam_vals)):
		model_params = {}
		log("Fitting model %d/%d" % (ind + 1, number_combinations), self.verbose)
		for hyperparam, hyperparam_val in zip(hyperparams, hyperparameter_tuple):
		model_params[hyperparam] = hyperparam_val
		log("hyperparameters: %s" % str(model_params), self.verbose)

		if logdir is not None:
		model_dir = os.path.join(logdir, str(ind))
		log("model_dir is %s" % model_dir, self.verbose)
		try:
		os.makedirs(model_dir)
		except OSError:
		if not os.path.isdir(model_dir):
		log("Error creating model_dir, using tempfile directory",
		self.verbose)
		model_dir = tempfile.mkdtemp()
		else:
		model_dir = tempfile.mkdtemp()

		model = self.model_class(model_params, model_dir)
		model.fit(train_dataset, **model_params)
		model.save()

		evaluator = Evaluator(model, valid_dataset, output_transformers)
		multitask_scores = evaluator.compute_model_performance([metric])
		valid_score = multitask_scores[metric.name]
		all_scores[str(hyperparameter_tuple)] = valid_score

		if (use_max and valid_score >= best_validation_score) or (
		not use_max and valid_score <= best_validation_score):
		best_validation_score = valid_score
		best_hyperparams = hyperparameter_tuple
		if best_model_dir is not None:
		shutil.rmtree(best_model_dir)
		best_model_dir = model_dir
		best_model = model
		else:
		shutil.rmtree(model_dir)

		log("Model %d/%d, Metric %s, Validation set %s: %f" %
		(ind + 1, number_combinations, metric.name, ind,
		valid_score), self.verbose)
		log("\tbest_validation_score so far: %f" % best_validation_score,
		self.verbose)
		if best_model is None:
		log("No models trained correctly.", self.verbose)
		# arbitrarily return last model
		best_model, best_hyperparams = model, hyperparameter_tuple
		return best_model, best_hyperparams, all_scores
		train_evaluator = Evaluator(best_model, train_dataset, output_transformers)
		multitask_scores = train_evaluator.compute_model_performance([metric])
		train_score = multitask_scores[metric.name]
		log("Best hyperparameters: %s" % str(best_hyperparams), self.verbose)
		log("train_score: %f" % train_score, self.verbose)
		log("validation_score: %f" % best_validation_score, self.verbose)
		return best_model, best_hyperparams, all_scores

deepchem/utils/dependencies.py

+6 −1

Original line number	Diff line number	Diff line
		@@ -10,7 +10,7 @@ def import_lazy(module_name):
		return module
		except:
		print("Warning: No %s installed on your system" % module_name)
		print("Attempting to run %s will throw runtime errors")
		print("Attempting to run %s will throw runtime errors" % module_name)
		return None


		@@ -18,3 +18,8 @@ xgboost = import_lazy("xgboost")
		pdbfixer = import_lazy("pdbfixer")
		simtk = import_lazy("simtk")
		mdtraj = import_lazy("mdtraj")
		pyGPGO_covfunc = import_lazy("pyGPGO.covfunc")
		pyGPGO_acquisition = import_lazy("pyGPGO.acquisition")
		pyGPGO_surrogates_GaussianProcess = import_lazy(
		"pyGPGO.surrogates.GaussianProcess")
		pyGPGO_GPGO = import_lazy("pyGPGO.GPGO")

examples/benchmark_hyperparam_search.py

deleted100644 → 0

+0 −119

Original line number	Diff line number	Diff line
		# -- coding: utf-8 --
		"""
		Created on Mon Nov 07 22:41:34 2016

		@author: Zhenqin Wu
		"""
		import numpy as np
		import deepchem as dc
		import os
		import time
		import sys
		from scipy.stats import truncnorm

		# main layer
		layer_sizes_0 = [1200]
		weight_init_stddevs_0 = [0.02]
		bias_init_consts_0 = [1.]
		dropouts_0 = [0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7]

		#bypass layer
		bypass_layer_sizes_0 = truncnorm(-1, 1, loc=200, scale=150)
		bypass_weight_init_stddevs_0 = [.02]
		bypass_bias_init_consts_0 = [1.]
		bypass_dropouts_0 = [0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7]

		#penalty
		penalty_0 = truncnorm(-1, 1, loc=0.3, scale=0.2)
		penalty_type_0 = ['l2']

		#general figure
		batch_size_0 = [50]
		nb_epoch_0 = [12]

		#learning rate
		learning_rate_0 = truncnorm(-1, 1, loc=-3.2, scale=1.2)

		#for graph-conv and random forest
		n_filters_0 = [64, 96, 128]
		n_fully_connected_nodes_0 = [100, 120, 140, 160, 200, 240, 300]
		n_estimators_0 = [500]
		seed = None

		out_path = '.'

		dname = sys.argv[1]
		model = sys.argv[2]

		parameters_printed = {
		'tf': [
		'layer_sizes', 'weight_init_stddevs', 'bias_init_consts', 'dropouts',
		'penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'
		],
		'tf_robust': [
		'layer_sizes', 'weight_init_stddevs', 'bias_init_consts', 'dropouts',
		'bypass_layer_sizes', 'bypass_weight_init_stddevs',
		'bypass_bias_init_consts', 'bypass_dropouts', 'penalty', 'penalty_type',
		'batch_size', 'nb_epoch', 'learning_rate'
		],
		'logreg':
		['penalty', 'penalty_type', 'batch_size', 'nb_epoch', 'learning_rate'],
		'graphconv': [
		'batch_size', 'nb_epoch', 'learning_rate', 'n_filters',
		'n_fully_connected_nodes'
		],
		'rf': ['n_estimators']
		}
		hps = {}
		for i in range(int(sys.argv[3])):
		layer_sizes = layer_sizes_0
		weight_init_stddevs = weight_init_stddevs_0
		bias_init_consts = bias_init_consts_0
		dropouts = [np.random.choice(dropouts_0)]

		bypass_layer_sizes = [int(bypass_layer_sizes_0.rvs())]
		bypass_weight_init_stddevs = bypass_weight_init_stddevs_0
		bypass_bias_init_consts = bypass_bias_init_consts_0
		bypass_dropouts = [np.random.choice(bypass_dropouts_0)]

		penalty = penalty_0.rvs()
		penalty_type = np.random.choice(penalty_type_0)

		batch_size = np.random.choice(batch_size_0)
		nb_epoch = np.random.choice(nb_epoch_0)

		learning_rate = 10**(learning_rate_0.rvs())

		n_filters = np.random.choice(n_filters_0)
		n_fully_connected_nodes = np.random.choice(n_fully_connected_nodes_0)
		n_estimators = np.random.choice(n_estimators_0)

		hps[model] = {
		'layer_sizes': layer_sizes,
		'weight_init_stddevs': weight_init_stddevs,
		'bias_init_consts': bias_init_consts,
		'dropouts': dropouts,
		'bypass_layer_sizes': bypass_layer_sizes,
		'bypass_weight_init_stddevs': bypass_weight_init_stddevs,
		'bypass_bias_init_consts': bypass_bias_init_consts,
		'bypass_dropouts': bypass_dropouts,
		'penalty': penalty,
		'penalty_type': penalty_type,
		'batch_size': batch_size,
		'nb_epoch': nb_epoch,
		'learning_rate': learning_rate,
		'n_filters': n_filters,
		'n_fully_connected_nodes': n_fully_connected_nodes,
		'n_estimators': n_estimators,
		'seed': seed
		}

		with open(os.path.join(out_path, 'hps.csv'), 'a') as f:
		f.write('\n' + str(i) + ',' + dname + ',')
		for item in hps[model][i]:
		if item in parameters_printed[model]:
		f.write(item + ',')
		f.write(str(hps[model][i][item]) + ',')

		dc.molnet.run_benchmark(
		[dname], str(model), out_path=out_path, hyper_parameters=hps)

Admin message