DeepChem tf changes (46c403da) · Commits · 钟慕尧 / deepchem

deepchem/hyper/init.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -17,7 +17,7 @@ class HyperparamOpt(object):
		Provides simple hyperparameter search capabilities.
		"""

		def __init__(self, model_class, verbosity=None):
		def __init__(self, model_class, verbosity="high"):
		self.model_class = model_class
		assert verbosity in [None, "low", "high"]
		self.verbosity = verbosity

deepchem/models/init.py

+6 −6

Original line number	Diff line number	Diff line
		@@ -10,15 +10,15 @@ from deepchem.models.sklearn_models import SklearnModel
		from deepchem.models.keras_models import KerasModel
		from deepchem.models.tf_keras_models.multitask_classifier import MultitaskGraphClassifier
		from deepchem.models.tf_keras_models.support_classifier import SupportGraphClassifier
		from deepchem.models.tensorflow_models import TensorflowModel
		#from deepchem.models.tensorflow_models import TensorflowModel
		from deepchem.models.multitask import SingletaskToMultitask

		# TODO(rbharath): I'm not sure if these belong here or in deepchem.nn
		# The issue is that these are not valid deepchem models. The solution might be
		# to make inherit from Model class
		from deepchem.models.keras_models.fcnet import MultiTaskDNN
		from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskRegressor
		from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier
		from deepchem.models.tensorflow_models.robust_multitask import RobustMultitaskRegressor
		from deepchem.models.tensorflow_models.robust_multitask import RobustMultitaskClassifier
		from deepchem.models.tensorflow_models.lr import TensorflowLogisticRegression

		# TODO(rbharath): I'm not sure if this model should be exposed. Not in
		# benchmark suite for example.
		from deepchem.models.keras_models.fcnet import MultiTaskDNN

deepchem/models/multitask.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -20,7 +20,7 @@ class SingletaskToMultitask(Model):

		Warning: This current implementation is only functional for sklearn models.
		"""
		def __init__(self, tasks, model_builder, model_dir=None, verbosity=None):
		def __init__(self, tasks, model_builder, model_dir=None, verbosity="high"):
		self.tasks = tasks
		if model_dir is not None:
		if not os.path.exists(model_dir):

deepchem/models/tensorflow_models/init.py

+87 −133

Original line number	Diff line number	Diff line
		@@ -55,9 +55,6 @@ class TensorflowGraph(object):
		def get_placeholder_scope(graph, name_scopes):
		"""Gets placeholder scope."""
		placeholder_root = "placeholders"
		#with graph.as_default():
		# with tf.name_scope(placeholder_root) as scope:
		# return scope
		return TensorflowGraph.shared_name_scope(placeholder_root, graph, name_scopes)

		@staticmethod
		@@ -86,7 +83,8 @@ class TensorflowGraph(object):
		return feed_dict


		class TensorflowGraphModel(object):
		#class TensorflowGraphModel(object):
		class TensorflowGraphModel(Model):
		"""Parent class for deepchem Tensorflow models.

		Classifier:
		@@ -162,6 +160,24 @@ class TensorflowGraphModel(object):
		self.train_graph = self.construct_graph(training=True)
		self.eval_graph = self.construct_graph(training=False)

		################################################################ DEBUG

		def save(self):
		"""
		No-op since tf models save themselves during fit()
		"""
		pass

		def reload(self):
		"""
		Loads model from disk. Thin wrapper around restore() for consistency.
		"""
		self.model_instance.restore()

		def get_num_tasks(self):
		return self.n_tasks
		################################################################ DEBUG


		def construct_graph(self, training):
		"""Returns a TensorflowGraph object."""
		@@ -263,7 +279,12 @@ class TensorflowGraphModel(object):
		log("About to shuffle dataset before epoch start.", self.verbosity)
		dataset.shuffle()
		for ind, (X_b, y_b, w_b, ids_b) in enumerate(
		dataset.iterbatches(batch_size, pad_batches=True)): # hardcode pad_batches=True to work around limitations in Tensorflow
		############################################################ DEBUG
		## hardcode pad_batches=True to work around limitations in Tensorflow
		#dataset.iterbatches(batch_size, pad_batches=True)):
		dataset.iterbatches(batch_size, pad_batches=False)):
		#dataset.iterbatches(batch_size, pad_batches=pad_batches)):
		############################################################ DEBUG
		if ind % log_every_N_batches == 0:
		log("On batch %d" % ind, self.verbosity)
		# Run training op.
		@@ -290,58 +311,6 @@ class TensorflowGraphModel(object):
		self.verbosity)
		############################################################## TIMING

		def predict_on_batch(self, X, pad_batch=False):
		"""Return model output for the provided input.

		Restore(checkpoint) must have previously been called on this object.

		Args:
		dataset: dc.data.dataset object.

		Returns:
		Tuple of three numpy arrays with shape n_examples x n_tasks (x ...):
		output: Model outputs.
		labels: True labels.
		weights: Example weights.
		Note that the output and labels arrays may be more than 2D, e.g. for
		classifier models that return class probabilities.

		Raises:
		AssertionError: If model is not in evaluation mode.
		ValueError: If output and labels are not both 3D or both 2D.
		"""
		if pad_batch:
		X = pad_features(self.batch_size, X)

		if not self._restored_model:
		self.restore()
		with self.eval_graph.graph.as_default():

		# run eval data through the model
		n_tasks = self.n_tasks
		output = []
		start = time.time()
		with self._get_shared_session(train=False).as_default():
		feed_dict = self.construct_feed_dict(X)
		data = self._get_shared_session(train=False).run(
		self.eval_graph.output, feed_dict=feed_dict)
		batch_output = np.asarray(data[:n_tasks], dtype=float)
		# reshape to batch_size x n_tasks x ...
		if batch_output.ndim == 3:
		batch_output = batch_output.transpose((1, 0, 2))
		elif batch_output.ndim == 2:
		batch_output = batch_output.transpose((1, 0))
		else:
		raise ValueError(
		'Unrecognized rank combination for output: %s' %
		(batch_output.shape,))
		output.append(batch_output)

		outputs = np.array(from_one_hot(
		np.squeeze(np.concatenate(output)), axis=-1))

		return np.copy(outputs)

		def add_output_ops(self, graph, output):
		"""Replace logits with softmax outputs."""
		with graph.as_default():
		@@ -529,6 +498,60 @@ class TensorflowClassifier(TensorflowGraphModel):
		name='labels_%d' % task)))
		return labels

		def predict_on_batch(self, X, pad_batch=False):
		"""Return model output for the provided input.

		Restore(checkpoint) must have previously been called on this object.

		Args:
		dataset: dc.data.dataset object.

		Returns:
		Tuple of three numpy arrays with shape n_examples x n_tasks (x ...):
		output: Model outputs.
		labels: True labels.
		weights: Example weights.
		Note that the output and labels arrays may be more than 2D, e.g. for
		classifier models that return class probabilities.

		Raises:
		AssertionError: If model is not in evaluation mode.
		ValueError: If output and labels are not both 3D or both 2D.
		"""
		len_unpadded = len(X)
		if pad_batch:
		X = pad_features(self.batch_size, X)

		if not self._restored_model:
		self.restore()
		with self.eval_graph.graph.as_default():

		# run eval data through the model
		n_tasks = self.n_tasks
		output = []
		start = time.time()
		with self._get_shared_session(train=False).as_default():
		feed_dict = self.construct_feed_dict(X)
		data = self._get_shared_session(train=False).run(
		self.eval_graph.output, feed_dict=feed_dict)
		batch_output = np.asarray(data[:n_tasks], dtype=float)
		# reshape to batch_size x n_tasks x ...
		if batch_output.ndim == 3:
		batch_output = batch_output.transpose((1, 0, 2))
		elif batch_output.ndim == 2:
		batch_output = batch_output.transpose((1, 0))
		else:
		raise ValueError(
		'Unrecognized rank combination for output: %s' %
		(batch_output.shape,))
		output.append(batch_output)

		outputs = np.array(from_one_hot(
		np.squeeze(np.concatenate(output)), axis=-1))

		outputs = np.copy(outputs)
		return outputs[:len_unpadded]

		def predict_proba_on_batch(self, X, pad_batch=False):
		"""Return model output for the provided input.

		@@ -628,7 +651,7 @@ class TensorflowRegressor(TensorflowGraphModel):
		name='labels_%d' % task)))
		return labels

		def predict_on_batch(self, X):
		def predict_on_batch(self, X, pad_batch=False):
		"""Return model output for the provided input.

		Restore(checkpoint) must have previously been called on this object.
		@@ -648,6 +671,10 @@ class TensorflowRegressor(TensorflowGraphModel):
		AssertionError: If model is not in evaluation mode.
		ValueError: If output and labels are not both 3D or both 2D.
		"""
		len_unpadded = len(X)
		if pad_batch:
		X = pad_features(self.batch_size, X)

		if not self._restored_model:
		self.restore()
		with self.train_graph.graph.as_default():
		@@ -659,12 +686,6 @@ class TensorflowRegressor(TensorflowGraphModel):
		n_samples = len(X)
		# TODO(rbharath): Should this be padding there? Shouldn't padding be
		# turned on in predict?
		#################################################################### DEBUG
		# Some tensorflow models can't handle variadic batches,
		# especially models using tf.pack, tf.split. Pad batch-size
		# to handle these cases.
		#X = pad_features(self.batch_size, X)
		#################################################################### DEBUG
		feed_dict = self.construct_feed_dict(X)
		data = self._get_shared_session(train=False).run(
		self.eval_graph.output, feed_dict=feed_dict)
		@@ -688,72 +709,5 @@ class TensorflowRegressor(TensorflowGraphModel):

		outputs = np.squeeze(np.concatenate(outputs))

		return np.copy(outputs)

		class TensorflowModel(Model):
		"""
		Abstract base class shared across all Tensorflow models.
		"""

		def __init__(self, model, verbosity=None, **kwargs):
		assert verbosity in [None, "low", "high"]
		self.verbosity = verbosity
		self.model_instance = model
		self.fit_transformers = None

		def fit(self, dataset, **kwargs):
		"""
		Fits TensorflowGraph to data.
		"""
		self.model_instance.fit(dataset, **kwargs)

		def predict(self, dataset, transformers=[], batch_size=None,
		pad_batches=False):
		"""
		Uses self to make predictions on provided Dataset object.

		This is overridden to make sure the batch size is always valid for Tensorflow.

		Returns:
		y_pred: numpy ndarray of shape (n_samples,)
		"""
		return Model.predict(self, dataset, transformers,
		self.model_instance.batch_size, True)

		def predict_on_batch(self, X, pad_batch=True):
		"""
		Makes predictions on batch of data.
		"""
		if pad_batch:
		len_unpadded = len(X)
		Xpad = pad_features(self.model_instance.batch_size, X)
		return self.model_instance.predict_on_batch(Xpad)[:len_unpadded]
		else:
		return self.model_instance.predict_on_batch(X)

		def predict_grad_on_batch(self, X):
		"""
		Calculates gradient of cost function on batch of data.
		"""
		return self.model_instance.predict_grad_on_batch(X)

		def predict_proba_on_batch(self, X, pad_batch=False):
		"""
		Makes predictions on batch of data.
		"""
		return self.model_instance.predict_proba_on_batch(X, pad_batch=pad_batch)

		def save(self):
		"""
		No-op since tf models save themselves during fit()
		"""
		pass

		def reload(self):
		"""
		Loads model from disk. Thin wrapper around restore() for consistency.
		"""
		self.model_instance.restore()

		def get_num_tasks(self):
		return self.model_instance.n_tasks
		outputs = np.copy(outputs)
		return outputs[:len_unpadded]

examples/benchmark.py

+74 −45

Original line number	Diff line number	Diff line
		@@ -18,8 +18,8 @@ pcba - dataloading: 30min
		- tf: 2h
		sider - dataloading: 10s
		- tf: 60s
		toxcast dataloading: 70s
		tf: 40min
		toxcast - dataloading: 70s
		- tf: 40min
		(will include more)

		Total time of running a benchmark test: 3~4h
		@@ -76,7 +76,7 @@ def benchmark_loading_datasets(base_dir_o, hyper_parameters,
		if model in ['graphconv']:
		featurizer = 'GraphConv'
		n_features = 71
		elif model in ['tf','logreg','rf']:
		elif model in ['tf', 'tf_robust', 'logreg', 'rf']:
		featurizer = 'ECFP'
		n_features = 1024
		else:
		@@ -183,12 +183,9 @@ def benchmark_train_and_valid(base_dir,train_dataset,valid_dataset,tasks,
		verbosity=verbosity,
		mode="classification")

		assert model in ['graphconv', 'tf', 'rf','logreg']
		assert model in ['graphconv', 'tf', 'tf_robust', 'rf','logreg']

		if model == 'tf':
		# Initialize model folder
		model_dir_tf = os.path.join(base_dir, "model_tf")

		# Building tensorflow MultiTaskDNN model
		dropouts = hyper_parameters['dropouts']
		learning_rate = hyper_parameters['learning_rate']
		@@ -196,26 +193,53 @@ def benchmark_train_and_valid(base_dir,train_dataset,valid_dataset,tasks,
		batch_size = hyper_parameters['batch_size']
		nb_epoch = hyper_parameters['nb_epoch']

		tensorflow_model = dc.models.TensorflowMultiTaskClassifier(len(tasks),
		model_tf = dc.models.TensorflowMultiTaskClassifier(len(tasks),
		n_features, learning_rate=learning_rate, layer_sizes=layer_sizes,
		dropouts=dropouts, batch_size=batch_size,
		verbosity=verbosity)
		model_tf = dc.models.TensorflowModel(tensorflow_model)

		print('-------------------------------------')
		print('Start fitting by tensorflow')
		model_tf.fit(train_dataset,nb_epoch = nb_epoch)

		train_scores['tensorflow'] = model_tf.evaluate(train_dataset,
		[classification_metric],transformers)
		train_scores['tensorflow'] = model_tf.evaluate(
		train_dataset, [classification_metric], transformers)

		valid_scores['tensorflow'] = model_tf.evaluate(valid_dataset,
		[classification_metric],transformers)
		valid_scores['tensorflow'] = model_tf.evaluate(
		valid_dataset, [classification_metric], transformers)

		if model == 'logreg':
		# Initialize model folder
		model_dir_logreg = os.path.join(base_dir, "model_logreg")
		if model == 'tf_robust':
		# Building tensorflow MultiTaskDNN model
		dropouts = hyper_parameters['dropouts']
		bypass_dropouts = hyper_parameters['bypass_dropouts']
		learning_rate = hyper_parameters['learning_rate']
		layer_sizes = hyper_parameters['layer_sizes']
		bypass_layer_sizes = hyper_parameters['bypass_layer_sizes']
		batch_size = hyper_parameters['batch_size']
		nb_epoch = hyper_parameters['nb_epoch']

		model_tf = dc.models.TensorflowMultiTaskClassifier(len(tasks),
		n_features, learning_rate=learning_rate, layer_sizes=layer_sizes,
		dropouts=dropouts, batch_size=batch_size,
		verbosity=verbosity)
		model_robust = dc.models.RobustMultitaskClassifier(
		len(tasks), n_features, learning_rate=learning_rate,
		layer_sizes=layer_sizes, bypass_layer_sizes=bypass_layer_sizes,
		dropouts=dropouts, bypass_dropouts=bypass_dropouts,
		batch_size=batch_size, verbosity="high")

		print('-------------------------------------')
		print('Start fitting by tensorflow')
		model_robust.fit(train_dataset,nb_epoch = nb_epoch)

		train_scores['tf_robust'] = model_robust.evaluate(
		train_dataset, [classification_metric], transformers)

		valid_scores['tf_robust'] = model_robust.evaluate(
		valid_dataset, [classification_metric], transformers)


		if model == 'logreg':
		# Building tensorflow logistic regression model
		learning_rate = hyper_parameters['learning_rate']
		penalty = hyper_parameters['penalty']
		@@ -223,11 +247,10 @@ def benchmark_train_and_valid(base_dir,train_dataset,valid_dataset,tasks,
		batch_size = hyper_parameters['batch_size']
		nb_epoch = hyper_parameters['nb_epoch']

		tensorflow_model = dc.models.TensorflowLogisticRegression(len(tasks),
		model_logreg = dc.models.TensorflowLogisticRegression(len(tasks),
		n_features, learning_rate=learning_rate, penalty=penalty,
		penalty_type=penalty_type, batch_size=batch_size,
		verbosity=verbosity)
		model_logreg = dc.models.TensorflowModel(tensorflow_model)

		print('-------------------------------------')
		print('Start fitting by logistic regression')
		@@ -317,21 +340,27 @@ if __name__ == '__main__':
		os.makedirs(base_dir_o)

		#Datasets and models used in the benchmark test, all=all the datasets
		dataset_name = 'tox21'
		dataset_name = 'muv'
		model = 'tf'

		#input hyperparameters
		#tf: dropouts, learning rate, layer_sizes, weight initial stddev,penalty,
		# batch_size
		hps = {}
		hps['tf'] = [{'dropouts':[0.25],'learning_rate':0.001,'layer_sizes':[1000],
		hps['tf'] = [{'dropouts': [0.25], 'learning_rate': 0.001,
		'layer_sizes': [1000], 'batch_size': 50, 'nb_epoch': 10}]

		hps['tf_robust'] = [{'dropouts': [0.5], 'bypass_dropouts': [0.5],
		'learning_rate': 0.001,
		'layer_sizes': [500], 'bypass_layer_sizes': [100],
		'batch_size': 50, 'nb_epoch': 10}]

		hps['logreg'] = [{'learning_rate': 0.001, 'penalty': 0.05,
		'penalty_type': 'l1', 'batch_size': 50, 'nb_epoch': 10}]

		hps['graphconv'] = [{'learning_rate': 0.001, 'n_filters': 64,
		'n_fully_connected_nodes':128, 'batch_size':50, 'nb_epoch':10}]
		'n_fully_connected_nodes': 128, 'batch_size': 50,
		'nb_epoch': 10}]

		hps['rf'] = [{'n_estimators': 500}]

Admin message