Progressive model passes basic sanity tests. (95222430) · Commits · 钟慕尧 / deepchem

deepchem/feat/featurize.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -138,7 +138,7 @@ class DataLoader(object):

		def featurize(self, input_files, data_dir=None, shard_size=8192,
		num_shards_per_batch=24, worker_pool=None,
		logging=True, debug=False):
		logging=True, debug=True):
		"""Featurize provided files and write to specified location."""
		############################################################## TIMING
		time1 = time.time()

deepchem/models/init.py

+0 −4

Original line number	Diff line number	Diff line
		@@ -18,7 +18,3 @@ from deepchem.models.tensorflow_models.robust_multitask import RobustMultitaskRe
		from deepchem.models.tensorflow_models.robust_multitask import RobustMultitaskClassifier
		from deepchem.models.tensorflow_models.lr import TensorflowLogisticRegression
		from deepchem.models.tensorflow_models.progressive_multitask import ProgressiveMultitaskRegressor

		# TODO(rbharath): I'm not sure if this model should be exposed. Not in
		# benchmark suite for example.
		from deepchem.models.keras_models.fcnet import MultiTaskDNN

deepchem/models/tensorflow_models/progressive_multitask.py

+60 −175

Original line number	Diff line number	Diff line
		@@ -43,6 +43,16 @@ class ProgressiveMultitaskRegressor(TensorflowMultiTaskRegressor):
		super(ProgressiveMultitaskRegressor, self).__init__(
		n_tasks, n_features, **kwargs)

		# Consistency check
		lengths_set = {
		len(self.layer_sizes),
		len(self.weight_init_stddevs),
		len(self.alpha_init_stddevs),
		len(self.bias_init_consts),
		len(self.dropouts),
		}
		assert len(lengths_set) == 1, "All layer params must have same length."

		def construct_graph(self, training, seed):
		"""Returns a TensorflowGraph object."""
		graph = tf.Graph()
		@@ -58,61 +68,20 @@ class ProgressiveMultitaskRegressor(TensorflowMultiTaskRegressor):
		with graph.as_default():
		if seed is not None:
		tf.set_random_seed(seed)
		output = self.build(graph, name_scopes, training)
		labels = self.add_label_placeholders(graph, name_scopes)
		weights = self.add_example_weight_placeholders(graph, name_scopes)
		outputs, labels, weights = self.build(graph, name_scopes, training)

		if training:
		loss = self.add_training_costs(graph, name_scopes, output, labels, weights)
		loss = self.add_training_costs(graph, name_scopes, outputs, labels, weights)
		else:
		loss = None
		output = self.add_output_ops(graph, output) # add softmax heads
		return TensorflowGraph(graph=graph,
		session=shared_session,
		name_scopes=name_scopes,
		output=output,
		output=outputs,
		labels=labels,
		weights=weights,
		loss=loss)

		def add_label_placeholders(self, graph, name_scopes):
		"""Add Placeholders for labels for each task.

		This method creates the following Placeholders for each task:
		labels_%d: Label tensor with shape batch_size.

		Placeholders are wrapped in identity ops to avoid the error caused by
		feeding and fetching the same tensor.
		"""
		placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes)
		with graph.as_default():
		batch_size = self.batch_size
		labels = []
		with placeholder_scope:
		for task in range(self.n_tasks):
		labels.append(tf.identity(
		tf.placeholder(tf.float32, shape=[None, 1],
		name='labels_%d' % task)))
		return labels

		def add_example_weight_placeholders(self, graph, name_scopes):
		"""Add Placeholders for example weights for each task.

		This method creates the following Placeholders for each task:
		weights_%d: Label tensor with shape batch_size.

		Placeholders are wrapped in identity ops to avoid the error caused by
		feeding and fetching the same tensor.
		"""
		weights = []
		placeholder_scope = TensorflowGraph.get_placeholder_scope(graph, name_scopes)
		with placeholder_scope:
		for task in range(self.n_tasks):
		weights.append(tf.identity(
		tf.placeholder(tf.float32, shape=[None, 1],
		name='weights_%d' % task)))
		return weights

		def build(self, graph, name_scopes, training):
		"""Constructs the graph architecture as specified in its config.

		@@ -126,86 +95,70 @@ class ProgressiveMultitaskRegressor(TensorflowMultiTaskRegressor):
		Indicates whether this graph is to be constructed in training
		or evaluation mode. Mainly used for dropout
		"""
		# Create the scope for placeholders
		# Create placeholders
		placeholder_scope = TensorflowGraph.get_placeholder_scope(
		graph, name_scopes)
		# Create the namescopes for each task
		self.task_scopes = {}
		for task in range(self.n_tasks):
		task_root = "task%d" % task
		self.task_scopes[task] = TensorflowGraph.shared_name_scope(
		"task%d" % task, graph, name_scopes)

		with graph.as_default():
		labels, weights = [], []
		with placeholder_scope:
		self.features = tf.placeholder(
		features = tf.placeholder(
		tf.float32, shape=[None, self.n_features], name='features')
		for task in range(self.n_tasks):
		weights.append(tf.identity(
		tf.placeholder(tf.float32, shape=[None, 1],
		name='weights_%d' % task)))
		labels.append(tf.identity(
		tf.placeholder(tf.float32, shape=[None, 1],
		name='labels_%d' % task)))

		# Define graph structure
		layer_sizes = self.layer_sizes
		alpha_init_stddevs = self.alpha_init_stddevs
		weight_init_stddevs = self.weight_init_stddevs
		bias_init_consts = self.bias_init_consts
		dropouts = self.dropouts

		# Consistency check
		lengths_set = {
		len(layer_sizes),
		len(weight_init_stddevs),
		len(alpha_init_stddevs),
		len(bias_init_consts),
		len(dropouts),
		}
		assert len(lengths_set) == 1, "All layer params must have same length."
		n_layers = lengths_set.pop()

		all_layers = {}
		for i in range(n_layers):
		for i in range(len(layer_sizes)):
		for task in range(self.n_tasks):
		task_scope = self.task_scopes[task]
		task_scope = TensorflowGraph.shared_name_scope(
		"task%d" % task, graph, name_scopes)
		print("Adding weights for task %d, layer %d" % (task, i))
		with task_scope:
		with task_scope as scope:
		# Create the non-linear adapter
		if i == 0:
		prev_layer = self.features
		prev_layer = features
		else:
		pass
		############################################################### DEBUG
		prev_layer = all_layers[(i-1, task)]
		if task > 0:
		prev_layers = []
		# Iterate over all previous tasks.
		for prev_task in range(task-1):
		for prev_task in range(task):
		prev_layers.append(all_layers[(i-1, prev_task)])
		# prev_layers is a list with elements of size
		# (batch_size, layer_sizes[i-1])
		prev_layer = tf.concat(1, prev_layers)
		alpha = tf.Variable(tf.truncated_normal(
		[1,], stddev=alpha_init_stddevs[i]))
		[1,], stddev=self.alpha_init_stddevs[i]))
		prev_layer = tf.mul(alpha, prev_layer)
		prev_layer_size = (task-1)*layer_sizes[i-1]
		prev_layer_size = task*layer_sizes[i-1]
		print("Creating V_layer_%d_task%d of shape %s" %
		(i, task, str([prev_layer_size, layer_sizes[i-1]])))
		V = tf.Variable(
		tf.truncated_normal(
		shape=[prev_layer_size, layer_sizes[i-1]],
		stddev=weight_init_stddevs[i]),
		stddev=self.weight_init_stddevs[i]),
		name="V_layer_%d_task%d" % (i, task), dtype=tf.float32)
		print("Creating b_lat_layer_%d_task%d of shape %s" %
		(i, task, str([prev_layer_size])))
		b_lat = tf.Variable(
		tf.constant(value=bias_init_consts[i],
		shape=[prev_layer_size]),
		tf.constant(value=self.bias_init_consts[i],
		shape=[layer_sizes[i-1]]),
		name='b_lat_layer_%d_task%d' % (i, task),
		dtype=tf.float32)
		prev_layer = tf.nn.xw_plus_b(prev_layer, V, b_lat)
		print("Creating V_layer_%d_task%d of shape %s" %
		prev_layer = tf.matmul(prev_layer, V) + b_lat
		print("Creating U_layer_%d_task%d of shape %s" %
		(i, task, str([layer_sizes[i-1], layer_sizes[i]])))
		U = tf.Variable(
		tf.truncated_normal(
		shape=[layer_sizes[i-1], layer_sizes[i]],
		stddev=weight_init_stddevs[i]),
		stddev=self.weight_init_stddevs[i]),
		name="U_layer_%d_task%d" % (i, task), dtype=tf.float32)
		lateral_contrib = tf.matmul(U, prev_layer)
		############################################################### DEBUG
		lateral_contrib = tf.matmul(prev_layer, U)

		if i == 0:
		prev_layer_size = self.n_features
		@@ -216,35 +169,23 @@ class ProgressiveMultitaskRegressor(TensorflowMultiTaskRegressor):
		W = tf.Variable(
		tf.truncated_normal(
		shape=[prev_layer_size, layer_sizes[i]],
		stddev=weight_init_stddevs[i]),
		stddev=self.weight_init_stddevs[i]),
		name='W_layer_%d_task%d' % (i, task), dtype=tf.float32)
		print("Creating b_layer_%d_task%d of shape %s" %
		(i, task, str([layer_sizes[i]])))
		b = tf.Variable(tf.constant(value=bias_init_consts[i],
		b = tf.Variable(tf.constant(value=self.bias_init_consts[i],
		shape=[layer_sizes[i]]),
		name='b_layer_%d_task%d' % (i, task), dtype=tf.float32)
		layer = tf.nn.xw_plus_b(prev_layer, W, b)
		#################################################### DEBUG
		print("layer")
		print(layer)
		#################################################### DEBUG
		layer = tf.matmul(prev_layer, W) + b
		if i > 0 and task > 0:
		layer = tf.add(layer, lateral_contrib)
		layer = tf.nn.relu(layer)
		#################################################### DEBUG
		print("layer")
		print(layer)
		#################################################### DEBUG
		# layer is of shape (batch_size, layer_sizes[i])
		layer = model_ops.dropout(layer, dropouts[i], training)
		#################################################### DEBUG
		print("layer")
		print(layer)
		#################################################### DEBUG
		layer = model_ops.dropout(layer, self.dropouts[i], training)
		all_layers[(i, task)] = layer
		# Gather up all the outputs to return.
		outputs = [all_layers[(i, task)] for task in range(self.n_tasks)]
		return outputs
		return outputs, labels, weights

		def add_training_costs(self, graph, name_scopes, outputs, labels, weights):
		"""Adds the training costs for each task.
		@@ -267,48 +208,23 @@ class ProgressiveMultitaskRegressor(TensorflowMultiTaskRegressor):
		weights: list
		List of weight placeholders for model.
		"""
		###################################################### DEBUG
		print("add_training_cost()")
		###################################################### DEBUG
		task_costs = {}
		with graph.as_default():
		epsilon = 1e-3 # small float to avoid dividing by zero
		weighted_costs = [] # weighted costs for each example
		gradient_costs = [] # costs used for gradient calculation

		with TensorflowGraph.shared_name_scope('costs', graph, name_scopes):
		for task in range(self.n_tasks):
		with TensorflowGraph.shared_name_scope(
		'cost_%d' % task, graph, name_scopes):
		with tf.name_scope('weighted'):
		weighted_cost = self.cost(outputs[task], labels[task],
		weights[task])
		###################################################### DEBUG
		print("weighted_cost")
		print(weighted_cost)
		print("outputs[task], labels[task], weights[task]")
		print(outputs[task], labels[task], weights[task])
		###################################################### DEBUG
		weighted_costs.append(weighted_cost)

		with tf.name_scope('gradient'):

		# Note that we divide by the batch size and not the number of
		# non-zero weight examples in the batch. Also, instead of using
		# tf.reduce_mean (which can put ops on the CPU) we explicitly
		# calculate with div/sum so it stays on the GPU.
		task_cost = tf.div(tf.reduce_sum(weighted_cost), self.batch_size)
		task_costs[task] = task_cost
		###################################################### DEBUG
		print("task_costs")
		print(task_costs)
		###################################################### DEBUG

		return task_costs

		def add_output_ops(self, graph, outputs):
		"""No-op for regression models since no softmax."""
		return outputs

		def fit(self, dataset, max_checkpoints_to_keep=5, **kwargs):
		"""Fit the model.

		@@ -366,25 +282,11 @@ class ProgressiveMultitaskRegressor(TensorflowMultiTaskRegressor):
		task_root = "task%d" % task
		task_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, task_root)
		opt = model_ops.optimizer(self.optimizer, self.learning_rate, self.momentum)
		######################################################## DEBUG
		print("get_training_op()")
		print("task")
		print(task)
		print("task_root")
		print(task_root)
		print("[task_var.name for task_var in task_vars]")
		print([task_var.name for task_var in task_vars])
		######################################################## DEBUG
		return opt.minimize(task_loss, name='train', var_list=task_vars)

		def _get_shared_session(self, train):
		# allow_soft_placement=True allows ops without a GPU implementation
		# to run on the CPU instead.
		############################################### DEBUG
		print("_get_shared_session")
		print("self.train_graph.session")
		print(self.train_graph.session)
		############################################### DEBUG
		if train:
		if not self.train_graph.session:
		config = tf.ConfigProto(allow_soft_placement=True)
		@@ -432,14 +334,6 @@ class ProgressiveMultitaskRegressor(TensorflowMultiTaskRegressor):
		time1 = time.time()
		############################################################## TIMING
		log("Training task %d for %d epochs" % (task, nb_epoch), self.verbosity)
		##with self.train_graph.graph.as_default():
		# #task_train_op = self.get_training_op(
		# # self.train_graph.graph, self.train_graph.loss, task)
		# #with self._get_shared_session(train=True) as sess:
		# #sess.run(tf.initialize_all_variables())
		# #saver = tf.train.Saver(max_to_keep=max_checkpoints_to_keep)
		# ## Save an initial checkpoint.
		# #saver.save(sess, self._save_path, global_step=0)
		for epoch in range(nb_epoch):
		avg_loss, n_batches = 0., 0
		for ind, (X_b, y_b, w_b, ids_b) in enumerate(
		@@ -462,8 +356,6 @@ class ProgressiveMultitaskRegressor(TensorflowMultiTaskRegressor):
		#saver.save(sess, self._save_path, global_step=epoch)
		avg_loss = float(avg_loss)/n_batches
		log('Ending epoch %d: Average loss %g' % (epoch, avg_loss), self.verbosity)
		# ## Always save a final checkpoint when complete.
		# #saver.save(sess, self._save_path, global_step=nb_epoch)
		############################################################## TIMING
		time2 = time.time()
		print("TIMING: model fitting took %0.3f s" % (time2-time1),
		@@ -496,11 +388,4 @@ class ProgressiveMultitaskRegressor(TensorflowMultiTaskRegressor):
		# Dummy placeholders
		orig_dict["weights_%d" % task] = np.ones(
		(self.batch_size,))
		################################################################ DEBUG
		#for key, value in orig_dict.iteritems():
		# print("key")
		# print(key)
		# print("value.shape")
		# print(value.shape)
		################################################################ DEBUG
		return TensorflowGraph.get_feed_dict(orig_dict)

deepchem/models/tensorflow_models/tests/test_progressive.py

+52 −0

Original line number	Diff line number	Diff line
		@@ -53,3 +53,55 @@ class TestProgressive(test_util.TensorFlowTestCase):
		batch_size=2, verbosity="high")

		prog_model.fit(dataset)

		def test_fit_lateral(self):
		"""Test that multilayer model fits correctly.

		Lateral connections and adapters are only added for multilayer models. Test
		that fit functions with multilayer models.
		"""
		n_tasks = 2
		n_samples = 10
		n_features = 100
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.zeros((n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		n_layers = 3
		prog_model = dc.models.ProgressiveMultitaskRegressor(
		n_tasks=n_tasks, n_features=n_features,
		alpha_init_stddevs=[.08]n_layers, layer_sizes=[100]n_layers,
		weight_init_stddevs=[.02]n_layers, bias_init_consts=[1.]n_layers,
		dropouts=[0.]*n_layers, learning_rate=0.003,
		batch_size=2, verbosity="high")

		prog_model.fit(dataset)

		def test_fit_lateral_multi(self):
		"""Test that multilayer model fits correctly.

		Test multilayer model with multiple tasks (> 2) to verify that lateral
		connections of growing size work correctly.
		"""
		n_tasks = 3
		n_samples = 10
		n_features = 100
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.zeros((n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))
		dataset = dc.data.NumpyDataset(X, y, w, ids)

		n_layers = 3
		prog_model = dc.models.ProgressiveMultitaskRegressor(
		n_tasks=n_tasks, n_features=n_features,
		alpha_init_stddevs=[.08]n_layers, layer_sizes=[100]n_layers,
		weight_init_stddevs=[.02]n_layers, bias_init_consts=[1.]n_layers,
		dropouts=[0.]*n_layers, learning_rate=0.003,
		batch_size=2, verbosity="high")

		prog_model.fit(dataset)

Admin message