Merge pull request #1054 from miaecle/irv (f3ccbc93) · Commits · 钟慕尧 / deepchem

deepchem/models/init.py

+1 −0

Original line number	Diff line number	Diff line
		@@ -13,6 +13,7 @@ from deepchem.models.multitask import SingletaskToMultitask
		from deepchem.models.tensorflow_models.fcnet import MultiTaskRegressor
		from deepchem.models.tensorflow_models.fcnet import MultiTaskClassifier
		from deepchem.models.tensorflow_models.fcnet import MultiTaskFitTransformRegressor
		from deepchem.models.tensorflow_models.IRV import TensorflowMultiTaskIRVClassifier
		from deepchem.models.tensorgraph.tensor_graph import TensorGraph
		from deepchem.models.tensorgraph.models.graph_models import WeaveTensorGraph, DTNNTensorGraph, DAGTensorGraph, GraphConvTensorGraph, MPNNTensorGraph
		from deepchem.models.tensorgraph.models.symmetry_function_regression import BPSymmetryFunctionRegression, ANIRegression

deepchem/models/tensorflow_models/IRV.py

+201 −100

Original line number	Diff line number	Diff line
		@@ -2,28 +2,153 @@ from __future__ import print_function
		from __future__ import division
		from __future__ import unicode_literals

		import warnings
		import time
		import numpy as np
		import tensorflow as tf

		from deepchem.utils.save import log
		'''
		class TensorflowMultiTaskIRVClassifier(TensorflowLogisticRegression):
		from deepchem.models.tensorgraph.tensor_graph import TensorGraph
		from deepchem.models.tensorgraph.layers import Layer, SigmoidCrossEntropy, \
		Sigmoid, Feature, Label, Weights, Concat, WeightedError
		from deepchem.models.tensorgraph.layers import convert_to_layers
		from deepchem.trans import undo_transforms


		class IRVLayer(Layer):
		""" Core layer of IRV classifier, architecture described in:
		https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/
		"""

		def __init__(self, n_tasks, K, **kwargs):
		"""
		Parameters
		----------
		n_tasks: int
		Number of tasks
		K: int
		Number of nearest neighbours used in classification
		"""
		self.n_tasks = n_tasks
		self.K = K
		self.V, self.W, self.b, self.b2 = None, None, None, None
		super(IRVLayer, self).__init__(**kwargs)

		def build(self):
		self.V = tf.Variable(tf.constant([0.01, 1.]), name="vote", dtype=tf.float32)
		self.W = tf.Variable(tf.constant([1., 1.]), name="w", dtype=tf.float32)
		self.b = tf.Variable(tf.constant([0.01]), name="b", dtype=tf.float32)
		self.b2 = tf.Variable(tf.constant([0.01]), name="b2", dtype=tf.float32)
		self.trainable_weights = [self.V, self.W, self.b, self.b2]

		def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
		if in_layers is None:
		in_layers = self.in_layers
		in_layers = convert_to_layers(in_layers)

		self.build()
		inputs = in_layers[0].out_tensor
		K = self.K
		outputs = []
		for count in range(self.n_tasks):
		# Similarity values
		similarity = inputs[:, 2 * K * count:(2 * K * count + K)]
		# Labels for all top K similar samples
		ys = tf.to_int32(inputs[:, (2 * K * count + K):2 * K * (count + 1)])

		R = self.b + self.W[0] * similarity + self.W[1] * tf.constant(
		np.arange(K) + 1, dtype=tf.float32)
		R = tf.sigmoid(R)
		z = tf.reduce_sum(R * tf.gather(self.V, ys), axis=1) + self.b2
		outputs.append(tf.reshape(z, shape=[-1, 1]))
		out_tensor = tf.concat(outputs, axis=1)

		if set_tensors:
		self.variables = self.trainable_weights
		self.out_tensor = out_tensor
		return out_tensor

		def none_tensors(self):
		V, W, b, b2 = self.V, self.W, self.b, self.b2
		self.V, self.W, self.b, self.b2 = None, None, None, None

		out_tensor, trainable_weights, variables = self.out_tensor, self.trainable_weights, self.variables
		self.out_tensor, self.trainable_weights, self.variables = None, [], []
		return V, W, b, b2, out_tensor, trainable_weights, variables

		def set_tensors(self, tensor):
		self.V, self.W, self.b, self.b2, self.out_tensor, self.trainable_weights, self.variables = tensor


		class IRVRegularize(Layer):
		""" Extracts the trainable weights in IRVLayer
		and return their L2-norm
		No in_layers is required, but should be built after target IRVLayer
		"""

		def __init__(self, IRVLayer, penalty=0.0, **kwargs):
		"""
		Parameters
		----------
		IRVLayer: IRVLayer
		Target layer for extracting weights and regularization
		penalty: float
		L2 Penalty strength
		"""
		self.IRVLayer = IRVLayer
		self.penalty = penalty
		super(IRVRegularize, self).__init__(**kwargs)

		def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
		assert self.IRVLayer.out_tensor is not None, "IRVLayer must be built first"
		out_tensor = tf.nn.l2_loss(self.IRVLayer.W) + \
		tf.nn.l2_loss(self.IRVLayer.V) + tf.nn.l2_loss(self.IRVLayer.b) + \
		tf.nn.l2_loss(self.IRVLayer.b2)
		out_tensor = out_tensor * self.penalty
		if set_tensors:
		self.out_tensor = out_tensor
		return out_tensor


		class Slice(Layer):
		""" Choose a slice of input on the last axis given order,
		Suppose input x has two dimensions,
		output f(x) = x[:, slice_num:slice_num+1]
		"""

		def __init__(self, slice_num, axis=1, **kwargs):
		"""
		Parameters
		----------
		slice_num: int
		index of slice number
		axis: int
		axis id
		"""
		self.slice_num = slice_num
		self.axis = axis
		super(Slice, self).__init__(**kwargs)

		def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
		if in_layers is None:
		in_layers = self.in_layers
		in_layers = convert_to_layers(in_layers)

		slice_num = self.slice_num
		axis = self.axis
		inputs = in_layers[0].out_tensor
		out_tensor = tf.slice(inputs, [0] * axis + [slice_num], [-1] * axis + [1])

		if set_tensors:
		self.out_tensor = out_tensor
		return out_tensor


		class TensorflowMultiTaskIRVClassifier(TensorGraph):

		def __init__(self,
		n_tasks,
		K=10,
		logdir=None,
		n_classes=2,
		penalty=0.0,
		penalty_type="l2",
		learning_rate=0.001,
		momentum=.8,
		optimizer="adam",
		batch_size=50,
		verbose=True,
		seed=None,
		mode="classification",
		**kwargs):
		"""Initialize TensorflowMultiTaskIRVClassifier

		@@ -33,101 +158,77 @@ class TensorflowMultiTaskIRVClassifier(TensorflowLogisticRegression):
		Number of tasks
		K: int
		Number of nearest neighbours used in classification
		logdir: str
		Location to save data
		n_classes: int
		number of different labels
		penalty: float
		Amount of penalty (l2 or l1 applied)
		penalty_type: str
		Either "l2" or "l1"
		learning_rate: float
		Learning rate for model.
		momentum: float
		Momentum. Only applied if optimizer=="momentum"
		optimizer: str
		Type of optimizer applied.
		batch_size: int
		Size of minibatches for training.
		verbose: True
		Perform logging.
		seed: int
		If not none, is used as random seed for tensorflow.

		"""
		warnings.warn("The TensorflowMultiTaskIRVClassifier is "
		"deprecated. Will be removed in DeepChem 1.4.",
		DeprecationWarning)
		self.n_tasks = n_tasks
		self.K = K
		self.n_features = 2 * self.K * self.n_tasks
		print("n_features after fit_transform: %d" % int(self.n_features))
		TensorflowGraphModel.__init__(
		self,
		n_tasks,
		self.n_features,
		logdir=logdir,
		layer_sizes=None,
		weight_init_stddevs=None,
		bias_init_consts=None,
		penalty=penalty,
		penalty_type=penalty_type,
		dropouts=None,
		n_classes=n_classes,
		learning_rate=learning_rate,
		momentum=momentum,
		optimizer=optimizer,
		batch_size=batch_size,
		pad_batches=False,
		verbose=verbose,
		seed=seed,
		**kwargs)

		def build(self, graph, name_scopes, training):
		self.penalty = penalty
		super(TensorflowMultiTaskIRVClassifier, self).__init__(**kwargs)
		self.build_graph()

		def build_graph(self):
		"""Constructs the graph architecture of IRV as described in:

		https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/
		"""
		placeholder_scope = TensorflowGraph.get_placeholder_scope(
		graph, name_scopes)
		K = self.K
		with graph.as_default():
		output = []
		with placeholder_scope:
		mol_features = tf.placeholder(
		tf.float32, shape=[None, self.n_features], name='mol_features')
		with tf.name_scope('variable'):
		V = tf.Variable(tf.constant([0.01, 1.]), name="vote", dtype=tf.float32)
		W = tf.Variable(tf.constant([1., 1.]), name="w", dtype=tf.float32)
		b = tf.Variable(tf.constant([0.01]), name="b", dtype=tf.float32)
		b2 = tf.Variable(tf.constant([0.01]), name="b2", dtype=tf.float32)

		label_placeholders = self.add_label_placeholders(graph, name_scopes)
		weight_placeholders = self.add_example_weight_placeholders(
		graph, name_scopes)
		if training:
		graph.queue = tf.FIFOQueue(
		capacity=5,
		dtypes=[tf.float32] *
		(len(label_placeholders) + len(weight_placeholders) + 1))
		graph.enqueue = graph.queue.enqueue(
		[mol_features] + label_placeholders + weight_placeholders)
		queue_outputs = graph.queue.dequeue()
		labels = queue_outputs[1:len(label_placeholders) + 1]
		weights = queue_outputs[len(label_placeholders) + 1:]
		features = queue_outputs[0]
		else:
		labels = label_placeholders
		weights = weight_placeholders
		features = mol_features
		self.mol_features = Feature(shape=(None, self.n_features))
		predictions = IRVLayer(self.n_tasks, self.K, in_layers=[self.mol_features])
		costs = []
		self.labels_fd = []
		for task in range(self.n_tasks):
		task_output = Slice(task, 1, in_layers=[predictions])
		sigmoid = Sigmoid(in_layers=[task_output])
		self.add_output(sigmoid)

		for count in range(self.n_tasks):
		similarity = features[:, 2 * K * count:(2 * K * count + K)]
		ys = tf.to_int32(features[:, (2 * K * count + K):2 * K * (count + 1)])
		R = b + W[0] * similarity + W[1] * tf.constant(
		np.arange(K) + 1, dtype=tf.float32)
		R = tf.sigmoid(R)
		z = tf.reduce_sum(R * tf.gather(V, ys), axis=1) + b2
		output.append(tf.reshape(z, shape=[-1, 1]))
		return (output, labels, weights)
		'''
		label = Label(shape=(None, 1))
		self.labels_fd.append(label)
		cost = SigmoidCrossEntropy(in_layers=[label, task_output])
		costs.append(cost)
		all_cost = Concat(in_layers=costs, axis=1)
		self.weights = Weights(shape=(None, self.n_tasks))
		loss = WeightedError(in_layers=[all_cost, self.weights]) + \
		IRVRegularize(predictions, self.penalty, in_layers=[predictions])
		self.set_loss(loss)

		def default_generator(self,
		dataset,
		epochs=1,
		predict=False,
		deterministic=True,
		pad_batches=True):
		"""TensorGraph style implementation """
		for epoch in range(epochs):
		if not predict:
		print('Starting epoch %i' % epoch)
		for (X_b, y_b, w_b, ids_b) in dataset.iterbatches(
		batch_size=self.batch_size,
		deterministic=deterministic,
		pad_batches=pad_batches):

		feed_dict = dict()
		if y_b is not None:
		for index, label in enumerate(self.labels_fd):
		feed_dict[label] = y_b[:, index:index + 1]
		if w_b is not None:
		feed_dict[self.weights] = w_b
		feed_dict[self.mol_features] = X_b

		yield feed_dict

		def predict(self, dataset, transformers=[], outputs=None):
		out = super(TensorflowMultiTaskIRVClassifier, self).predict(
		dataset, transformers=transformers, outputs=outputs)
		out = np.concatenate(out, axis=1)
		out = np.round(out).astype(int)
		return out

		def predict_proba(self, dataset, transformers=[], outputs=None):
		out = super(TensorflowMultiTaskIRVClassifier, self).predict_proba(
		dataset, transformers=transformers, outputs=outputs)
		out = np.concatenate(out, axis=1)
		out = np.stack([1 - out, out], axis=2)
		return out

deepchem/models/tensorflow_models/lr.py

deleted100644 → 0

+0 −248

Original line number	Diff line number	Diff line
		# -- coding: utf-8 --
		"""
		Created on Tue Nov 08 14:10:02 2016

		@author: Zhenqin Wu
		"""
		import warnings
		import tensorflow as tf
		import numpy as np
		import os
		import time

		from deepchem.metrics import from_one_hot
		from deepchem.nn import model_ops
		from deepchem.utils.save import log
		from deepchem.data import pad_features
		from deepchem.metrics import to_one_hot


		def weight_decay(penalty_type, penalty):
		# due to the different shape of weight(ndims=2) and bias(ndims=1),
		# will using this version for logreg
		variables = []
		# exclude bias variables
		for v in tf.trainable_variables():
		if v.get_shape().as_list()[0] > 1:
		variables.append(v)

		with tf.name_scope('weight_decay'):
		if penalty_type == 'l1':
		cost = tf.add_n([tf.reduce_sum(tf.abs(v)) for v in variables])
		elif penalty_type == 'l2':
		cost = tf.add_n([tf.nn.l2_loss(v) for v in variables])
		else:
		raise NotImplementedError('Unsupported penalty_type %s' % penalty_type)
		cost *= penalty
		tf.summary.scalar('Weight Decay Cost', cost)
		return cost


		'''
		class TensorflowLogisticRegression(TensorflowGraphModel):
		""" A simple tensorflow based logistic regression model. """

		def build(self, graph, name_scopes, training):
		"""Constructs the graph architecture of model: n_tasks * sigmoid nodes.

		This method creates the following Placeholders:
		mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape
		batch_size x n_features.
		"""
		warnings.warn("TensorflowLogisticRegression is deprecated. "
		"Will be removed in DeepChem 1.4.", DeprecationWarning)
		placeholder_scope = TensorflowGraph.get_placeholder_scope(
		graph, name_scopes)
		n_features = self.n_features
		with graph.as_default():
		with placeholder_scope:
		mol_features = tf.placeholder(
		tf.float32, shape=[None, n_features], name='mol_features')

		weight_init_stddevs = self.weight_init_stddevs
		bias_init_consts = self.bias_init_consts
		lg_list = []

		label_placeholders = self.add_label_placeholders(graph, name_scopes)
		weight_placeholders = self.add_example_weight_placeholders(
		graph, name_scopes)
		if training:
		graph.queue = tf.FIFOQueue(
		capacity=5,
		dtypes=[tf.float32] *
		(len(label_placeholders) + len(weight_placeholders) + 1))
		graph.enqueue = graph.queue.enqueue(
		[mol_features] + label_placeholders + weight_placeholders)
		queue_outputs = graph.queue.dequeue()
		labels = queue_outputs[1:len(label_placeholders) + 1]
		weights = queue_outputs[len(label_placeholders) + 1:]
		prev_layer = queue_outputs[0]
		else:
		labels = label_placeholders
		weights = weight_placeholders
		prev_layer = mol_features

		for task in range(self.n_tasks):
		#setting up n_tasks nodes(output nodes)
		lg = model_ops.fully_connected_layer(
		tensor=prev_layer,
		size=1,
		weight_init=tf.truncated_normal(
		shape=[self.n_features, 1], stddev=weight_init_stddevs[0]),
		bias_init=tf.constant(value=bias_init_consts[0], shape=[1]))
		lg_list.append(lg)
		return (lg_list, labels, weights)

		def add_label_placeholders(self, graph, name_scopes):
		#label placeholders with size batch_size * 1
		labels = []
		placeholder_scope = TensorflowGraph.get_placeholder_scope(
		graph, name_scopes)
		with placeholder_scope:
		for task in range(self.n_tasks):
		labels.append(
		tf.identity(
		tf.placeholder(
		tf.float32, shape=[None, 1], name='labels_%d' % task)))
		return labels

		def add_training_cost(self, graph, name_scopes, output, labels, weights):
		with graph.as_default():
		epsilon = 1e-3 # small float to avoid dividing by zero
		weighted_costs = [] # weighted costs for each example
		gradient_costs = [] # costs used for gradient calculation

		with TensorflowGraph.shared_name_scope('costs', graph, name_scopes):
		for task in range(self.n_tasks):
		task_str = str(task).zfill(len(str(self.n_tasks)))
		with TensorflowGraph.shared_name_scope('cost_{}'.format(task_str),
		graph, name_scopes):
		with tf.name_scope('weighted'):
		weighted_cost = self.cost(output[task], labels[task],
		weights[task])
		weighted_costs.append(weighted_cost)

		with tf.name_scope('gradient'):
		# Note that we divide by the batch size and not the number of
		# non-zero weight examples in the batch. Also, instead of using
		# tf.reduce_mean (which can put ops on the CPU) we explicitly
		# calculate with div/sum so it stays on the GPU.
		gradient_cost = tf.div(
		tf.reduce_sum(weighted_cost), self.batch_size)
		gradient_costs.append(gradient_cost)

		# aggregated costs
		with TensorflowGraph.shared_name_scope('aggregated', graph,
		name_scopes):
		with tf.name_scope('gradient'):
		loss = tf.add_n(gradient_costs)

		# weight decay
		if self.penalty != 0.0:
		# using self-defined regularization
		penalty = weight_decay(self.penalty_type, self.penalty)
		loss += penalty

		return loss

		def cost(self, logits, labels, weights):
		return tf.multiply(
		tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=labels),
		weights)

		def add_output_ops(self, graph, output):
		# adding output nodes of sigmoid function
		with graph.as_default():
		sigmoid = []
		with tf.name_scope('inference'):
		for i, logits in enumerate(output):
		sigmoid.append(tf.nn.sigmoid(logits, name='sigmoid_%d' % i))
		output = sigmoid
		return output

		def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):

		orig_dict = {}
		orig_dict["mol_features"] = X_b
		for task in range(self.n_tasks):
		if y_b is not None:
		y_2column = to_one_hot(y_b[:, task])
		# fix the size to be [?,1]
		orig_dict["labels_%d" % task] = y_2column[:, 1:2]
		else:
		# Dummy placeholders
		orig_dict["labels_%d" % task] = np.zeros((self.batch_size, 1))
		if w_b is not None:
		orig_dict["weights_%d" % task] = w_b[:, task]
		else:
		# Dummy placeholders
		orig_dict["weights_%d" % task] = np.ones((self.batch_size,))
		return TensorflowGraph.get_feed_dict(orig_dict)

		def predict_proba_on_batch(self, X):
		if self.pad_batches:
		X = pad_features(self.batch_size, X)
		if not self._restored_model:
		self.restore()
		with self.eval_graph.graph.as_default():
		# run eval data through the model
		n_tasks = self.n_tasks
		with self._get_shared_session(train=False).as_default():
		feed_dict = self.construct_feed_dict(X)
		data = self._get_shared_session(train=False).run(
		self.eval_graph.output, feed_dict=feed_dict)
		batch_outputs = np.asarray(data[:n_tasks], dtype=float)
		# transfer 2D prediction tensor to 2D x n_classes(=2)
		complimentary = np.ones(np.shape(batch_outputs))
		complimentary = complimentary - batch_outputs
		batch_outputs = np.concatenate(
		[complimentary, batch_outputs], axis=batch_outputs.ndim - 1)
		# reshape to batch_size x n_tasks x ...
		if batch_outputs.ndim == 3:
		batch_outputs = batch_outputs.transpose((1, 0, 2))
		elif batch_outputs.ndim == 2:
		batch_outputs = batch_outputs.transpose((1, 0))
		else:
		raise ValueError('Unrecognized rank combination for output: %s ' %
		(batch_outputs.shape,))

		outputs = batch_outputs

		return np.copy(outputs)

		def predict_on_batch(self, X):

		if self.pad_batches:
		X = pad_features(self.batch_size, X)

		if not self._restored_model:
		self.restore()
		with self.eval_graph.graph.as_default():

		# run eval data through the model
		n_tasks = self.n_tasks
		output = []
		start = time.time()
		with self._get_shared_session(train=False).as_default():
		feed_dict = self.construct_feed_dict(X)
		data = self._get_shared_session(train=False).run(
		self.eval_graph.output, feed_dict=feed_dict)
		batch_output = np.asarray(data[:n_tasks], dtype=float)
		# transfer 2D prediction tensor to 2D x n_classes(=2)
		complimentary = np.ones(np.shape(batch_output))
		complimentary = complimentary - batch_output
		batch_output = np.concatenate(
		[complimentary, batch_output], axis=batch_output.ndim - 1)
		# reshape to batch_size x n_tasks x ...
		if batch_output.ndim == 3:
		batch_output = batch_output.transpose((1, 0, 2))
		elif batch_output.ndim == 2:
		batch_output = batch_output.transpose((1, 0))
		else:
		raise ValueError('Unrecognized rank combination for output: %s' %
		(batch_output.shape,))
		output.append(batch_output)
		outputs = np.array(from_one_hot(np.concatenate(output), axis=-1))

		return np.copy(outputs)
		'''

deepchem/models/tensorgraph/layers.py

+50 −0

Original line number	Diff line number	Diff line
		@@ -1184,6 +1184,29 @@ class SoftMax(Layer):
		return out_tensor


		class Sigmoid(Layer):
		""" Compute the sigmoid of input: f(x) = sigmoid(x)
		Only one input is allowed, output will have the same shape as input
		"""

		def __init__(self, in_layers=None, **kwargs):
		super(Sigmoid, self).__init__(in_layers, **kwargs)
		try:
		self._shape = tuple(self.in_layers[0].shape)
		except:
		pass

		def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
		inputs = self._get_input_tensors(in_layers)
		if len(inputs) != 1:
		raise ValueError("Sigmoid must have a single input layer.")
		parent = inputs[0]
		out_tensor = tf.nn.sigmoid(parent)
		if set_tensors:
		self.out_tensor = out_tensor
		return out_tensor


		class Concat(Layer):

		def __init__(self, in_layers=None, axis=1, **kwargs):
		@@ -1531,6 +1554,33 @@ class SoftMaxCrossEntropy(Layer):
		return out_tensor


		class SigmoidCrossEntropy(Layer):
		""" Compute the sigmoid cross entropy of inputs: [labels, logits]
		`labels` hold the binary labels(with no axis of n_classes),
		`logits` hold the log probabilities for positive class(label=1),
		`labels` and `logits` should have same shape and type.
		Output will have the same shape as `logits`
		"""

		def __init__(self, in_layers=None, **kwargs):
		super(SigmoidCrossEntropy, self).__init__(in_layers, **kwargs)
		try:
		self._shape = self.in_layers[1].shape
		except:
		pass

		def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
		inputs = self._get_input_tensors(in_layers, True)
		if len(inputs) != 2:
		raise ValueError()
		labels, logits = inputs[0], inputs[1]
		out_tensor = tf.nn.sigmoid_cross_entropy_with_logits(
		logits=logits, labels=labels)
		if set_tensors:
		self.out_tensor = out_tensor
		return out_tensor


		class ReduceMean(Layer):

		def __init__(self, in_layers=None, axis=None, **kwargs):

deepchem/models/tensorgraph/tests/test_layers.py

+59 −0

File changed.

Preview size limit exceeded, changes collapsed.

Admin message