add Progressive classifier (f1244991) · Commits · 钟慕尧 / deepchem

deepchem/models/init.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -16,7 +16,7 @@ from deepchem.models.tensorgraph.fcnet import MultiTaskFitTransformRegressor
		from deepchem.models.tensorgraph.IRV import TensorflowMultiTaskIRVClassifier
		from deepchem.models.tensorgraph.robust_multitask import RobustMultitaskClassifier
		from deepchem.models.tensorgraph.robust_multitask import RobustMultitaskRegressor
		from deepchem.models.tensorgraph.progressive_multitask import ProgressiveMultitaskRegressor
		from deepchem.models.tensorgraph.progressive_multitask import ProgressiveMultitaskRegressor, ProgressiveMultitaskClassifier
		from deepchem.models.tensorgraph.models.graph_models import WeaveTensorGraph, DTNNTensorGraph, DAGTensorGraph, GraphConvTensorGraph, MPNNTensorGraph
		from deepchem.models.tensorgraph.models.symmetry_function_regression import BPSymmetryFunctionRegression, ANIRegression

deepchem/models/tensorgraph/progressive_multitask.py

+78 −14

Original line number	Diff line number	Diff line
		@@ -11,12 +11,12 @@ from deepchem.metrics import to_one_hot
		from deepchem.metrics import from_one_hot
		from deepchem.models.tensorgraph.tensor_graph import TensorGraph, TFWrapper
		from deepchem.models.tensorgraph.layers import Layer, Feature, Label, Weights, \
		WeightedError, Dense, Dropout, WeightDecay, Reshape, SoftMaxCrossEntropy, \
		L2Loss, ReduceSum, Concat, Stack, TensorWrapper, ReLU
		WeightedError, Dense, Dropout, WeightDecay, Reshape, SparseSoftMaxCrossEntropy, \
		L2Loss, ReduceSum, Concat, Stack, TensorWrapper, ReLU, Squeeze, SoftMax


		class ProgressiveMultitaskRegressor(TensorGraph):
		"""Implements a progressive multitask neural network.
		"""Implements a progressive multitask neural network for regression.

		Progressive Networks: https://arxiv.org/pdf/1606.04671v3.pdf

		@@ -37,6 +37,7 @@ class ProgressiveMultitaskRegressor(TensorGraph):
		weight_decay_penalty_type="l2",
		dropouts=0.5,
		activation_fns=tf.nn.relu,
		n_outputs=1,
		**kwargs):
		"""Creates a progressive network.

		@@ -82,6 +83,7 @@ class ProgressiveMultitaskRegressor(TensorGraph):
		self.bias_init_consts = bias_init_consts
		self.dropouts = dropouts
		self.activation_fns = activation_fns
		self.n_outputs = n_outputs

		n_layers = len(layer_sizes)
		if not isinstance(weight_init_stddevs, collections.Sequence):
		@@ -133,7 +135,7 @@ class ProgressiveMultitaskRegressor(TensorGraph):
		prev_layer = all_layers[(n_layers - 1, task)]
		layer = Dense(
		in_layers=[prev_layer],
		out_channels=1,
		out_channels=n_outputs,
		weights_initializer=TFWrapper(
		tf.truncated_normal_initializer,
		stddev=self.weight_init_stddevs[-1]),
		@@ -146,14 +148,13 @@ class ProgressiveMultitaskRegressor(TensorGraph):
		n_layers)
		task_layers.extend(trainables)
		layer = layer + lateral_contrib
		outputs.append(layer)
		self.add_output(layer)
		task_label = Label(shape=(None, 1))
		task_weight = Weights(shape=(None, 1))
		weighted_loss = ReduceSum(
		L2Loss(in_layers=[task_label, layer, task_weight]))
		output_layer = self.create_output(layer)
		outputs.append(output_layer)
		self.add_output(output_layer)

		task_loss = self.create_loss(layer)
		self.create_submodel(
		layers=task_layers, loss=weighted_loss, optimizer=None)
		layers=task_layers, loss=task_loss, optimizer=None)
		# Weight decay not activated
		"""
		if weight_decay_penalty != 0.0:
		@@ -163,6 +164,16 @@ class ProgressiveMultitaskRegressor(TensorGraph):
		in_layers=[weighted_loss])
		"""

		def create_loss(self, layer):
		task_label = Label(shape=(None, 1))
		task_weight = Weights(shape=(None, 1))
		weighted_loss = ReduceSum(
		L2Loss(in_layers=[task_label, layer, task_weight]))
		return weighted_loss

		def create_output(self, layer):
		return layer

		def add_adapter(self, all_layers, task, layer_num):
		"""Add an adapter connection for given task/layer combo"""
		i = layer_num
		@@ -175,7 +186,7 @@ class ProgressiveMultitaskRegressor(TensorGraph):
		weight_init_stddev = self.weight_init_stddevs[i]
		bias_init_const = self.bias_init_consts[i]
		elif i == len(self.layer_sizes):
		layer_sizes = self.layer_sizes + [1]
		layer_sizes = self.layer_sizes + [self.n_outputs]
		alpha_init_stddev = self.alpha_init_stddevs[-1]
		weight_init_stddev = self.weight_init_stddevs[-1]
		bias_init_const = self.bias_init_consts[-1]
		@@ -278,6 +289,59 @@ class ProgressiveMultitaskRegressor(TensorGraph):
		def predict(self, dataset, transformers=[], outputs=None):
		retval = super(ProgressiveMultitaskRegressor, self).predict(
		dataset, transformers, outputs)
		# Results is of shape (n_samples, n_tasks, 1)
		# Results is of shape (n_samples, n_tasks, n_outputs)
		out = np.stack(retval, axis=1)
		return out

		class ProgressiveMultitaskClassifier(ProgressiveMultitaskRegressor):
		"""Implements a progressive multitask neural network for classification.

		Progressive Networks: https://arxiv.org/pdf/1606.04671v3.pdf

		Progressive networks allow for multitask learning where each task
		gets a new column of weights. As a result, there is no exponential
		forgetting where previous tasks are ignored.

		"""

		def __init__(self,
		n_tasks,
		n_features,
		alpha_init_stddevs=0.02,
		layer_sizes=[1000],
		weight_init_stddevs=0.02,
		bias_init_consts=1.0,
		weight_decay_penalty=0.0,
		weight_decay_penalty_type="l2",
		dropouts=0.5,
		activation_fns=tf.nn.relu,
		**kwargs):
		n_outputs = 2
		super(ProgressiveMultitaskClassifier, self).__init__(
		n_tasks,
		n_features,
		alpha_init_stddevs=alpha_init_stddevs,
		layer_sizes=layer_sizes,
		weight_init_stddevs=weight_init_stddevs,
		bias_init_consts=bias_init_consts,
		weight_decay_penalty=weight_decay_penalty,
		weight_decay_penalty_type=weight_decay_penalty_type,
		dropouts=dropouts,
		activation_fns=activation_fns,
		n_outputs=n_outputs,
		**kwargs)

		def create_loss(self, layer):
		task_label = Label(shape=(None, 1), dtype=tf.int32)
		task_weight = Weights(shape=(None, 1))

		label = Squeeze(squeeze_dims=1, in_layers=[task_label])
		weight = Squeeze(squeeze_dims=1, in_layers=[task_weight])

		loss = SparseSoftMaxCrossEntropy(in_layers=[label, layer])
		weighted_loss = WeightedError(in_layers=[loss, weight])
		return weighted_loss

		def create_output(self, layer):
		output = SoftMax(in_layers=[layer])
		return output
		No newline at end of file

deepchem/models/tests/test_overfit.py

+38 −0

Original line number	Diff line number	Diff line
		@@ -869,6 +869,44 @@ class TestOverfit(test_util.TensorFlowTestCase):

		assert scores[regression_metric.name] > .9

		def test_tf_progressive_classification_overfit(self):
		"""Test tf progressive multitask overfits tiny data."""
		np.random.seed(123)
		n_tasks = 5
		n_samples = 10
		n_features = 3

		# Generate dummy dataset
		np.random.seed(123)
		ids = np.arange(n_samples)
		X = np.random.rand(n_samples, n_features)
		y = np.ones((n_samples, n_tasks))
		w = np.ones((n_samples, n_tasks))

		dataset = dc.data.NumpyDataset(X, y, w, ids)

		metric = dc.metrics.Metric(
		dc.metrics.accuracy_score, task_averager=np.mean)
		model = dc.models.ProgressiveMultitaskClassifier(
		n_tasks,
		n_features,
		layer_sizes=[50],
		bypass_layer_sizes=[10],
		dropouts=[0.],
		learning_rate=0.003,
		weight_init_stddevs=[.1],
		alpha_init_stddevs=[.02],
		batch_size=n_samples,
		use_queue=False)

		# Fit trained model
		model.fit(dataset, nb_epoch=20)

		# Eval model on train
		scores = model.evaluate(dataset, [metric])
		assert scores[metric.name] > .9


		def test_tf_progressive_regression_overfit(self):
		"""Test tf progressive multitask overfits tiny data."""
		np.random.seed(123)

Admin message