AttnLSTM overfit runs, but doesn't eval (failing) (03fffc0d) · Commits · 钟慕尧 / deepchem

deepchem/models/multitask.py

+4 −2

Original line number	Diff line number	Diff line
		@@ -165,8 +165,10 @@ class SingletaskToMultitask(Model):
		return y_pred

		def save(self):
		"""Save all models"""
		# Saving is done on-the-fly
		"""Save all models

		TODO(rbharath): Saving is not yet supported for this model.
		"""
		pass

		def reload(self):

deepchem/models/tests/test_overfit.py

+31 −27

Original line number	Diff line number	Diff line
		@@ -689,10 +689,10 @@ class TestOverfitAPI(test_util.TensorFlowTestCase):
		verbosity = "high"
		classification_metric = Metric(metrics.accuracy_score, verbosity=verbosity)

		n_atoms = 50
		#n_atoms = 50
		n_feat = 71
		batch_size = 10
		graph_model = SequentialGraphModel(n_atoms, n_feat)
		graph_model = SequentialGraphModel(n_feat)
		graph_model.add(GraphConv(64, activation='relu'))
		graph_model.add(BatchNormalization(epsilon=1e-5, mode=1))
		graph_model.add(GraphPool())
		@@ -703,9 +703,9 @@ class TestOverfitAPI(test_util.TensorFlowTestCase):

		with self.test_session() as sess:
		model = MultitaskGraphClassifier(
		sess, graph_model, n_tasks, self.model_dir, learning_rate=1e-3,
		learning_rate_decay_time=1000, optimizer_type="adam", beta1=.9,
		beta2=.999, verbosity="high")
		sess, graph_model, n_tasks, self.model_dir, batch_size=batch_size,
		learning_rate=1e-3, learning_rate_decay_time=1000,
		optimizer_type="adam", beta1=.9, beta2=.999, verbosity="high")

		# Fit trained model
		model.fit(dataset, nb_epoch=30)
		@@ -726,10 +726,10 @@ class TestOverfitAPI(test_util.TensorFlowTestCase):
		def test_attn_lstm_multitask_classification_overfit(self):
		"""Test support graph-conv multitask overfits tiny data."""
		n_tasks = 1
		n_test = 5
		n_support = 9
		n_samples = 10
		n_features = 3
		n_test = 5
		n_support = 9
		n_classes = 2
		max_depth = 4

		@@ -749,47 +749,51 @@ class TestOverfitAPI(test_util.TensorFlowTestCase):
		verbosity = "high"
		classification_metric = Metric(metrics.accuracy_score, verbosity=verbosity)

		n_atoms = 50
		n_feat = 71
		batch_size = 10
		test_batch_size = 10
		support_batch_size = 10

		support_model = SequentialSupportGraphModel(n_test, n_support, n_feat)
		support_model = SequentialSupportGraphModel(n_feat)

		# Add layers
		# output will be (n_atoms, 64)
		support_model.add(GraphConv(64, activation='relu'))
		# Need to add batch-norm separately to test/support due to differing
		# shapes.
		# output will be (n_atoms, 64)
		support_model.add_test(BatchNormalization(epsilon=1e-5, mode=1))
		# output will be (n_atoms, 64)
		support_model.add_support(BatchNormalization(epsilon=1e-5, mode=1))
		support_model.add(GraphPool())
		support_model.add_test(GraphGather(test_batch_size))
		support_model.add_support(GraphGather(support_batch_size))

		# Apply an attention lstm layer
		support_model.join(AttnLSTMEmbedding(max_depth))
		support_model.join(AttnLSTMEmbedding(test_batch_size, support_batch_size,
		max_depth))

		# Gather Projection
		support_model.add(Dense(128, activation='relu'))
		support_model.add_test(BatchNormalization(epsilon=1e-5, mode=1))
		support_model.add_support(BatchNormalization(epsilon=1e-5, mode=1))
		support_model.add(GraphGather(batch_size, activation="tanh"))
		## Gather Projection
		#support_model.add(Dense(128, activation='relu'))
		#support_model.add_test(BatchNormalization(epsilon=1e-5, mode=1))
		#support_model.add_support(BatchNormalization(epsilon=1e-5, mode=1))
		#support_model.add(GraphGather(batch_size, activation="tanh"))

		with self.test_session() as sess:
		model = SupportGraphClassifier(
		sess, support_model, n_tasks, self.model_dir, learning_rate=1e-3,
		learning_rate_decay_time=1000, optimizer_type="adam", beta1=.9,
		beta2=.999, verbosity="high")
		sess, support_model, n_tasks, self.model_dir,
		test_batch_size=test_batch_size, support_batch_size=support_batch_size,
		learning_rate=1e-3, learning_rate_decay_time=1000,
		optimizer_type="adam", beta1=.9, beta2=.999, verbosity="high")

		# Fit trained model
		model.fit(dataset, nb_epoch=30)
		model.fit(dataset, nb_epoch=3, n_trials_per_epoch=5)
		model.save()

		# Eval model on train
		transformers = []
		evaluator = Evaluator(model, dataset, transformers, verbosity=verbosity)
		scores = evaluator.compute_model_performance([classification_metric])

		############################################################ DEBUG
		scores = model.evaluate(dataset, range(n_tasks),
		[classification_metric], n_trials=1)
		print("scores")
		print(scores)
		############################################################ DEBUG

		assert scores[classification_metric.name] > .9

deepchem/models/tf_keras_models/graph_models.py

+8 −20

Original line number	Diff line number	Diff line
		@@ -24,16 +24,15 @@ class SequentialGraphModel(object):
		placeholders from GraphTopology to each graph layer (from keras_layers) added
		to the network. Non graph layers don't get the extra placeholders.
		"""
		def __init__(self, n_atoms, n_feat):
		def __init__(self, n_feat):
		"""
		Parameters
		----------
		n_atoms: int
		(Max?) Number of atoms in system.
		n_feat: int
		Number of features per atom.
		"""
		self.graph_topology = GraphTopology(n_atoms, n_feat)
		#self.graph_topology = GraphTopology(n_atoms, n_feat)
		self.graph_topology = GraphTopology(n_feat)
		self.output = self.graph_topology.get_atom_features_placeholder()
		# Keep track of the layers
		self.layers = []
		@@ -72,27 +71,16 @@ class SequentialGraphModel(object):

		class SequentialSupportGraphModel(object):
		"""An analog of Keras Sequential model for test/support models."""
		def __init__(self, n_test, n_support, n_feat, max_atoms_per_mol=60):
		def __init__(self, n_feat, max_atoms_per_mol=60):
		"""
		Parameters
		----------
		n_test: int
		Number of test molecules.
		n_support: int
		Number of support support.
		n_feat: int
		Number of atomic features.
		"""
		self.n_test = n_test
		self.n_support = n_support

		# Create graph topology and x
		n_test_atoms = n_test * max_atoms_per_mol
		n_support_atoms = n_support * max_atoms_per_mol
		self.test_graph_topology = GraphTopology(
		n_test_atoms, n_feat, name='test')
		self.support_graph_topology = GraphTopology(
		n_support_atoms, n_feat, name='support')
		self.test_graph_topology = GraphTopology(n_feat, name='test')
		self.support_graph_topology = GraphTopology(n_feat, name='support')
		self.test = self.test_graph_topology.get_atom_features_placeholder()
		self.support = self.support_graph_topology.get_atom_features_placeholder()

		@@ -127,7 +115,7 @@ class SequentialSupportGraphModel(object):
		self.layers.append(layer)

		# Update new value of x
		if type(layer).__name__ in ['GraphConv', 'GraphPool']:
		if type(layer).__name__ in ['GraphConv', 'GraphPool', 'GraphGather']:
		self.test = layer([self.test] + self.test_graph_topology.topology)
		else:
		self.test = layer(self.test)
		@@ -137,7 +125,7 @@ class SequentialSupportGraphModel(object):
		self.layers.append(layer)

		# Update new value of x
		if type(layer).__name__ in ['GraphConv', 'GraphPool']:
		if type(layer).__name__ in ['GraphConv', 'GraphPool', 'GraphGather']:
		self.support = layer([self.support] + self.support_graph_topology.topology)
		else:
		self.support = layer(self.support)

deepchem/models/tf_keras_models/graph_topology.py

+2 −4

Original line number	Diff line number	Diff line
		@@ -26,7 +26,7 @@ def merge_dicts(l):

		class GraphTopology(object):
		"""Manages placeholders associated with batch of graphs and their topology"""
		def __init__(self, n_atoms, n_feat, name='topology', max_deg=6,
		def __init__(self, n_feat, name='topology', max_deg=6,
		min_deg=0):
		"""
		Note that batch size is not specified in a GraphTopology object. A batch
		@@ -35,8 +35,6 @@ class GraphTopology(object):

		Parameters
		----------
		n_atoms: int
		Number of atoms (max) in graphs.
		n_feat: int
		Number of features per atom.
		name: str, optional
		@@ -47,7 +45,7 @@ class GraphTopology(object):
		Minimum #bonds for atoms in molecules.
		"""

		self.n_atoms = n_atoms
		#self.n_atoms = n_atoms
		self.n_feat = n_feat

		self.name = name

deepchem/models/tf_keras_models/keras_layers.py

+12 −10

Original line number	Diff line number	Diff line
		@@ -465,8 +465,8 @@ class AttnLSTMEmbedding(Layer):
		Order Matters: Sequence to sequence for sets
		https://arxiv.org/abs/1511.06391
		"""
		def __init__(self, max_depth, init='glorot_uniform', activation='linear',
		dropout=None, **kwargs):
		def __init__(self, n_test, n_support, max_depth, init='glorot_uniform',
		activation='linear', dropout=None, **kwargs):
		"""
		Parameters
		----------
		@@ -484,7 +484,8 @@ class AttnLSTMEmbedding(Layer):
		self.init = initializations.get(init) # Set weight initialization
		self.activation = activations.get(activation) # Get activations
		self.max_depth = max_depth

		self.n_test = n_test
		self.n_support = n_support

		def build(self, input_shape):
		"""Initializes trainable weights."""
		@@ -492,14 +493,15 @@ class AttnLSTMEmbedding(Layer):
		# xp_input_shape = (N_support, N_feat)
		x_input_shape, xp_input_shape = input_shape #Unpack

		N_test = x_input_shape[0]
		N_support = xp_input_shape[0]
		N_feat = xp_input_shape[1]
		n_feat = xp_input_shape[1]

		self.lstm = LSTMStep(N_feat)
		self.q_init = K.zeros([N_test,N_feat])
		self.r_init = K.zeros([N_test,N_feat])
		self.states_init = self.lstm.get_initial_states([N_test,N_feat])
		self.lstm = LSTMStep(n_feat)
		############################################## DEBUG
		print("AttnLSTMEmbedding.build()")
		############################################## DEBUG
		self.q_init = K.zeros([self.n_test, n_feat])
		self.r_init = K.zeros([self.n_test, n_feat])
		self.states_init = self.lstm.get_initial_states([self.n_test, n_feat])

		self.trainable_weights = [self.q_init, self.r_init]

Admin message