Loading deepchem/models/__init__.py +1 −1 Original line number Diff line number Diff line Loading @@ -25,4 +25,4 @@ from deepchem.models.tensorflow_models.progressive_multitask import ProgressiveM from deepchem.models.tensorflow_models.progressive_joint import ProgressiveJointRegressor from deepchem.models.tensorflow_models.IRV import TensorflowMultiTaskIRVClassifier from deepchem.models.tensorgraph.tensor_graph import TensorGraph, MultiTaskTensorGraph from deepchem.models.tensorgraph.models.graph_models import WeaveTensorGraph, DTNNTensorGraph No newline at end of file from deepchem.models.tensorgraph.models.graph_models import WeaveTensorGraph, DTNNTensorGraph, DAGTensorGraph No newline at end of file deepchem/models/tensorgraph/graph_layers.py +242 −61 Original line number Diff line number Diff line Loading @@ -35,12 +35,8 @@ class Separate_AP(Layer): self.out_tensor = self.in_layers[0].out_tensor[0] class WeaveLayer(Layer): """" Main layer of Weave model For each molecule, atom features and pair features are recombined to generate new atom(pair) features Detailed structure and explanations: https://arxiv.org/abs/1603.00856 """ TensorGraph style implementation The same as deepchem.nn.WeaveLayer """ Loading Loading @@ -96,7 +92,7 @@ class WeaveLayer(Layer): super(WeaveLayer, self).__init__(**kwargs) def build(self): """"Construct internal trainable weights. """ Construct internal trainable weights. """ self.W_AA = self.init([self.n_atom_input_feat, self.n_hidden_AA]) Loading Loading @@ -137,6 +133,9 @@ class WeaveLayer(Layer): [self.W_AP, self.b_AP, self.W_PP, self.b_PP, self.W_P, self.b_P]) def _create_tensor(self): """ description and explanation refer to deepchem.nn.WeaveLayer parent layers: [atom_features, pair_features], pair_split, atom_to_pair """ self.build() atom_features = self.in_layers[0].out_tensor[0] Loading Loading @@ -175,9 +174,8 @@ class WeaveLayer(Layer): self.out_tensor = [A, P] class WeaveGather(Layer): """" Gather layer of Weave model a batch of normalized atom features go through a hidden layer, then summed to form molecular features """ TensorGraph style implementation The same as deepchem.nn.WeaveGather """ def __init__(self, Loading Loading @@ -218,7 +216,9 @@ class WeaveGather(Layer): self.trainable_weights = None def _create_tensor(self): # Add trainable weights """ description and explanation refer to deepchem.nn.WeaveGather parent layers: atom_features, atom_split """ self.build() outputs = self.in_layers[0].out_tensor atom_split = self.in_layers[1].out_tensor Loading Loading @@ -252,7 +252,8 @@ class WeaveGather(Layer): class DTNNEmbedding(Layer): """Generate embeddings for all atoms in the batch """ TensorGraph style implementation The same as deepchem.nn.DTNNEmbedding """ def __init__(self, Loading @@ -273,17 +274,8 @@ class DTNNEmbedding(Layer): self.trainable_weights = [self.embedding_list] def _create_tensor(self): """Execute this layer on input tensors. Parameters ---------- x: Tensor 1D tensor of length n_atoms (atomic number) Returns ------- tf.Tensor Of shape (n_atoms, n_embedding), where n_embedding is number of atom features """description and explanation refer to deepchem.nn.DTNNEmbedding parent layers: atom_number """ self.build() atom_number = self.in_layers[0].out_tensor Loading @@ -292,10 +284,8 @@ class DTNNEmbedding(Layer): class DTNNStep(Layer): """A convolution step that merge in distance and atom info of all other atoms into current atom. model based on https://arxiv.org/abs/1609.08259 """ TensorGraph style implementation The same as deepchem.nn.DTNNStep """ def __init__(self, Loading Loading @@ -323,29 +313,14 @@ class DTNNStep(Layer): self.b_df = model_ops.zeros(shape=[ self.n_hidden, ]) #self.b_fc = model_ops.zeros(shape=[self.n_embedding,]) self.trainable_weights = [ self.W_cf, self.W_df, self.W_fc, self.b_cf, self.b_df ] def _create_tensor(self): """Execute this layer on input tensors. Parameters ---------- x: list of Tensor should be [atom_features: n_atoms*n_embedding, distance_matrix: n_pairs*n_distance, atom_membership: n_atoms distance_membership_i: n_pairs, distance_membership_j: n_pairs, ] Returns ------- tf.Tensor new embeddings for atoms, same shape as x[0] """description and explanation refer to deepchem.nn.DTNNStep parent layers: atom_features, distance, distance_membership_i, distance_membership_j """ self.build() atom_features = self.in_layers[0].out_tensor Loading @@ -353,9 +328,7 @@ class DTNNStep(Layer): distance_membership_i = self.in_layers[2].out_tensor distance_membership_j = self.in_layers[3].out_tensor distance_hidden = tf.matmul(distance, self.W_df) + self.b_df #distance_hidden = self.activation(distance_hidden) atom_features_hidden = tf.matmul(atom_features, self.W_cf) + self.b_cf #atom_features_hidden = self.activation(atom_features_hidden) outputs = tf.multiply(distance_hidden, tf.gather(atom_features_hidden, distance_membership_j)) Loading @@ -376,9 +349,9 @@ class DTNNStep(Layer): class DTNNGather(Layer): """Map the atomic features into molecular properties and sum """ TensorGraph style implementation The same as deepchem.nn.DTNNGather """ def __init__(self, n_embedding=30, n_outputs=100, Loading Loading @@ -413,18 +386,8 @@ class DTNNGather(Layer): self.trainable_weights = self.W_list + self.b_list def _create_tensor(self): """Execute this layer on input tensors. Parameters ---------- x: list of Tensor should be [embedding tensor of molecules, of shape (batch_size*max_n_atoms*n_embedding), mask tensor of molecules, of shape (batch_size*max_n_atoms)] Returns ------- list of tf.Tensor Of shape (batch_size) """description and explanation refer to deepchem.nn.DTNNGather parent layers: atom_features, atom_membership """ self.build() output = self.in_layers[0].out_tensor Loading @@ -434,3 +397,221 @@ class DTNNGather(Layer): output = self.activation(output) output = tf.segment_sum(output, atom_membership) self.out_tensor = output class DAGLayer(Layer): """ TensorGraph style implementation The same as deepchem.nn.DAGLayer """ def __init__(self, n_graph_feat=30, n_atom_feat=75, layer_sizes=[100], init='glorot_uniform', activation='relu', dropout=None, max_atoms=50, **kwargs): """ Parameters ---------- n_graph_feat: int Number of features for each node(and the whole grah). n_atom_feat: int Number of features listed per atom. layer_sizes: list of int, optional(default=[1000]) Structure of hidden layer(s) init: str, optional Weight initialization for filters. activation: str, optional Activation function applied dropout: float, optional Dropout probability, not supported here max_atoms: int, optional Maximum number of atoms in molecules. """ super(DAGLayer, self).__init__(**kwargs) self.init = initializations.get(init) # Set weight initialization self.activation = activations.get(activation) # Get activations self.layer_sizes = layer_sizes self.dropout = dropout self.max_atoms = max_atoms self.n_inputs = n_atom_feat + (self.max_atoms - 1) * n_graph_feat # number of inputs each step self.n_graph_feat = n_graph_feat self.n_outputs = n_graph_feat self.n_atom_feat = n_atom_feat def build(self): """"Construct internal trainable weights. """ self.W_list = [] self.b_list = [] prev_layer_size = self.n_inputs for layer_size in self.layer_sizes: self.W_list.append(self.init([prev_layer_size, layer_size])) self.b_list.append(model_ops.zeros(shape=[ layer_size, ])) prev_layer_size = layer_size self.W_list.append(self.init([prev_layer_size, self.n_outputs])) self.b_list.append(model_ops.zeros(shape=[ self.n_outputs, ])) self.trainable_weights = self.W_list + self.b_list def _create_tensor(self): """description and explanation refer to deepchem.nn.DAGLayer parent layers: atom_features, parents, calculation_orders, membership """ # Add trainable weights self.build() atom_features = self.in_layers[0].out_tensor # each atom corresponds to a graph, which is represented by the `max_atoms*max_atoms` int32 matrix of index # each gragh include `max_atoms` of steps(corresponding to rows) of calculating graph features parents = self.in_layers[1].out_tensor # target atoms for each step: (batch_size*max_atoms) * max_atoms calculation_orders = self.in_layers[2].out_tensor membership = self.in_layers[3].out_tensor n_atoms = atom_features.get_shape()[0] # initialize graph features for each graph graph_features = tf.Variable( tf.constant(0., shape=(n_atoms, self.max_atoms + 1, self.n_graph_feat)), trainable=False) # add dummy atom_features = tf.concat( axis=0, values=[ atom_features, tf.constant(0., shape=(1, self.n_atom_feat)) ]) for count in range(self.max_atoms): batch_atom_features = tf.gather(atom_features, calculation_orders[:, count]) index = tf.stack( [ tf.reshape( tf.stack([tf.range(n_atoms)] * (self.max_atoms - 1), axis=1), [-1]), tf.reshape(parents[:, count, 1:], [-1]) ], axis=1) batch_graph_features = tf.reshape( tf.gather_nd(graph_features, index), [-1, (self.max_atoms - 1) * self.n_graph_feat]) batch_inputs = tf.concat( axis=1, values=[batch_atom_features, batch_graph_features]) batch_outputs = self.DAGgraph_step(batch_inputs, self.W_list, self.b_list) target_index = tf.stack([tf.range(n_atoms), parents[:, count, 0]], axis=1) # index for dummies target_index2 = tf.stack( [tf.range(n_atoms), tf.constant(self.max_atoms, shape=(n_atoms,))], axis=1) # update the graph features for target atoms graph_features = tf.scatter_nd_update(graph_features, target_index, batch_outputs) # recover dummies to zeros if being updated graph_features = tf.scatter_nd_update(graph_features, target_index2, tf.zeros( (n_atoms, self.n_graph_feat))) # last step generates graph features for all target atoms # masking the outputs outputs = tf.multiply(batch_outputs, tf.expand_dims(tf.to_float(membership), axis=1)) self.out_tensor = outputs def DAGgraph_step(self, batch_inputs, W_list, b_list): outputs = batch_inputs for idw, W in enumerate(W_list): outputs = tf.nn.xw_plus_b(outputs, W, b_list[idw]) outputs = self.activation(outputs) return outputs class DAGGather(Layer): """ TensorGraph style implementation The same as deepchem.nn.DAGGather """ def __init__(self, n_graph_feat=30, n_outputs=30, layer_sizes=[1000], init='glorot_uniform', activation='relu', dropout=None, max_atoms=50, **kwargs): """ Parameters ---------- n_graph_feat: int Number of features for each atom n_outputs: int Number of features for each molecule. layer_sizes: list of int, optional(default=[1000]) Structure of hidden layer(s) init: str, optional Weight initialization for filters. activation: str, optional Activation function applied dropout: float, optional Dropout probability, not supported max_atoms: int, optional Maximum number of atoms in molecules. """ super(DAGGather, self).__init__(**kwargs) self.init = initializations.get(init) # Set weight initialization self.activation = activations.get(activation) # Get activations self.layer_sizes = layer_sizes self.dropout = dropout self.max_atoms = max_atoms self.n_graph_feat = n_graph_feat self.n_outputs = n_outputs def build(self): """"Construct internal trainable weights. """ self.W_list = [] self.b_list = [] prev_layer_size = self.n_graph_feat for layer_size in self.layer_sizes: self.W_list.append(self.init([prev_layer_size, layer_size])) self.b_list.append(model_ops.zeros(shape=[ layer_size, ])) prev_layer_size = layer_size self.W_list.append(self.init([prev_layer_size, self.n_outputs])) self.b_list.append(model_ops.zeros(shape=[ self.n_outputs, ])) self.trainable_weights = self.W_list + self.b_list def _create_tensor(self): """description and explanation refer to deepchem.nn.DAGGather parent layers: atom_features """ # Add trainable weights self.build() # Extract atom_features atom_features = self.in_layers[0].out_tensor graph_features = tf.reshape(atom_features, [-1, self.max_atoms, self.n_graph_feat]) graph_features = tf.reduce_sum(graph_features, axis=1) # sum all graph outputs outputs = self.DAGgraph_step(graph_features, self.W_list, self.b_list) self.out_tensor = outputs def DAGgraph_step(self, batch_inputs, W_list, b_list): outputs = batch_inputs for idw, W in enumerate(W_list): outputs = tf.nn.xw_plus_b(outputs, W, b_list[idw]) outputs = self.activation(outputs) return outputs deepchem/models/tensorgraph/models/graph_models.py +169 −1 Original line number Diff line number Diff line Loading @@ -7,7 +7,7 @@ from deepchem.utils.evaluate import GeneratorEvaluator from deepchem.models.tensorgraph.layers import Input, BatchNormLayer, Dense, \ SoftMax, SoftMaxCrossEntropy, L2LossLayer, Concat, WeightedError, Label, Weights, Feature from deepchem.models.tensorgraph.graph_layers import WeaveLayer, WeaveGather, \ Combine_AP, Separate_AP, DTNNEmbedding, DTNNStep, DTNNGather Combine_AP, Separate_AP, DTNNEmbedding, DTNNStep, DTNNGather, DAGLayer, DAGGather from deepchem.metrics import to_one_hot, from_one_hot from deepchem.trans import undo_transforms Loading Loading @@ -323,3 +323,171 @@ class DTNNTensorGraph(TensorGraph): result = undo_transforms(result, transformers) results.append(result) return np.concatenate(results, axis=0) class DAGTensorGraph(TensorGraph): def __init__(self, n_tasks, max_atoms=50, n_atom_feat=75, n_graph_feat=30, n_outputs=30, **kwargs): self.n_tasks = n_tasks self.max_atoms = max_atoms self.n_atom_feat = n_atom_feat self.n_graph_feat = n_graph_feat self.n_outputs = n_outputs super(DAGTensorGraph, self).__init__(**kwargs) self.build_graph() def build_graph(self): self.atom_features = Feature(shape=(self.batch_size*self.max_atoms, self.n_atom_feat)) self.parents = Feature(shape=(self.batch_size*self.max_atoms, self.max_atoms, self.max_atoms), dtype=tf.int32) self.calculation_orders = Feature(shape=(self.batch_size*self.max_atoms, self.max_atoms), dtype=tf.int32) self.membership = Feature(shape=(self.batch_size*self.max_atoms), dtype=tf.int32) dag_layer1 = DAGLayer( n_graph_feat=self.n_graph_feat, n_atom_feat=self.n_atom_feat, max_atoms=self.max_atoms, in_layers=[self.atom_features, self.parents, self.calculation_orders, self.membership]) dag_gather = DAGGather( n_graph_feat=self.n_graph_feat, n_outputs=self.n_outputs, max_atoms=self.max_atoms, in_layers=[dag_layer1]) costs = [] self.labels_fd = [] for task in range(self.n_tasks): if self.mode == "classification": classification = Dense(out_channels=2, activation_fn=None, in_layers=[dag_gather]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.labels_fd.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == "regression": regression = Dense(out_channels=1, activation_fn=None, in_layers=[dag_gather]) self.add_output(regression) label = Label(shape=(None, 1)) self.labels_fd.append(label) cost = L2LossLayer(in_layers=[label, regression]) costs.append(cost) all_cost = Concat(in_layers=costs) self.weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[all_cost, self.weights]) self.set_loss(loss) def default_generator(self, dataset, epochs=1, predict=False, pad_batches=True): for epoch in range(epochs): for (X_b, y_b, w_b, ids_b) in dataset.iterbatches( batch_size=self.batch_size, deterministic=True, pad_batches=pad_batches): feed_dict = dict() if y_b is not None and not predict: for index, label in enumerate(self.labels_fd): if self.mode == "classification": feed_dict[label] = to_one_hot(y_b[:, index]) if self.mode == "regression": feed_dict[label] = y_b[:, index:index+1] if w_b is not None and not predict: feed_dict[self.weights] = w_b atoms_per_mol = [mol.get_num_atoms() for mol in X_b] n_atom_features = X_b[0].get_atom_features().shape[1] membership = np.concatenate( [ np.array([1] * n_atoms + [0] * (self.max_atoms - n_atoms)) for n_atoms in atoms_per_mol ], axis=0) atoms_all = [] parents_all = [] calculation_orders = [] for idm, mol in enumerate(X_b): atom_features_padded = np.concatenate( [ mol.get_atom_features(), np.zeros( (self.max_atoms - atoms_per_mol[idm], n_atom_features)) ], axis=0) atoms_all.append(atom_features_padded) parents = mol.parents assert len(parents) == atoms_per_mol[idm] parents_all.extend(parents[:]) parents_all.extend([ self.max_atoms * np.ones((self.max_atoms, self.max_atoms), dtype=int) for i in range(self.max_atoms - atoms_per_mol[idm]) ]) for parent in parents: calculation_orders.append(self.index_changing(parent[:, 0], idm)) calculation_orders.extend([ self.batch_size * self.max_atoms * np.ones( (self.max_atoms,), dtype=int) for i in range(self.max_atoms - atoms_per_mol[idm]) ]) feed_dict[self.atom_features] = np.concatenate(atoms_all, axis=0) feed_dict[self.parents] = np.stack(parents_all, axis=0) feed_dict[self.calculation_orders] = np.stack(calculation_orders, axis=0) feed_dict[self.membership] = membership yield feed_dict def index_changing(self, index, n_mol): output = np.zeros_like(index) for ide, element in enumerate(index): if element < self.max_atoms: output[ide] = element + n_mol * self.max_atoms else: output[ide] = self.batch_size * self.max_atoms return output def predict(self, dataset, transformers=[], batch_size=None): generator = self.default_generator(dataset, predict=True, pad_batches=False) return self.predict_on_generator(generator) def predict_proba(self, dataset, transformers=[], batch_size=None): generator = self.default_generator(dataset, predict=True, pad_batches=False) return self.predict_proba_on_generator(generator) def predict_on_generator(self, generator, transformers=[]): retval = self.predict_proba_on_generator(generator, transformers) if self.mode == 'classification': retval = np.expand_dims(from_one_hot(retval, axis=2), axis=1) return retval def predict_proba_on_generator(self, generator, transformers=[]): if not self.built: self.build() with self._get_tf("Graph").as_default(): with tf.Session() as sess: saver = tf.train.Saver() saver.restore(sess, self.last_checkpoint) out_tensors = [x.out_tensor for x in self.outputs] results = [] for feed_dict in generator: feed_dict = { self.layers[k.name].out_tensor: v for k, v in six.iteritems(feed_dict) } result = np.array(sess.run(out_tensors, feed_dict=feed_dict)) if len(result.shape) == 3: result = np.transpose(result, axes=[1, 0, 2]) if len(transformers) > 0: result = undo_transforms(result, transformers) results.append(result) return np.concatenate(results, axis=0) No newline at end of file examples/delaney/delaney_tensorgraph_DAG.py 0 → 100644 +56 −0 Original line number Diff line number Diff line """ Script that trains weave models on delaney dataset. """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals import numpy as np np.random.seed(123) import tensorflow as tf tf.set_random_seed(123) import deepchem as dc # Load Delaney dataset delaney_tasks, delaney_datasets, transformers = dc.molnet.load_delaney( featurizer='GraphConv', split='index') train_dataset, valid_dataset, test_dataset = delaney_datasets # Fit models metric = dc.metrics.Metric(dc.metrics.pearson_r2_score, np.mean) n_atom_feat = 75 batch_size = 64 max_atoms_train = max([mol.get_num_atoms() for mol in train_dataset.X]) max_atoms_valid = max([mol.get_num_atoms() for mol in valid_dataset.X]) max_atoms_test = max([mol.get_num_atoms() for mol in test_dataset.X]) max_atoms = max([max_atoms_train, max_atoms_valid, max_atoms_test]) reshard_size = 512 transformer = dc.trans.DAGTransformer(max_atoms=max_atoms) train_dataset.reshard(reshard_size) train_dataset = transformer.transform(train_dataset) valid_dataset.reshard(reshard_size) valid_dataset = transformer.transform(valid_dataset) model = dc.models.DAGTensorGraph( len(delaney_tasks), max_atoms=max_atoms, n_atom_feat=n_atom_feat, batch_size=batch_size, learning_rate=1e-3, use_queue=False, mode='regression') # Fit trained model model.fit(train_dataset, nb_epoch=1) print("Evaluating model") train_scores = model.evaluate(train_dataset, [metric], transformers) valid_scores = model.evaluate(valid_dataset, [metric], transformers) print("Train scores") print(train_scores) print("Validation scores") print(valid_scores) Loading
deepchem/models/__init__.py +1 −1 Original line number Diff line number Diff line Loading @@ -25,4 +25,4 @@ from deepchem.models.tensorflow_models.progressive_multitask import ProgressiveM from deepchem.models.tensorflow_models.progressive_joint import ProgressiveJointRegressor from deepchem.models.tensorflow_models.IRV import TensorflowMultiTaskIRVClassifier from deepchem.models.tensorgraph.tensor_graph import TensorGraph, MultiTaskTensorGraph from deepchem.models.tensorgraph.models.graph_models import WeaveTensorGraph, DTNNTensorGraph No newline at end of file from deepchem.models.tensorgraph.models.graph_models import WeaveTensorGraph, DTNNTensorGraph, DAGTensorGraph No newline at end of file
deepchem/models/tensorgraph/graph_layers.py +242 −61 Original line number Diff line number Diff line Loading @@ -35,12 +35,8 @@ class Separate_AP(Layer): self.out_tensor = self.in_layers[0].out_tensor[0] class WeaveLayer(Layer): """" Main layer of Weave model For each molecule, atom features and pair features are recombined to generate new atom(pair) features Detailed structure and explanations: https://arxiv.org/abs/1603.00856 """ TensorGraph style implementation The same as deepchem.nn.WeaveLayer """ Loading Loading @@ -96,7 +92,7 @@ class WeaveLayer(Layer): super(WeaveLayer, self).__init__(**kwargs) def build(self): """"Construct internal trainable weights. """ Construct internal trainable weights. """ self.W_AA = self.init([self.n_atom_input_feat, self.n_hidden_AA]) Loading Loading @@ -137,6 +133,9 @@ class WeaveLayer(Layer): [self.W_AP, self.b_AP, self.W_PP, self.b_PP, self.W_P, self.b_P]) def _create_tensor(self): """ description and explanation refer to deepchem.nn.WeaveLayer parent layers: [atom_features, pair_features], pair_split, atom_to_pair """ self.build() atom_features = self.in_layers[0].out_tensor[0] Loading Loading @@ -175,9 +174,8 @@ class WeaveLayer(Layer): self.out_tensor = [A, P] class WeaveGather(Layer): """" Gather layer of Weave model a batch of normalized atom features go through a hidden layer, then summed to form molecular features """ TensorGraph style implementation The same as deepchem.nn.WeaveGather """ def __init__(self, Loading Loading @@ -218,7 +216,9 @@ class WeaveGather(Layer): self.trainable_weights = None def _create_tensor(self): # Add trainable weights """ description and explanation refer to deepchem.nn.WeaveGather parent layers: atom_features, atom_split """ self.build() outputs = self.in_layers[0].out_tensor atom_split = self.in_layers[1].out_tensor Loading Loading @@ -252,7 +252,8 @@ class WeaveGather(Layer): class DTNNEmbedding(Layer): """Generate embeddings for all atoms in the batch """ TensorGraph style implementation The same as deepchem.nn.DTNNEmbedding """ def __init__(self, Loading @@ -273,17 +274,8 @@ class DTNNEmbedding(Layer): self.trainable_weights = [self.embedding_list] def _create_tensor(self): """Execute this layer on input tensors. Parameters ---------- x: Tensor 1D tensor of length n_atoms (atomic number) Returns ------- tf.Tensor Of shape (n_atoms, n_embedding), where n_embedding is number of atom features """description and explanation refer to deepchem.nn.DTNNEmbedding parent layers: atom_number """ self.build() atom_number = self.in_layers[0].out_tensor Loading @@ -292,10 +284,8 @@ class DTNNEmbedding(Layer): class DTNNStep(Layer): """A convolution step that merge in distance and atom info of all other atoms into current atom. model based on https://arxiv.org/abs/1609.08259 """ TensorGraph style implementation The same as deepchem.nn.DTNNStep """ def __init__(self, Loading Loading @@ -323,29 +313,14 @@ class DTNNStep(Layer): self.b_df = model_ops.zeros(shape=[ self.n_hidden, ]) #self.b_fc = model_ops.zeros(shape=[self.n_embedding,]) self.trainable_weights = [ self.W_cf, self.W_df, self.W_fc, self.b_cf, self.b_df ] def _create_tensor(self): """Execute this layer on input tensors. Parameters ---------- x: list of Tensor should be [atom_features: n_atoms*n_embedding, distance_matrix: n_pairs*n_distance, atom_membership: n_atoms distance_membership_i: n_pairs, distance_membership_j: n_pairs, ] Returns ------- tf.Tensor new embeddings for atoms, same shape as x[0] """description and explanation refer to deepchem.nn.DTNNStep parent layers: atom_features, distance, distance_membership_i, distance_membership_j """ self.build() atom_features = self.in_layers[0].out_tensor Loading @@ -353,9 +328,7 @@ class DTNNStep(Layer): distance_membership_i = self.in_layers[2].out_tensor distance_membership_j = self.in_layers[3].out_tensor distance_hidden = tf.matmul(distance, self.W_df) + self.b_df #distance_hidden = self.activation(distance_hidden) atom_features_hidden = tf.matmul(atom_features, self.W_cf) + self.b_cf #atom_features_hidden = self.activation(atom_features_hidden) outputs = tf.multiply(distance_hidden, tf.gather(atom_features_hidden, distance_membership_j)) Loading @@ -376,9 +349,9 @@ class DTNNStep(Layer): class DTNNGather(Layer): """Map the atomic features into molecular properties and sum """ TensorGraph style implementation The same as deepchem.nn.DTNNGather """ def __init__(self, n_embedding=30, n_outputs=100, Loading Loading @@ -413,18 +386,8 @@ class DTNNGather(Layer): self.trainable_weights = self.W_list + self.b_list def _create_tensor(self): """Execute this layer on input tensors. Parameters ---------- x: list of Tensor should be [embedding tensor of molecules, of shape (batch_size*max_n_atoms*n_embedding), mask tensor of molecules, of shape (batch_size*max_n_atoms)] Returns ------- list of tf.Tensor Of shape (batch_size) """description and explanation refer to deepchem.nn.DTNNGather parent layers: atom_features, atom_membership """ self.build() output = self.in_layers[0].out_tensor Loading @@ -434,3 +397,221 @@ class DTNNGather(Layer): output = self.activation(output) output = tf.segment_sum(output, atom_membership) self.out_tensor = output class DAGLayer(Layer): """ TensorGraph style implementation The same as deepchem.nn.DAGLayer """ def __init__(self, n_graph_feat=30, n_atom_feat=75, layer_sizes=[100], init='glorot_uniform', activation='relu', dropout=None, max_atoms=50, **kwargs): """ Parameters ---------- n_graph_feat: int Number of features for each node(and the whole grah). n_atom_feat: int Number of features listed per atom. layer_sizes: list of int, optional(default=[1000]) Structure of hidden layer(s) init: str, optional Weight initialization for filters. activation: str, optional Activation function applied dropout: float, optional Dropout probability, not supported here max_atoms: int, optional Maximum number of atoms in molecules. """ super(DAGLayer, self).__init__(**kwargs) self.init = initializations.get(init) # Set weight initialization self.activation = activations.get(activation) # Get activations self.layer_sizes = layer_sizes self.dropout = dropout self.max_atoms = max_atoms self.n_inputs = n_atom_feat + (self.max_atoms - 1) * n_graph_feat # number of inputs each step self.n_graph_feat = n_graph_feat self.n_outputs = n_graph_feat self.n_atom_feat = n_atom_feat def build(self): """"Construct internal trainable weights. """ self.W_list = [] self.b_list = [] prev_layer_size = self.n_inputs for layer_size in self.layer_sizes: self.W_list.append(self.init([prev_layer_size, layer_size])) self.b_list.append(model_ops.zeros(shape=[ layer_size, ])) prev_layer_size = layer_size self.W_list.append(self.init([prev_layer_size, self.n_outputs])) self.b_list.append(model_ops.zeros(shape=[ self.n_outputs, ])) self.trainable_weights = self.W_list + self.b_list def _create_tensor(self): """description and explanation refer to deepchem.nn.DAGLayer parent layers: atom_features, parents, calculation_orders, membership """ # Add trainable weights self.build() atom_features = self.in_layers[0].out_tensor # each atom corresponds to a graph, which is represented by the `max_atoms*max_atoms` int32 matrix of index # each gragh include `max_atoms` of steps(corresponding to rows) of calculating graph features parents = self.in_layers[1].out_tensor # target atoms for each step: (batch_size*max_atoms) * max_atoms calculation_orders = self.in_layers[2].out_tensor membership = self.in_layers[3].out_tensor n_atoms = atom_features.get_shape()[0] # initialize graph features for each graph graph_features = tf.Variable( tf.constant(0., shape=(n_atoms, self.max_atoms + 1, self.n_graph_feat)), trainable=False) # add dummy atom_features = tf.concat( axis=0, values=[ atom_features, tf.constant(0., shape=(1, self.n_atom_feat)) ]) for count in range(self.max_atoms): batch_atom_features = tf.gather(atom_features, calculation_orders[:, count]) index = tf.stack( [ tf.reshape( tf.stack([tf.range(n_atoms)] * (self.max_atoms - 1), axis=1), [-1]), tf.reshape(parents[:, count, 1:], [-1]) ], axis=1) batch_graph_features = tf.reshape( tf.gather_nd(graph_features, index), [-1, (self.max_atoms - 1) * self.n_graph_feat]) batch_inputs = tf.concat( axis=1, values=[batch_atom_features, batch_graph_features]) batch_outputs = self.DAGgraph_step(batch_inputs, self.W_list, self.b_list) target_index = tf.stack([tf.range(n_atoms), parents[:, count, 0]], axis=1) # index for dummies target_index2 = tf.stack( [tf.range(n_atoms), tf.constant(self.max_atoms, shape=(n_atoms,))], axis=1) # update the graph features for target atoms graph_features = tf.scatter_nd_update(graph_features, target_index, batch_outputs) # recover dummies to zeros if being updated graph_features = tf.scatter_nd_update(graph_features, target_index2, tf.zeros( (n_atoms, self.n_graph_feat))) # last step generates graph features for all target atoms # masking the outputs outputs = tf.multiply(batch_outputs, tf.expand_dims(tf.to_float(membership), axis=1)) self.out_tensor = outputs def DAGgraph_step(self, batch_inputs, W_list, b_list): outputs = batch_inputs for idw, W in enumerate(W_list): outputs = tf.nn.xw_plus_b(outputs, W, b_list[idw]) outputs = self.activation(outputs) return outputs class DAGGather(Layer): """ TensorGraph style implementation The same as deepchem.nn.DAGGather """ def __init__(self, n_graph_feat=30, n_outputs=30, layer_sizes=[1000], init='glorot_uniform', activation='relu', dropout=None, max_atoms=50, **kwargs): """ Parameters ---------- n_graph_feat: int Number of features for each atom n_outputs: int Number of features for each molecule. layer_sizes: list of int, optional(default=[1000]) Structure of hidden layer(s) init: str, optional Weight initialization for filters. activation: str, optional Activation function applied dropout: float, optional Dropout probability, not supported max_atoms: int, optional Maximum number of atoms in molecules. """ super(DAGGather, self).__init__(**kwargs) self.init = initializations.get(init) # Set weight initialization self.activation = activations.get(activation) # Get activations self.layer_sizes = layer_sizes self.dropout = dropout self.max_atoms = max_atoms self.n_graph_feat = n_graph_feat self.n_outputs = n_outputs def build(self): """"Construct internal trainable weights. """ self.W_list = [] self.b_list = [] prev_layer_size = self.n_graph_feat for layer_size in self.layer_sizes: self.W_list.append(self.init([prev_layer_size, layer_size])) self.b_list.append(model_ops.zeros(shape=[ layer_size, ])) prev_layer_size = layer_size self.W_list.append(self.init([prev_layer_size, self.n_outputs])) self.b_list.append(model_ops.zeros(shape=[ self.n_outputs, ])) self.trainable_weights = self.W_list + self.b_list def _create_tensor(self): """description and explanation refer to deepchem.nn.DAGGather parent layers: atom_features """ # Add trainable weights self.build() # Extract atom_features atom_features = self.in_layers[0].out_tensor graph_features = tf.reshape(atom_features, [-1, self.max_atoms, self.n_graph_feat]) graph_features = tf.reduce_sum(graph_features, axis=1) # sum all graph outputs outputs = self.DAGgraph_step(graph_features, self.W_list, self.b_list) self.out_tensor = outputs def DAGgraph_step(self, batch_inputs, W_list, b_list): outputs = batch_inputs for idw, W in enumerate(W_list): outputs = tf.nn.xw_plus_b(outputs, W, b_list[idw]) outputs = self.activation(outputs) return outputs
deepchem/models/tensorgraph/models/graph_models.py +169 −1 Original line number Diff line number Diff line Loading @@ -7,7 +7,7 @@ from deepchem.utils.evaluate import GeneratorEvaluator from deepchem.models.tensorgraph.layers import Input, BatchNormLayer, Dense, \ SoftMax, SoftMaxCrossEntropy, L2LossLayer, Concat, WeightedError, Label, Weights, Feature from deepchem.models.tensorgraph.graph_layers import WeaveLayer, WeaveGather, \ Combine_AP, Separate_AP, DTNNEmbedding, DTNNStep, DTNNGather Combine_AP, Separate_AP, DTNNEmbedding, DTNNStep, DTNNGather, DAGLayer, DAGGather from deepchem.metrics import to_one_hot, from_one_hot from deepchem.trans import undo_transforms Loading Loading @@ -323,3 +323,171 @@ class DTNNTensorGraph(TensorGraph): result = undo_transforms(result, transformers) results.append(result) return np.concatenate(results, axis=0) class DAGTensorGraph(TensorGraph): def __init__(self, n_tasks, max_atoms=50, n_atom_feat=75, n_graph_feat=30, n_outputs=30, **kwargs): self.n_tasks = n_tasks self.max_atoms = max_atoms self.n_atom_feat = n_atom_feat self.n_graph_feat = n_graph_feat self.n_outputs = n_outputs super(DAGTensorGraph, self).__init__(**kwargs) self.build_graph() def build_graph(self): self.atom_features = Feature(shape=(self.batch_size*self.max_atoms, self.n_atom_feat)) self.parents = Feature(shape=(self.batch_size*self.max_atoms, self.max_atoms, self.max_atoms), dtype=tf.int32) self.calculation_orders = Feature(shape=(self.batch_size*self.max_atoms, self.max_atoms), dtype=tf.int32) self.membership = Feature(shape=(self.batch_size*self.max_atoms), dtype=tf.int32) dag_layer1 = DAGLayer( n_graph_feat=self.n_graph_feat, n_atom_feat=self.n_atom_feat, max_atoms=self.max_atoms, in_layers=[self.atom_features, self.parents, self.calculation_orders, self.membership]) dag_gather = DAGGather( n_graph_feat=self.n_graph_feat, n_outputs=self.n_outputs, max_atoms=self.max_atoms, in_layers=[dag_layer1]) costs = [] self.labels_fd = [] for task in range(self.n_tasks): if self.mode == "classification": classification = Dense(out_channels=2, activation_fn=None, in_layers=[dag_gather]) softmax = SoftMax(in_layers=[classification]) self.add_output(softmax) label = Label(shape=(None, 2)) self.labels_fd.append(label) cost = SoftMaxCrossEntropy(in_layers=[label, classification]) costs.append(cost) if self.mode == "regression": regression = Dense(out_channels=1, activation_fn=None, in_layers=[dag_gather]) self.add_output(regression) label = Label(shape=(None, 1)) self.labels_fd.append(label) cost = L2LossLayer(in_layers=[label, regression]) costs.append(cost) all_cost = Concat(in_layers=costs) self.weights = Weights(shape=(None, self.n_tasks)) loss = WeightedError(in_layers=[all_cost, self.weights]) self.set_loss(loss) def default_generator(self, dataset, epochs=1, predict=False, pad_batches=True): for epoch in range(epochs): for (X_b, y_b, w_b, ids_b) in dataset.iterbatches( batch_size=self.batch_size, deterministic=True, pad_batches=pad_batches): feed_dict = dict() if y_b is not None and not predict: for index, label in enumerate(self.labels_fd): if self.mode == "classification": feed_dict[label] = to_one_hot(y_b[:, index]) if self.mode == "regression": feed_dict[label] = y_b[:, index:index+1] if w_b is not None and not predict: feed_dict[self.weights] = w_b atoms_per_mol = [mol.get_num_atoms() for mol in X_b] n_atom_features = X_b[0].get_atom_features().shape[1] membership = np.concatenate( [ np.array([1] * n_atoms + [0] * (self.max_atoms - n_atoms)) for n_atoms in atoms_per_mol ], axis=0) atoms_all = [] parents_all = [] calculation_orders = [] for idm, mol in enumerate(X_b): atom_features_padded = np.concatenate( [ mol.get_atom_features(), np.zeros( (self.max_atoms - atoms_per_mol[idm], n_atom_features)) ], axis=0) atoms_all.append(atom_features_padded) parents = mol.parents assert len(parents) == atoms_per_mol[idm] parents_all.extend(parents[:]) parents_all.extend([ self.max_atoms * np.ones((self.max_atoms, self.max_atoms), dtype=int) for i in range(self.max_atoms - atoms_per_mol[idm]) ]) for parent in parents: calculation_orders.append(self.index_changing(parent[:, 0], idm)) calculation_orders.extend([ self.batch_size * self.max_atoms * np.ones( (self.max_atoms,), dtype=int) for i in range(self.max_atoms - atoms_per_mol[idm]) ]) feed_dict[self.atom_features] = np.concatenate(atoms_all, axis=0) feed_dict[self.parents] = np.stack(parents_all, axis=0) feed_dict[self.calculation_orders] = np.stack(calculation_orders, axis=0) feed_dict[self.membership] = membership yield feed_dict def index_changing(self, index, n_mol): output = np.zeros_like(index) for ide, element in enumerate(index): if element < self.max_atoms: output[ide] = element + n_mol * self.max_atoms else: output[ide] = self.batch_size * self.max_atoms return output def predict(self, dataset, transformers=[], batch_size=None): generator = self.default_generator(dataset, predict=True, pad_batches=False) return self.predict_on_generator(generator) def predict_proba(self, dataset, transformers=[], batch_size=None): generator = self.default_generator(dataset, predict=True, pad_batches=False) return self.predict_proba_on_generator(generator) def predict_on_generator(self, generator, transformers=[]): retval = self.predict_proba_on_generator(generator, transformers) if self.mode == 'classification': retval = np.expand_dims(from_one_hot(retval, axis=2), axis=1) return retval def predict_proba_on_generator(self, generator, transformers=[]): if not self.built: self.build() with self._get_tf("Graph").as_default(): with tf.Session() as sess: saver = tf.train.Saver() saver.restore(sess, self.last_checkpoint) out_tensors = [x.out_tensor for x in self.outputs] results = [] for feed_dict in generator: feed_dict = { self.layers[k.name].out_tensor: v for k, v in six.iteritems(feed_dict) } result = np.array(sess.run(out_tensors, feed_dict=feed_dict)) if len(result.shape) == 3: result = np.transpose(result, axes=[1, 0, 2]) if len(transformers) > 0: result = undo_transforms(result, transformers) results.append(result) return np.concatenate(results, axis=0) No newline at end of file
examples/delaney/delaney_tensorgraph_DAG.py 0 → 100644 +56 −0 Original line number Diff line number Diff line """ Script that trains weave models on delaney dataset. """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals import numpy as np np.random.seed(123) import tensorflow as tf tf.set_random_seed(123) import deepchem as dc # Load Delaney dataset delaney_tasks, delaney_datasets, transformers = dc.molnet.load_delaney( featurizer='GraphConv', split='index') train_dataset, valid_dataset, test_dataset = delaney_datasets # Fit models metric = dc.metrics.Metric(dc.metrics.pearson_r2_score, np.mean) n_atom_feat = 75 batch_size = 64 max_atoms_train = max([mol.get_num_atoms() for mol in train_dataset.X]) max_atoms_valid = max([mol.get_num_atoms() for mol in valid_dataset.X]) max_atoms_test = max([mol.get_num_atoms() for mol in test_dataset.X]) max_atoms = max([max_atoms_train, max_atoms_valid, max_atoms_test]) reshard_size = 512 transformer = dc.trans.DAGTransformer(max_atoms=max_atoms) train_dataset.reshard(reshard_size) train_dataset = transformer.transform(train_dataset) valid_dataset.reshard(reshard_size) valid_dataset = transformer.transform(valid_dataset) model = dc.models.DAGTensorGraph( len(delaney_tasks), max_atoms=max_atoms, n_atom_feat=n_atom_feat, batch_size=batch_size, learning_rate=1e-3, use_queue=False, mode='regression') # Fit trained model model.fit(train_dataset, nb_epoch=1) print("Evaluating model") train_scores = model.evaluate(train_dataset, [metric], transformers) valid_scores = model.evaluate(valid_dataset, [metric], transformers) print("Train scores") print(train_scores) print("Validation scores") print(valid_scores)
