Loading deepchem/models/models.py +1 −1 Original line number Diff line number Diff line Loading @@ -61,7 +61,7 @@ class Model(object): raise NotImplementedError( "Each model is responsible for its own fit_on_batch method.") def predict_on_batch(self, X): def predict_on_batch(self, X, **kwargs): """ Makes predictions on given batch of new data. Loading deepchem/models/tensorgraph/layers.py +81 −1 Original line number Diff line number Diff line Loading @@ -20,6 +20,14 @@ class Layer(object): return ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(4)) def none_tensors(self): out_tensor = self.out_tensor self.out_tensor = None return out_tensor def set_tensors(self, tensor): self.out_tensor = tensor class Conv1DLayer(Layer): Loading Loading @@ -175,7 +183,6 @@ class Input(Layer): def __call__(self, *parents): self.out_tensor = tf.placeholder(tf.float32, shape=self.t_shape) class LossLayer(Layer): def __init__(self, **kwargs): Loading Loading @@ -237,3 +244,76 @@ class ReduceMean(Layer): parent_tensor = parents[0].out_tensor self.out_tensor = tf.reduce_mean(parent_tensor) return self.out_tensor class Conv2d(Layer): def __init__(self, num_outputs, kernel_size=5, **kwargs): self.num_outputs = num_outputs self.kernel_size = kernel_size super().__init__(**kwargs) def __call__(self, *parents): parent_tensor = parents[0].out_tensor out_tensor = tf.contrib.layers.conv2d( parent_tensor, num_outputs=self.num_outputs, kernel_size=self.kernel_size, padding="SAME", activation_fn=tf.nn.relu, normalizer_fn=tf.contrib.layers.batch_norm) self.out_tensor = out_tensor class MaxPool(Layer): def __init__(self, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME", **kwargs): self.ksize = ksize self.strides = strides self.padding = padding super().__init__(**kwargs) def __call__(self, *parents): in_tensor = parents[0].out_tensor self.out_tensor = tf.nn.max_pool( in_tensor, ksize=self.ksize, strides=self.strides, padding=self.padding) return self.out_tensor class InputFifoQueue(Layer): """ This Queue Is used to allow asynchronous batching of inputs During the fitting process """ def __init__(self, shapes, names, dtypes=None, capacity=5, **kwargs): self.shapes = shapes self.names = names self.capacity = capacity self.dtypes = dtypes super().__init__(**kwargs) def __call__(self, *parents): if self.dtypes is None: self.dtypes = [tf.float32] * len(self.shapes) self.queue = tf.FIFOQueue( self.capacity, self.dtypes, shapes=self.shapes, names=self.names) feed_dict = {x.name: x.out_tensor for x in parents} self.out_tensor = self.queue.enqueue(feed_dict) self.out_tensors = self.queue.dequeue() def none_tensors(self): queue, out_tensors, out_tensor = self.queue, self.out_tensor, self.out_tensor self.queue, self.out_tensor, self.out_tensors = None, None, None return queue, out_tensors, out_tensor def set_tensors(self, tensors): self.queue, self.out_tensor, self.out_tensors = tensors def close(self): self.queue.close() deepchem/models/tensorgraph/tensor_graph.py +195 −59 Original line number Diff line number Diff line import threading import os import pickle import time Loading @@ -9,14 +11,20 @@ import numpy as np from deepchem.data import NumpyDataset from deepchem.metrics import to_one_hot, from_one_hot from deepchem.models.models import Model from deepchem.models.tensorgraph.layers import InputFifoQueue from deepchem.trans import undo_transforms class TensorGraph(Model): def __init__(self, tensorboard=False, learning_rate=0.001, **kwargs): def __init__(self, tensorboard=False, tensorboard_log_frequency=100, learning_rate=0.001, batch_size=100, mode="classification", **kwargs): """ TODO(LESWING) allow a model to change its learning rate TODO(LESWING) DOCUMENTATION AND TESTING Parameters ---------- tensorboard: bool Loading @@ -33,6 +41,7 @@ class TensorGraph(Model): self.features = list() self.labels = list() self.outputs = list() self.task_weights = list() self.loss = None self.built = False Loading @@ -45,11 +54,15 @@ class TensorGraph(Model): "train_op": None, "summary_op": None, } self.epoch = 0 self.last_checkpoint = None self.tensorboard = tensorboard self.tensorboard_log_frequency = tensorboard_log_frequency self.mode = mode self.global_step = 0 self.last_checkpoint = None self.input_queue = None self.learning_rate = learning_rate self.batch_size = batch_size super().__init__(**kwargs) self.save_file = "%s/%s" % (self.model_dir, "model") Loading @@ -62,41 +75,46 @@ class TensorGraph(Model): self.nxgraph.add_edge(parent.name, layer.name) self.parents[layer.name] = parents def _add_parent(self, layer, parent): self.nxgraph.add_edge(parent.name, layer.name) self.parents[layer.name].append(parent) def fit(self, dataset, nb_epoch=10, max_checkpoints_to_keep=5, log_every_N_batches=50, batch_size=50, checkpoint_interval=10): if not self.built: self.build() with self._get_tf("Graph").as_default(): time1 = time.time() print("Training for %d epochs" % nb_epoch) train_op = self._get_tf('train_op') saver = tf.train.Saver(max_to_keep=max_checkpoints_to_keep) with tf.Session() as sess: self._initialize_weights(sess, saver) feed_dict = {} for self.epoch in range(self.epoch, self.epoch + nb_epoch): avg_loss, n_batches = 0., 0 for ind, (X_b, y_b, w_b, ids_b) in enumerate( dataset.iterbatches(batch_size, pad_batches=True)): if ind % log_every_N_batches == 0: print("On batch %d" % ind) feed_dict = self._construct_feed_dict(X_b, y_b, w_b, ids_b) avg_loss, n_batches = 0.0, 0.0 for epoch in range(nb_epoch): coord = tf.train.Coordinator() n_samples = 0 enqueue_thread = threading.Thread( target=_enqueue_batch, args=(self, dataset, self._get_tf("Graph"), sess, coord)) enqueue_thread.start() while not coord.should_stop() or n_samples < coord.num_samples: output_tensors = [x.out_tensor for x in self.outputs] fetches = output_tensors + [train_op, self.loss.out_tensor] fetched_values = sess.run(fetches, feed_dict=feed_dict) fetched_values = sess.run(fetches) loss = fetched_values[-1] avg_loss += loss n_batches += 1 if self.epoch % checkpoint_interval == checkpoint_interval - 1: saver.save(sess, self.save_file, global_step=self.epoch) self._log_tensorboard(sess, feed_dict) self.global_step += 1 n_samples += 1 if epoch % checkpoint_interval == checkpoint_interval - 1: saver.save(sess, self.save_file, global_step=self.global_step) avg_loss = float(avg_loss) / n_batches print('Ending epoch %d: Average loss %g' % (self.epoch, avg_loss)) print('Ending epoch %d: Average loss %g' % (epoch, avg_loss)) saver.save(sess, self.save_file, global_step=self.global_step) self.last_checkpoint = saver.last_checkpoints[-1] ############################################################## TIMING time2 = time.time() Loading @@ -118,10 +136,9 @@ class TensorGraph(Model): if not self.tensorboard: return summary = sess.run(self._get_tf("summary_op"), feed_dict=feed_dict) print("Loggin") writer = self._get_tf("FileWriter") writer.reopen() writer.add_summary(summary, global_step=self.epoch) writer.add_summary(summary, global_step=self.global_step) writer.close() def fit_on_batch(self, X, y, w): Loading @@ -138,7 +155,7 @@ class TensorGraph(Model): feed_dict[self.features[0].out_tensor] = X_b return feed_dict def predict_on_batch(self, X): def predict_on_batch(self, X, sess=None): """Generates output predictions for the input samples, processing the samples in a batched way. Loading @@ -150,45 +167,109 @@ class TensorGraph(Model): # Returns A Numpy array of predictions. """ if len(self.features) != 1: raise ValueError("Only allow one input set of features") features = self.features[0] retval = self.predict_proba_on_batch(X, sess) if self.mode == 'classification': return from_one_hot(retval, axis=2) return retval def predict_proba_on_batch(self, X, sess=None): if not self.built: self.build() with self.tensor_objects['Graph'].as_default(): close_session = sess is None with self._get_tf("Graph").as_default(): saver = tf.train.Saver() with tf.Session() as sess: if sess is None: sess = tf.Session() saver.restore(sess, self.last_checkpoint) fetches = [x.out_tensor for x in self.outputs] feed_dict = {features.out_tensor: X} out_tensors = [x.out_tensor for x in self.outputs] fetches = out_tensors feed_dict = self._construct_feed_dict(X, None, None, None) fetched_values = sess.run(fetches, feed_dict=feed_dict) return np.array(fetched_values) retval = np.array(fetched_values) if self.mode == 'classification': # sample, task, class retval = np.transpose(retval, axes=[1, 0, 2]) elif self.mode == 'regression': # sample, task retval = np.transpose(retval, axes=[1, 0]) if close_session: sess.close() return retval def predict(self, dataset, transformers=[], batch_size=None): """ Uses self to make predictions on provided Dataset object. def predict_proba_on_batch(self, X): Returns: y_pred: numpy ndarray of shape (n_samples,) """ if not self.built: self.build() with self.tensor_objects['Graph'].as_default(): with self._get_tf("Graph").as_default(): saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, self.last_checkpoint) out_tensors = [x.out_tensor for x in self.outputs] fetches = out_tensors feed_dict = self._construct_feed_dict(X, None, None, None) fetched_values = sess.run(fetches, feed_dict=feed_dict) return np.array(fetched_values) y_preds = [] n_tasks = self.get_num_tasks() for (X_batch, y_b, w_b, ids_batch) in dataset.iterbatches( batch_size, deterministic=True): y_pred_batch = self.predict_on_batch(X_batch, sess=sess) y_pred_batch = undo_transforms(y_pred_batch, transformers) y_preds.append(y_pred_batch) y_pred = np.vstack(y_preds) # The iterbatches does padding with zero-weight examples on the last batch. # Remove padded examples. n_samples = len(dataset) y_pred = y_pred[:n_samples] y_pred = np.reshape(y_pred, (n_samples, n_tasks)) return y_pred def predict_proba(self, dataset, transformers=[], batch_size=None): """ TODO: Do transformers even make sense here? Returns: y_pred: numpy ndarray of shape (n_samples, n_classes*n_tasks) """ if not self.built: self.build() with self._get_tf("Graph").as_default(): saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, self.last_checkpoint) y_preds = [] n_tasks = self.get_num_tasks() for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches( batch_size, deterministic=True): n_samples = len(X_batch) y_pred_batch = self.predict_proba_on_batch(X_batch, sess=sess) y_pred_batch = y_pred_batch[:n_samples] y_pred_batch = undo_transforms(y_pred_batch, transformers) y_preds.append(y_pred_batch) y_pred = np.vstack(y_preds) # The iterbatches does padding with zero-weight examples on the last batch. # Remove padded examples. n_samples = len(dataset) y_pred = y_pred[:n_samples] return y_pred def topsort(self): return nx.topological_sort(self.nxgraph) def build(self): with self._get_tf("Graph").as_default(): self._install_queue() order = self.topsort() print(order) for node in order: node_layer = self.layers[node] parents = self.parents[node] with tf.name_scope(node): node_layer.__call__(*parents) self.built = True self.input_queue.out_tensors = None for layer in self.layers.values(): if layer.tensorboard: Loading @@ -202,31 +283,55 @@ class TensorGraph(Model): writer.add_graph(self._get_tf("Graph")) writer.close() def _install_queue(self): if self.input_queue is not None: return names = [] shapes = [] pre_q_inputs = [] for layer in self.features + self.labels + self.task_weights: pre_q_input = layer.create_pre_q(self.batch_size) shapes.append(pre_q_input.shape) names.append(pre_q_input.name) self.add_layer(pre_q_input) pre_q_inputs.append(pre_q_input) q = InputFifoQueue(shapes, names) self.add_layer(q, pre_q_inputs) for layer in self.features + self.labels + self.task_weights: self._add_parent(layer, q) self.input_queue = q def set_loss(self, layer): self.loss = layer def add_label(self, layer): self.add_layer(layer) self.labels.append(layer) def add_feature(self, layer): self.add_layer(layer) self.features.append(layer) def add_output(self, layer): self.outputs.append(layer) def add_task_weight(self, layer): self.add_layer(layer) self.task_weights.append(layer) def save(self): # Remove out_tensor from the object to be pickled must_restore = False out_tensors = [] tensor_objects = self.tensor_objects self.tensor_objects = {} out_tensors = [] if self.built: must_restore = True out_tensors = [] for node in self.topsort(): node_layer = self.layers[node] out_tensors.append(node_layer.out_tensor) node_layer.out_tensor = None out_tensors.append(node_layer.none_tensors()) self.built = False # Pickle itself Loading @@ -238,7 +343,7 @@ class TensorGraph(Model): if must_restore: for index, node in enumerate(self.topsort()): node_layer = self.layers[node] node_layer.out_tensor = out_tensors[index] node_layer.set_tensors(out_tensors[index]) self.built = True self.tensor_objects = tensor_objects Loading Loading @@ -284,13 +389,18 @@ class TensorGraph(Model): """ if self.last_checkpoint is None: sess.run(tf.global_variables_initializer()) saver.save(sess, self.save_file, global_step=self.epoch) saver.save(sess, self.save_file, global_step=self.global_step) else: saver.restore(sess, self.last_checkpoint) def get_num_tasks(self): return len(self.labels) def get_pre_q_input(self, input_layer): layer_name = input_layer.name pre_q_name = "%s_pre_q" % layer_name return self.layers[pre_q_name] @staticmethod def load_from_dir(model_dir): pickle_name = os.path.join(model_dir, "model.pickle") Loading @@ -299,6 +409,37 @@ class TensorGraph(Model): return tensorgraph def _enqueue_batch(tg, dataset, graph, sess, coord): """ Function to load data into Parameters ---------- tg dataset graph sess coord Returns ------- """ with graph.as_default(): num_samples = 0 for ind, (X_b, y_b, w_b, ids_b) in enumerate( dataset.iterbatches(tg.batch_size, pad_batches=True)): feed_dict = tg._construct_feed_dict(X_b, y_b, w_b, ids_b) enq = {} for layer in tg.features + tg.labels + tg.task_weights: enq[tg.get_pre_q_input(layer).out_tensor] = feed_dict[layer.out_tensor] sess.run(tg.input_queue.out_tensor, feed_dict=enq) num_samples += 1 if tg.tensorboard and num_samples % tg.tensorboard_log_frequency == 0: tg._log_tensorboard(sess, feed_dict) coord.num_samples = num_samples coord.request_stop() class MultiTaskTensorGraph(TensorGraph): """ Class created for legacy sake Loading @@ -306,14 +447,9 @@ class MultiTaskTensorGraph(TensorGraph): classification metrics """ def __init__(self, mode='classification', **kwargs): self.task_weights = None self.mode = mode def __init__(self, **kwargs): super().__init__(**kwargs) def set_task_weights(self, layer): self.task_weights = layer def _construct_feed_dict(self, X_b, y_b, w_b, ids_b): feed_dict = dict() if y_b is not None: Loading @@ -325,14 +461,14 @@ class MultiTaskTensorGraph(TensorGraph): feed_dict[self.features[0].out_tensor] = X_b return feed_dict def predict_on_batch(self, X): def predict_on_batch(self, X, tf_initialized=False): # sample x task # Class is implied by the value of task [0,1] prediction = super(MultiTaskTensorGraph, self).predict_on_batch(X) prediction = np.transpose(from_one_hot(prediction, axis=2)) return prediction def predict_proba_on_batch(self, X): def predict_proba_on_batch(self, X, tf_initialized=False): prediction = super(MultiTaskTensorGraph, self).predict_on_batch(X) # sample x task x class prediction1 = np.transpose(prediction, axes=[1, 0, 2]) Loading deepchem/models/tensorgraph/tests/test_tensor_graph.py +9 −11 Original line number Diff line number Diff line Loading @@ -3,8 +3,8 @@ import unittest import numpy as np import deepchem as dc from models.tensorgraph.layers import Input, Dense, LossLayer, Flatten from models import TensorGraph from deepchem.models.tensorgraph.layers import Input, Dense, LossLayer, Flatten from deepchem.models.tensorgraph.tensor_graph import TensorGraph class TestTensorGraph(unittest.TestCase): Loading @@ -12,7 +12,7 @@ class TestTensorGraph(unittest.TestCase): Test that graph topologies work correctly. """ def test_graph_save_load(self): def test_graph_save(self): n_samples = 10 n_features = 11 n_tasks = 1 Loading @@ -25,27 +25,25 @@ class TestTensorGraph(unittest.TestCase): g = TensorGraph(model_dir='/tmp/tmpss5_ki5_') inLayer = Input(shape=(None, n_samples, n_features)) g.add_layer(inLayer) g.add_feature(inLayer) flatten = Flatten() g.add_layer(flatten, parents=[inLayer]) dense = Dense(out_channels=1) g.add_layer(dense, parents=[flatten]) g.add_output(dense) label_out = Input(shape=(None, 1)) g.add_layer(label_out) g.add_label(label_out) loss = LossLayer() g.add_layer(loss, parents=[dense, label_out]) g.add_feature(inLayer) g.add_label(label_out) g.set_loss(loss) g.add_output(dense) g.fit(dataset, nb_epoch=100) g.save() g1 = TensorGraph.load_from_dir('/tmp/tmpss5_ki5_') prediction = g1.predict_on_batch(X) assert (np.sum(prediction) > 9.9) print(g1) print(g1.predict_on_batch(X)) examples/tensorgraph/MNIST.py 0 → 100644 +0 −0 Empty file added. Loading
deepchem/models/models.py +1 −1 Original line number Diff line number Diff line Loading @@ -61,7 +61,7 @@ class Model(object): raise NotImplementedError( "Each model is responsible for its own fit_on_batch method.") def predict_on_batch(self, X): def predict_on_batch(self, X, **kwargs): """ Makes predictions on given batch of new data. Loading
deepchem/models/tensorgraph/layers.py +81 −1 Original line number Diff line number Diff line Loading @@ -20,6 +20,14 @@ class Layer(object): return ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(4)) def none_tensors(self): out_tensor = self.out_tensor self.out_tensor = None return out_tensor def set_tensors(self, tensor): self.out_tensor = tensor class Conv1DLayer(Layer): Loading Loading @@ -175,7 +183,6 @@ class Input(Layer): def __call__(self, *parents): self.out_tensor = tf.placeholder(tf.float32, shape=self.t_shape) class LossLayer(Layer): def __init__(self, **kwargs): Loading Loading @@ -237,3 +244,76 @@ class ReduceMean(Layer): parent_tensor = parents[0].out_tensor self.out_tensor = tf.reduce_mean(parent_tensor) return self.out_tensor class Conv2d(Layer): def __init__(self, num_outputs, kernel_size=5, **kwargs): self.num_outputs = num_outputs self.kernel_size = kernel_size super().__init__(**kwargs) def __call__(self, *parents): parent_tensor = parents[0].out_tensor out_tensor = tf.contrib.layers.conv2d( parent_tensor, num_outputs=self.num_outputs, kernel_size=self.kernel_size, padding="SAME", activation_fn=tf.nn.relu, normalizer_fn=tf.contrib.layers.batch_norm) self.out_tensor = out_tensor class MaxPool(Layer): def __init__(self, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME", **kwargs): self.ksize = ksize self.strides = strides self.padding = padding super().__init__(**kwargs) def __call__(self, *parents): in_tensor = parents[0].out_tensor self.out_tensor = tf.nn.max_pool( in_tensor, ksize=self.ksize, strides=self.strides, padding=self.padding) return self.out_tensor class InputFifoQueue(Layer): """ This Queue Is used to allow asynchronous batching of inputs During the fitting process """ def __init__(self, shapes, names, dtypes=None, capacity=5, **kwargs): self.shapes = shapes self.names = names self.capacity = capacity self.dtypes = dtypes super().__init__(**kwargs) def __call__(self, *parents): if self.dtypes is None: self.dtypes = [tf.float32] * len(self.shapes) self.queue = tf.FIFOQueue( self.capacity, self.dtypes, shapes=self.shapes, names=self.names) feed_dict = {x.name: x.out_tensor for x in parents} self.out_tensor = self.queue.enqueue(feed_dict) self.out_tensors = self.queue.dequeue() def none_tensors(self): queue, out_tensors, out_tensor = self.queue, self.out_tensor, self.out_tensor self.queue, self.out_tensor, self.out_tensors = None, None, None return queue, out_tensors, out_tensor def set_tensors(self, tensors): self.queue, self.out_tensor, self.out_tensors = tensors def close(self): self.queue.close()
deepchem/models/tensorgraph/tensor_graph.py +195 −59 Original line number Diff line number Diff line import threading import os import pickle import time Loading @@ -9,14 +11,20 @@ import numpy as np from deepchem.data import NumpyDataset from deepchem.metrics import to_one_hot, from_one_hot from deepchem.models.models import Model from deepchem.models.tensorgraph.layers import InputFifoQueue from deepchem.trans import undo_transforms class TensorGraph(Model): def __init__(self, tensorboard=False, learning_rate=0.001, **kwargs): def __init__(self, tensorboard=False, tensorboard_log_frequency=100, learning_rate=0.001, batch_size=100, mode="classification", **kwargs): """ TODO(LESWING) allow a model to change its learning rate TODO(LESWING) DOCUMENTATION AND TESTING Parameters ---------- tensorboard: bool Loading @@ -33,6 +41,7 @@ class TensorGraph(Model): self.features = list() self.labels = list() self.outputs = list() self.task_weights = list() self.loss = None self.built = False Loading @@ -45,11 +54,15 @@ class TensorGraph(Model): "train_op": None, "summary_op": None, } self.epoch = 0 self.last_checkpoint = None self.tensorboard = tensorboard self.tensorboard_log_frequency = tensorboard_log_frequency self.mode = mode self.global_step = 0 self.last_checkpoint = None self.input_queue = None self.learning_rate = learning_rate self.batch_size = batch_size super().__init__(**kwargs) self.save_file = "%s/%s" % (self.model_dir, "model") Loading @@ -62,41 +75,46 @@ class TensorGraph(Model): self.nxgraph.add_edge(parent.name, layer.name) self.parents[layer.name] = parents def _add_parent(self, layer, parent): self.nxgraph.add_edge(parent.name, layer.name) self.parents[layer.name].append(parent) def fit(self, dataset, nb_epoch=10, max_checkpoints_to_keep=5, log_every_N_batches=50, batch_size=50, checkpoint_interval=10): if not self.built: self.build() with self._get_tf("Graph").as_default(): time1 = time.time() print("Training for %d epochs" % nb_epoch) train_op = self._get_tf('train_op') saver = tf.train.Saver(max_to_keep=max_checkpoints_to_keep) with tf.Session() as sess: self._initialize_weights(sess, saver) feed_dict = {} for self.epoch in range(self.epoch, self.epoch + nb_epoch): avg_loss, n_batches = 0., 0 for ind, (X_b, y_b, w_b, ids_b) in enumerate( dataset.iterbatches(batch_size, pad_batches=True)): if ind % log_every_N_batches == 0: print("On batch %d" % ind) feed_dict = self._construct_feed_dict(X_b, y_b, w_b, ids_b) avg_loss, n_batches = 0.0, 0.0 for epoch in range(nb_epoch): coord = tf.train.Coordinator() n_samples = 0 enqueue_thread = threading.Thread( target=_enqueue_batch, args=(self, dataset, self._get_tf("Graph"), sess, coord)) enqueue_thread.start() while not coord.should_stop() or n_samples < coord.num_samples: output_tensors = [x.out_tensor for x in self.outputs] fetches = output_tensors + [train_op, self.loss.out_tensor] fetched_values = sess.run(fetches, feed_dict=feed_dict) fetched_values = sess.run(fetches) loss = fetched_values[-1] avg_loss += loss n_batches += 1 if self.epoch % checkpoint_interval == checkpoint_interval - 1: saver.save(sess, self.save_file, global_step=self.epoch) self._log_tensorboard(sess, feed_dict) self.global_step += 1 n_samples += 1 if epoch % checkpoint_interval == checkpoint_interval - 1: saver.save(sess, self.save_file, global_step=self.global_step) avg_loss = float(avg_loss) / n_batches print('Ending epoch %d: Average loss %g' % (self.epoch, avg_loss)) print('Ending epoch %d: Average loss %g' % (epoch, avg_loss)) saver.save(sess, self.save_file, global_step=self.global_step) self.last_checkpoint = saver.last_checkpoints[-1] ############################################################## TIMING time2 = time.time() Loading @@ -118,10 +136,9 @@ class TensorGraph(Model): if not self.tensorboard: return summary = sess.run(self._get_tf("summary_op"), feed_dict=feed_dict) print("Loggin") writer = self._get_tf("FileWriter") writer.reopen() writer.add_summary(summary, global_step=self.epoch) writer.add_summary(summary, global_step=self.global_step) writer.close() def fit_on_batch(self, X, y, w): Loading @@ -138,7 +155,7 @@ class TensorGraph(Model): feed_dict[self.features[0].out_tensor] = X_b return feed_dict def predict_on_batch(self, X): def predict_on_batch(self, X, sess=None): """Generates output predictions for the input samples, processing the samples in a batched way. Loading @@ -150,45 +167,109 @@ class TensorGraph(Model): # Returns A Numpy array of predictions. """ if len(self.features) != 1: raise ValueError("Only allow one input set of features") features = self.features[0] retval = self.predict_proba_on_batch(X, sess) if self.mode == 'classification': return from_one_hot(retval, axis=2) return retval def predict_proba_on_batch(self, X, sess=None): if not self.built: self.build() with self.tensor_objects['Graph'].as_default(): close_session = sess is None with self._get_tf("Graph").as_default(): saver = tf.train.Saver() with tf.Session() as sess: if sess is None: sess = tf.Session() saver.restore(sess, self.last_checkpoint) fetches = [x.out_tensor for x in self.outputs] feed_dict = {features.out_tensor: X} out_tensors = [x.out_tensor for x in self.outputs] fetches = out_tensors feed_dict = self._construct_feed_dict(X, None, None, None) fetched_values = sess.run(fetches, feed_dict=feed_dict) return np.array(fetched_values) retval = np.array(fetched_values) if self.mode == 'classification': # sample, task, class retval = np.transpose(retval, axes=[1, 0, 2]) elif self.mode == 'regression': # sample, task retval = np.transpose(retval, axes=[1, 0]) if close_session: sess.close() return retval def predict(self, dataset, transformers=[], batch_size=None): """ Uses self to make predictions on provided Dataset object. def predict_proba_on_batch(self, X): Returns: y_pred: numpy ndarray of shape (n_samples,) """ if not self.built: self.build() with self.tensor_objects['Graph'].as_default(): with self._get_tf("Graph").as_default(): saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, self.last_checkpoint) out_tensors = [x.out_tensor for x in self.outputs] fetches = out_tensors feed_dict = self._construct_feed_dict(X, None, None, None) fetched_values = sess.run(fetches, feed_dict=feed_dict) return np.array(fetched_values) y_preds = [] n_tasks = self.get_num_tasks() for (X_batch, y_b, w_b, ids_batch) in dataset.iterbatches( batch_size, deterministic=True): y_pred_batch = self.predict_on_batch(X_batch, sess=sess) y_pred_batch = undo_transforms(y_pred_batch, transformers) y_preds.append(y_pred_batch) y_pred = np.vstack(y_preds) # The iterbatches does padding with zero-weight examples on the last batch. # Remove padded examples. n_samples = len(dataset) y_pred = y_pred[:n_samples] y_pred = np.reshape(y_pred, (n_samples, n_tasks)) return y_pred def predict_proba(self, dataset, transformers=[], batch_size=None): """ TODO: Do transformers even make sense here? Returns: y_pred: numpy ndarray of shape (n_samples, n_classes*n_tasks) """ if not self.built: self.build() with self._get_tf("Graph").as_default(): saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, self.last_checkpoint) y_preds = [] n_tasks = self.get_num_tasks() for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches( batch_size, deterministic=True): n_samples = len(X_batch) y_pred_batch = self.predict_proba_on_batch(X_batch, sess=sess) y_pred_batch = y_pred_batch[:n_samples] y_pred_batch = undo_transforms(y_pred_batch, transformers) y_preds.append(y_pred_batch) y_pred = np.vstack(y_preds) # The iterbatches does padding with zero-weight examples on the last batch. # Remove padded examples. n_samples = len(dataset) y_pred = y_pred[:n_samples] return y_pred def topsort(self): return nx.topological_sort(self.nxgraph) def build(self): with self._get_tf("Graph").as_default(): self._install_queue() order = self.topsort() print(order) for node in order: node_layer = self.layers[node] parents = self.parents[node] with tf.name_scope(node): node_layer.__call__(*parents) self.built = True self.input_queue.out_tensors = None for layer in self.layers.values(): if layer.tensorboard: Loading @@ -202,31 +283,55 @@ class TensorGraph(Model): writer.add_graph(self._get_tf("Graph")) writer.close() def _install_queue(self): if self.input_queue is not None: return names = [] shapes = [] pre_q_inputs = [] for layer in self.features + self.labels + self.task_weights: pre_q_input = layer.create_pre_q(self.batch_size) shapes.append(pre_q_input.shape) names.append(pre_q_input.name) self.add_layer(pre_q_input) pre_q_inputs.append(pre_q_input) q = InputFifoQueue(shapes, names) self.add_layer(q, pre_q_inputs) for layer in self.features + self.labels + self.task_weights: self._add_parent(layer, q) self.input_queue = q def set_loss(self, layer): self.loss = layer def add_label(self, layer): self.add_layer(layer) self.labels.append(layer) def add_feature(self, layer): self.add_layer(layer) self.features.append(layer) def add_output(self, layer): self.outputs.append(layer) def add_task_weight(self, layer): self.add_layer(layer) self.task_weights.append(layer) def save(self): # Remove out_tensor from the object to be pickled must_restore = False out_tensors = [] tensor_objects = self.tensor_objects self.tensor_objects = {} out_tensors = [] if self.built: must_restore = True out_tensors = [] for node in self.topsort(): node_layer = self.layers[node] out_tensors.append(node_layer.out_tensor) node_layer.out_tensor = None out_tensors.append(node_layer.none_tensors()) self.built = False # Pickle itself Loading @@ -238,7 +343,7 @@ class TensorGraph(Model): if must_restore: for index, node in enumerate(self.topsort()): node_layer = self.layers[node] node_layer.out_tensor = out_tensors[index] node_layer.set_tensors(out_tensors[index]) self.built = True self.tensor_objects = tensor_objects Loading Loading @@ -284,13 +389,18 @@ class TensorGraph(Model): """ if self.last_checkpoint is None: sess.run(tf.global_variables_initializer()) saver.save(sess, self.save_file, global_step=self.epoch) saver.save(sess, self.save_file, global_step=self.global_step) else: saver.restore(sess, self.last_checkpoint) def get_num_tasks(self): return len(self.labels) def get_pre_q_input(self, input_layer): layer_name = input_layer.name pre_q_name = "%s_pre_q" % layer_name return self.layers[pre_q_name] @staticmethod def load_from_dir(model_dir): pickle_name = os.path.join(model_dir, "model.pickle") Loading @@ -299,6 +409,37 @@ class TensorGraph(Model): return tensorgraph def _enqueue_batch(tg, dataset, graph, sess, coord): """ Function to load data into Parameters ---------- tg dataset graph sess coord Returns ------- """ with graph.as_default(): num_samples = 0 for ind, (X_b, y_b, w_b, ids_b) in enumerate( dataset.iterbatches(tg.batch_size, pad_batches=True)): feed_dict = tg._construct_feed_dict(X_b, y_b, w_b, ids_b) enq = {} for layer in tg.features + tg.labels + tg.task_weights: enq[tg.get_pre_q_input(layer).out_tensor] = feed_dict[layer.out_tensor] sess.run(tg.input_queue.out_tensor, feed_dict=enq) num_samples += 1 if tg.tensorboard and num_samples % tg.tensorboard_log_frequency == 0: tg._log_tensorboard(sess, feed_dict) coord.num_samples = num_samples coord.request_stop() class MultiTaskTensorGraph(TensorGraph): """ Class created for legacy sake Loading @@ -306,14 +447,9 @@ class MultiTaskTensorGraph(TensorGraph): classification metrics """ def __init__(self, mode='classification', **kwargs): self.task_weights = None self.mode = mode def __init__(self, **kwargs): super().__init__(**kwargs) def set_task_weights(self, layer): self.task_weights = layer def _construct_feed_dict(self, X_b, y_b, w_b, ids_b): feed_dict = dict() if y_b is not None: Loading @@ -325,14 +461,14 @@ class MultiTaskTensorGraph(TensorGraph): feed_dict[self.features[0].out_tensor] = X_b return feed_dict def predict_on_batch(self, X): def predict_on_batch(self, X, tf_initialized=False): # sample x task # Class is implied by the value of task [0,1] prediction = super(MultiTaskTensorGraph, self).predict_on_batch(X) prediction = np.transpose(from_one_hot(prediction, axis=2)) return prediction def predict_proba_on_batch(self, X): def predict_proba_on_batch(self, X, tf_initialized=False): prediction = super(MultiTaskTensorGraph, self).predict_on_batch(X) # sample x task x class prediction1 = np.transpose(prediction, axes=[1, 0, 2]) Loading
deepchem/models/tensorgraph/tests/test_tensor_graph.py +9 −11 Original line number Diff line number Diff line Loading @@ -3,8 +3,8 @@ import unittest import numpy as np import deepchem as dc from models.tensorgraph.layers import Input, Dense, LossLayer, Flatten from models import TensorGraph from deepchem.models.tensorgraph.layers import Input, Dense, LossLayer, Flatten from deepchem.models.tensorgraph.tensor_graph import TensorGraph class TestTensorGraph(unittest.TestCase): Loading @@ -12,7 +12,7 @@ class TestTensorGraph(unittest.TestCase): Test that graph topologies work correctly. """ def test_graph_save_load(self): def test_graph_save(self): n_samples = 10 n_features = 11 n_tasks = 1 Loading @@ -25,27 +25,25 @@ class TestTensorGraph(unittest.TestCase): g = TensorGraph(model_dir='/tmp/tmpss5_ki5_') inLayer = Input(shape=(None, n_samples, n_features)) g.add_layer(inLayer) g.add_feature(inLayer) flatten = Flatten() g.add_layer(flatten, parents=[inLayer]) dense = Dense(out_channels=1) g.add_layer(dense, parents=[flatten]) g.add_output(dense) label_out = Input(shape=(None, 1)) g.add_layer(label_out) g.add_label(label_out) loss = LossLayer() g.add_layer(loss, parents=[dense, label_out]) g.add_feature(inLayer) g.add_label(label_out) g.set_loss(loss) g.add_output(dense) g.fit(dataset, nb_epoch=100) g.save() g1 = TensorGraph.load_from_dir('/tmp/tmpss5_ki5_') prediction = g1.predict_on_batch(X) assert (np.sum(prediction) > 9.9) print(g1) print(g1.predict_on_batch(X))
