Loading deepchem/models/__init__.py +1 −5 Original line number Diff line number Diff line Loading @@ -8,12 +8,7 @@ from __future__ import unicode_literals from deepchem.models.models import Model from deepchem.models.sklearn_models import SklearnModel from deepchem.models.xgboost_models import XGBoostModel from deepchem.models.tf_new_models.multitask_classifier import MultitaskGraphClassifier from deepchem.models.tf_new_models.multitask_regressor import MultitaskGraphRegressor, DTNNMultitaskGraphRegressor from deepchem.models.tf_new_models.support_classifier import SupportGraphClassifier from deepchem.models.multitask import SingletaskToMultitask from deepchem.models.sequential import Sequential from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskRegressor from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier Loading @@ -34,3 +29,4 @@ from deepchem.models.tensorgraph.models.symmetry_function_regression import BPSy from deepchem.models.tensorgraph.models.seqtoseq import SeqToSeq from deepchem.models.tensorgraph.models.gan import GAN, WGAN from deepchem.models.tensorgraph.models.text_cnn import TextCNNTensorGraph from deepchem.models.tensorgraph.sequential import Sequential deepchem/models/sequential.pydeleted 100644 → 0 +0 −341 Original line number Diff line number Diff line """ Contains Sequential model adapted from keras/keras/models.py. This class is adapted from Keras directly. Have cut out functionality and changed API to match DeepChem style. """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals __author__ = "Bharath Ramsundar" __copyright__ = "Copyright 2017, Stanford University" __license__ = "MIT" import time import os import tempfile import numpy as np import tensorflow as tf from deepchem.models.models import Model from deepchem.nn import model_ops from deepchem.nn.copy import Layer from deepchem.nn.copy import InputLayer class Sequential(Model): """Linear stack of layers. Parameters ---------- layers: list of layers to add to the model. Note ---- The first layer passed to a Sequential model should have a defined input shape. What that means is that it should have received an `input_shape` or `batch_input_shape` argument, or for some type of layers (recurrent, Dense...) an `input_dim` argument. Example ------- >>> import deepchem as dc >>> model = dc.models.Sequential() >>> # Add features >>> model.add_features(dc.nn.Input(shape=(50,))) >>> # Add labels >>> model.add_labels(dc.nn.Input(shape=(1,))) >>> model.add(dc.nn.Dense(32, 50)) >>> model.add(dc.nn.Dense(64, 32)) """ def __init__(self, name=None, logdir=None): super(Sequential, self).__init__(self, model_dir=logdir) self.layers = [] # stack of layers self.outputs = None # tensors (length 1) if not name: prefix = 'sequential_' name = prefix + str(model_ops.get_uid(prefix)) self.name = name self.graph = tf.Graph() config = tf.ConfigProto(allow_soft_placement=True) self.session = tf.Session(graph=self.graph, config=config) # Path to save checkpoint files self._save_path = os.path.join(self.model_dir, 'model.ckpt') def add(self, layer): """Adds a layer instance on top of the layer stack. Parameters ---------- layer: layer instance. """ if not isinstance(layer, Layer): raise TypeError("The added layer must be an instance of class Layer. " "Found: " + str(layer)) with self.graph.as_default(): if not self.layers: raise ValueError("Call add_features() before calling add()") # first layer in model: check that it is an input layer else: self.outputs = layer(self.outputs) self.layers.append(layer) def add_features(self, layer): """Adds an input layer.""" if self.layers: raise ValueError( "add_features() has to be called before layers are added.") if not isinstance(layer, InputLayer): raise ValueError("First layer in sequential model must be InputLayer") with self.graph.as_default(): self.features = layer()[0] self.outputs = self.features self.layers = [layer] def add_labels(self, layer): """Adds a layer for labels""" with self.graph.as_default(): self.labels = layer()[0] def add_loss(self, loss, inputs=None): """Adds a loss to model. Parameters ---------- losses: list """ # Add losses to graph with self.graph.as_default(): # Loss for each batch element batch_loss = loss(self.outputs, self.labels) # Loss should be a float self.loss = tf.reduce_sum(batch_loss) @property def uses_learning_phase(self): return self.uses_learning_phase def fit(self, dataset, nb_epoch=10, max_checkpoints_to_keep=5, log_every_N_batches=50, learning_rate=.001, batch_size=50, checkpoint_interval=10): """Trains the model for a fixed number of epochs. TODO(rbharath0: This is mostly copied from TensorflowGraphModel. Should eventually refactor both together. Parameters ---------- dataset: dc.data.Dataset nb_epoch: 10 Number of training epochs. Dataset object holding training data batch_size: integer. Number of samples per gradient update. nb_epoch: integer, the number of epochs to train the model. verbose: 0 for no logging to stdout, 1 for progress bar logging, 2 for one log line per epoch. initial_epoch: epoch at which to start training (useful for resuming a previous training run) checkpoint_interval: int Frequency at which to write checkpoints, measured in epochs """ ############################################################## TIMING time1 = time.time() ############################################################## TIMING print("Training for %d epochs" % nb_epoch) with self.graph.as_default(): opt = model_ops.optimizer("adam", learning_rate) train_op = opt.minimize(self.loss, name='train') with self.session as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(max_to_keep=max_checkpoints_to_keep) # Save an initial checkpoint. saver.save(sess, self._save_path, global_step=0) for epoch in range(nb_epoch): avg_loss, n_batches = 0., 0 # TODO(rbharath): Don't support example weighting yet. for ind, (X_b, y_b, w_b, ids_b) in enumerate(dataset.iterbatches(batch_size)): if ind % log_every_N_batches == 0: print("On batch %d" % ind) feed_dict = {self.features: X_b, self.labels: y_b} fetches = [self.outputs] + [train_op, self.loss] fetched_values = sess.run(fetches, feed_dict=feed_dict) output = fetched_values[:1] loss = fetched_values[-1] avg_loss += loss y_pred = np.squeeze(np.array(output)) y_b = y_b.flatten() n_batches += 1 if epoch % checkpoint_interval == checkpoint_interval - 1: saver.save(sess, self._save_path, global_step=epoch) avg_loss = float(avg_loss) / n_batches print('Ending epoch %d: Average loss %g' % (epoch, avg_loss)) # Always save a final checkpoint when complete. saver.save(sess, self._save_path, global_step=epoch + 1) ############################################################## TIMING time2 = time.time() print("TIMING: model fitting took %0.3f s" % (time2 - time1)) ############################################################## TIMING def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None, **kwargs): """Computes the loss on some input data, batch by batch. Parameters ---------- x: input data, as a Numpy array or list of Numpy arrays (if the model has multiple inputs). y: labels, as a Numpy array. batch_size: integer. Number of samples per gradient update. verbose: verbosity mode, 0 or 1. sample_weight: sample weights, as a Numpy array. Returns ------- Scalar test loss (if the model has no metrics) or list of scalars (if the model computes other metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ if self.model is None: raise RuntimeError('The model needs to be compiled ' 'before being used.') if 'show_accuracy' in kwargs: kwargs.pop('show_accuracy') warnings.warn('The "show_accuracy" argument is deprecated, ' 'instead you should pass the "accuracy" metric to ' 'the model at compile time:\n' '`model.compile(optimizer, loss, ' 'metrics=["accuracy"])`') if kwargs: raise TypeError('Received unknown keyword arguments: ' + str(kwargs)) return self.model.evaluate( x, y, batch_size=batch_size, verbose=verbose, sample_weight=sample_weight) def predict(self, x, batch_size=32, verbose=0): """Generates output predictions for the input samples, processing the samples in a batched way. # Arguments x: the input data, as a Numpy array. batch_size: integer. verbose: verbosity mode, 0 or 1. # Returns A Numpy array of predictions. """ if self.model is None: self.build() return self.model.predict(x, batch_size=batch_size, verbose=verbose) def predict_on_batch(self, x): """Returns predictions for a single batch of samples. """ if self.model is None: self.build() return self.model.predict_on_batch(x) def train_on_batch(self, x, y, class_weight=None, sample_weight=None, **kwargs): """Single gradient update over one batch of samples. # Arguments x: input data, as a Numpy array or list of Numpy arrays (if the model has multiple inputs). y: labels, as a Numpy array. class_weight: dictionary mapping classes to a weight value, used for scaling the loss function (during training only). sample_weight: sample weights, as a Numpy array. # Returns Scalar training loss (if the model has no metrics) or list of scalars (if the model computes other metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ if self.model is None: raise RuntimeError('The model needs to be compiled ' 'before being used.') if 'accuracy' in kwargs: kwargs.pop('accuracy') warnings.warn('The "accuracy" argument is deprecated, ' 'instead you should pass the "accuracy" metric to ' 'the model at compile time:\n' '`model.compile(optimizer, loss, ' 'metrics=["accuracy"])`') if kwargs: raise TypeError('Received unknown keyword arguments: ' + str(kwargs)) return self.model.train_on_batch( x, y, sample_weight=sample_weight, class_weight=class_weight) def test_on_batch(self, x, y, sample_weight=None, **kwargs): """Evaluates the model over a single batch of samples. # Arguments x: input data, as a Numpy array or list of Numpy arrays (if the model has multiple inputs). y: labels, as a Numpy array. sample_weight: sample weights, as a Numpy array. # Returns Scalar test loss (if the model has no metrics) or list of scalars (if the model computes other metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ if self.model is None: raise RuntimeError('The model needs to be compiled ' 'before being used.') if 'accuracy' in kwargs: kwargs.pop('accuracy') warnings.warn('The "accuracy" argument is deprecated, ' 'instead you should pass the "accuracy" metric to ' 'the model at compile time:\n' '`model.compile(optimizer, loss, ' 'metrics=["accuracy"])`') if kwargs: raise TypeError('Received unknown keyword arguments: ' + str(kwargs)) return self.model.test_on_batch(x, y, sample_weight=sample_weight) def predict_proba(self, x, batch_size=32, verbose=1): """Generates class probability predictions for the input samples batch by batch. # Arguments x: input data, as a Numpy array or list of Numpy arrays (if the model has multiple inputs). batch_size: integer. verbose: verbosity mode, 0 or 1. # Returns A Numpy array of probability predictions. """ preds = self.predict(x, batch_size, verbose) if preds.min() < 0. or preds.max() > 1.: warnings.warn('Network returning invalid probability values. ' 'The last layer might not normalize predictions ' 'into probabilities ' '(like softmax or sigmoid would).') return preds deepchem/models/tensorgraph/layers.py +1 −1 Original line number Diff line number Diff line Loading @@ -1084,7 +1084,7 @@ class SoftMax(Layer): def create_tensor(self, in_layers=None, set_tensors=True, **kwargs): inputs = self._get_input_tensors(in_layers) if len(inputs) != 1: raise ValueError("Must only Softmax single parent") raise ValueError("Softmax must have a single input layer.") parent = inputs[0] out_tensor = tf.contrib.layers.softmax(parent) if set_tensors: Loading deepchem/models/tensorgraph/models/gan.py +120 −45 Original line number Diff line number Diff line Loading @@ -50,14 +50,33 @@ class GAN(TensorGraph): create_generator_loss() create_discriminator_loss() get_noise_batch() This class allows a GAN to have multiple generators and discriminators, a model known as MIX+GAN. It is described in Arora et al., "Generalization and Equilibrium in Generative Adversarial Nets (GANs)" (https://arxiv.org/abs/1703.00573). This can lead to better models, and is especially useful for reducing mode collapse, since different generators can learn different parts of the distribution. To use this technique, simply specify the number of generators and discriminators when calling the constructor. You can then tell predict_gan_generator() which generator to use for predicting samples. """ def __init__(self, **kwargs): def __init__(self, n_generators=1, n_discriminators=1, **kwargs): """Construct a GAN. This class accepts all the keyword arguments from TensorGraph. In addition to the parameters listed below, this class accepts all the keyword arguments from TensorGraph. Parameters ---------- n_generators: int the number of generators to include n_discriminators: int the number of discriminators to include """ super(GAN, self).__init__(use_queue=False, **kwargs) self.n_generators = n_generators self.n_discriminators = n_discriminators # Create the inputs. Loading @@ -69,40 +88,49 @@ class GAN(TensorGraph): for shape in self.get_conditional_input_shapes(): self.conditional_inputs.append(layers.Feature(shape=shape)) # Create the generator. # Create the generators. self.generator = self.create_generator(self.noise_input, self.generators = [] for i in range(n_generators): generator = self.create_generator(self.noise_input, self.conditional_inputs) if not isinstance(self.generator, Sequence): if not isinstance(generator, Sequence): raise ValueError('create_generator() must return a list of Layers') if len(self.generator) != len(self.data_inputs): if len(generator) != len(self.data_inputs): raise ValueError( 'The number of generator outputs must match the number of data inputs' ) for g, d in zip(self.generator, self.data_inputs): for g, d in zip(generator, self.data_inputs): if g.shape != d.shape: raise ValueError( 'The shapes of the generator outputs must match the shapes of the data inputs' ) for g in self.generator: for g in generator: self.add_output(g) self.generators.append(generator) # Create the discriminator. # Create the discriminators. self.discrim_train = self.create_discriminator(self.data_inputs, self.discrim_train = [] self.discrim_gen = [] for i in range(n_discriminators): discrim_train = self.create_discriminator(self.data_inputs, self.conditional_inputs) self.discrim_train.append(discrim_train) # Make a copy of the discriminator that takes the generator's output as # Make a copy of the discriminator that takes each generator's output as # its input. for generator in self.generators: replacements = {} for g, d in zip(self.generator, self.data_inputs): for g, d in zip(generator, self.data_inputs): replacements[d] = g for c in self.conditional_inputs: replacements[c] = c self.discrim_gen = self.discrim_train.copy(replacements, shared=True) discrim_gen = discrim_train.copy(replacements, shared=True) self.discrim_gen.append(discrim_gen) # Make a list of all layers in the generator and discriminator. # Make a list of all layers in the generators and discriminators. def add_layers_to_set(layer, layers): if layer not in layers: Loading @@ -111,21 +139,63 @@ class GAN(TensorGraph): add_layers_to_set(i, layers) gen_layers = set() for layer in self.generator: for generator in self.generators: for layer in generator: add_layers_to_set(layer, gen_layers) discrim_layers = set() add_layers_to_set(self.discrim_train, discrim_layers) for discriminator in self.discrim_train: add_layers_to_set(discriminator, discrim_layers) discrim_layers -= gen_layers # Create submodels for training the generator and discriminator. # Compute the loss functions. gen_losses = [self.create_generator_loss(d) for d in self.discrim_gen] discrim_losses = [] for i in range(n_discriminators): for j in range(n_generators): discrim_losses.append( self.create_discriminator_loss( self.discrim_train[i], self.discrim_gen[i * n_generators + j])) if n_generators == 1 and n_discriminators == 1: total_gen_loss = gen_losses[0] total_discrim_loss = discrim_losses[0] else: # Create learnable weights for the generators and discriminators. gen_alpha = layers.Variable(np.ones((1, n_generators))) gen_weights = layers.SoftMax(gen_alpha) discrim_alpha = layers.Variable(np.ones((1, n_discriminators))) discrim_weights = layers.SoftMax(discrim_alpha) # Compute the weighted errors weight_products = layers.Reshape( (n_generators * n_discriminators,), in_layers=layers.Reshape( (n_discriminators, 1), in_layers=discrim_weights) * layers.Reshape( (1, n_generators), in_layers=gen_weights)) total_gen_loss = layers.WeightedError((layers.Stack(gen_losses, axis=0), weight_products)) total_discrim_loss = layers.WeightedError((layers.Stack( discrim_losses, axis=0), weight_products)) gen_layers.add(gen_alpha) discrim_layers.add(gen_alpha) discrim_layers.add(discrim_alpha) # Add an entropy term to the loss. entropy = -( layers.ReduceSum(layers.Log(gen_weights)) / n_generators + layers.ReduceSum(layers.Log(discrim_weights)) / n_discriminators) total_discrim_loss += entropy # Create submodels for training the generators and discriminators. gen_loss = self.create_generator_loss(self.discrim_gen) discrim_loss = self.create_discriminator_loss(self.discrim_train, self.discrim_gen) self.generator_submodel = self.create_submodel( layers=gen_layers, loss=gen_loss) layers=gen_layers, loss=total_gen_loss) self.discriminator_submodel = self.create_submodel( layers=discrim_layers, loss=discrim_loss) layers=discrim_layers, loss=total_discrim_loss) def get_noise_input_shape(self): """Get the shape of the generator's noise input layer. Loading Loading @@ -370,7 +440,8 @@ class GAN(TensorGraph): def predict_gan_generator(self, batch_size=1, noise_input=None, conditional_inputs=[]): conditional_inputs=[], generator_index=0): """Use the GAN to generate a batch of samples. Parameters Loading @@ -386,6 +457,9 @@ class GAN(TensorGraph): conditional_inputs: list of arrays the values to use for all conditional inputs. This must be specified if the GAN has any conditional inputs. generator_index: int the index of the generator (between 0 and n_generators-1) to use for generating the samples. Returns ------- Loading @@ -402,7 +476,8 @@ class GAN(TensorGraph): batch[self.noise_input] = noise_input for layer, value in zip(self.conditional_inputs, conditional_inputs): batch[layer] = value return self.predict_on_generator([batch]) return self.predict_on_generator( [batch], outputs=self.generators[generator_index]) def _set_empty_inputs(self, feed_dict, layers): """Set entries in a feed dict corresponding to a batch size of 0.""" Loading deepchem/models/tensorgraph/models/graph_models.py +127 −115 Original line number Diff line number Diff line Loading @@ -338,6 +338,18 @@ class DTNNTensorGraph(TensorGraph): yield feed_dict def predict(self, dataset, transformers=[], outputs=None): if outputs is None: outputs = self.outputs if transformers != [] and not isinstance(outputs, collections.Sequence): raise ValueError( "DTNN does not support single tensor output with transformers") retval = super(DTNNTensorGraph, self).predict(dataset, outputs=outputs) if not isinstance(outputs, collections.Sequence): return retval retval = np.concatenate(retval, axis=-1) return undo_transforms(retval, transformers) class DAGTensorGraph(TensorGraph): Loading Loading
deepchem/models/__init__.py +1 −5 Original line number Diff line number Diff line Loading @@ -8,12 +8,7 @@ from __future__ import unicode_literals from deepchem.models.models import Model from deepchem.models.sklearn_models import SklearnModel from deepchem.models.xgboost_models import XGBoostModel from deepchem.models.tf_new_models.multitask_classifier import MultitaskGraphClassifier from deepchem.models.tf_new_models.multitask_regressor import MultitaskGraphRegressor, DTNNMultitaskGraphRegressor from deepchem.models.tf_new_models.support_classifier import SupportGraphClassifier from deepchem.models.multitask import SingletaskToMultitask from deepchem.models.sequential import Sequential from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskRegressor from deepchem.models.tensorflow_models.fcnet import TensorflowMultiTaskClassifier Loading @@ -34,3 +29,4 @@ from deepchem.models.tensorgraph.models.symmetry_function_regression import BPSy from deepchem.models.tensorgraph.models.seqtoseq import SeqToSeq from deepchem.models.tensorgraph.models.gan import GAN, WGAN from deepchem.models.tensorgraph.models.text_cnn import TextCNNTensorGraph from deepchem.models.tensorgraph.sequential import Sequential
deepchem/models/sequential.pydeleted 100644 → 0 +0 −341 Original line number Diff line number Diff line """ Contains Sequential model adapted from keras/keras/models.py. This class is adapted from Keras directly. Have cut out functionality and changed API to match DeepChem style. """ from __future__ import print_function from __future__ import division from __future__ import unicode_literals __author__ = "Bharath Ramsundar" __copyright__ = "Copyright 2017, Stanford University" __license__ = "MIT" import time import os import tempfile import numpy as np import tensorflow as tf from deepchem.models.models import Model from deepchem.nn import model_ops from deepchem.nn.copy import Layer from deepchem.nn.copy import InputLayer class Sequential(Model): """Linear stack of layers. Parameters ---------- layers: list of layers to add to the model. Note ---- The first layer passed to a Sequential model should have a defined input shape. What that means is that it should have received an `input_shape` or `batch_input_shape` argument, or for some type of layers (recurrent, Dense...) an `input_dim` argument. Example ------- >>> import deepchem as dc >>> model = dc.models.Sequential() >>> # Add features >>> model.add_features(dc.nn.Input(shape=(50,))) >>> # Add labels >>> model.add_labels(dc.nn.Input(shape=(1,))) >>> model.add(dc.nn.Dense(32, 50)) >>> model.add(dc.nn.Dense(64, 32)) """ def __init__(self, name=None, logdir=None): super(Sequential, self).__init__(self, model_dir=logdir) self.layers = [] # stack of layers self.outputs = None # tensors (length 1) if not name: prefix = 'sequential_' name = prefix + str(model_ops.get_uid(prefix)) self.name = name self.graph = tf.Graph() config = tf.ConfigProto(allow_soft_placement=True) self.session = tf.Session(graph=self.graph, config=config) # Path to save checkpoint files self._save_path = os.path.join(self.model_dir, 'model.ckpt') def add(self, layer): """Adds a layer instance on top of the layer stack. Parameters ---------- layer: layer instance. """ if not isinstance(layer, Layer): raise TypeError("The added layer must be an instance of class Layer. " "Found: " + str(layer)) with self.graph.as_default(): if not self.layers: raise ValueError("Call add_features() before calling add()") # first layer in model: check that it is an input layer else: self.outputs = layer(self.outputs) self.layers.append(layer) def add_features(self, layer): """Adds an input layer.""" if self.layers: raise ValueError( "add_features() has to be called before layers are added.") if not isinstance(layer, InputLayer): raise ValueError("First layer in sequential model must be InputLayer") with self.graph.as_default(): self.features = layer()[0] self.outputs = self.features self.layers = [layer] def add_labels(self, layer): """Adds a layer for labels""" with self.graph.as_default(): self.labels = layer()[0] def add_loss(self, loss, inputs=None): """Adds a loss to model. Parameters ---------- losses: list """ # Add losses to graph with self.graph.as_default(): # Loss for each batch element batch_loss = loss(self.outputs, self.labels) # Loss should be a float self.loss = tf.reduce_sum(batch_loss) @property def uses_learning_phase(self): return self.uses_learning_phase def fit(self, dataset, nb_epoch=10, max_checkpoints_to_keep=5, log_every_N_batches=50, learning_rate=.001, batch_size=50, checkpoint_interval=10): """Trains the model for a fixed number of epochs. TODO(rbharath0: This is mostly copied from TensorflowGraphModel. Should eventually refactor both together. Parameters ---------- dataset: dc.data.Dataset nb_epoch: 10 Number of training epochs. Dataset object holding training data batch_size: integer. Number of samples per gradient update. nb_epoch: integer, the number of epochs to train the model. verbose: 0 for no logging to stdout, 1 for progress bar logging, 2 for one log line per epoch. initial_epoch: epoch at which to start training (useful for resuming a previous training run) checkpoint_interval: int Frequency at which to write checkpoints, measured in epochs """ ############################################################## TIMING time1 = time.time() ############################################################## TIMING print("Training for %d epochs" % nb_epoch) with self.graph.as_default(): opt = model_ops.optimizer("adam", learning_rate) train_op = opt.minimize(self.loss, name='train') with self.session as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(max_to_keep=max_checkpoints_to_keep) # Save an initial checkpoint. saver.save(sess, self._save_path, global_step=0) for epoch in range(nb_epoch): avg_loss, n_batches = 0., 0 # TODO(rbharath): Don't support example weighting yet. for ind, (X_b, y_b, w_b, ids_b) in enumerate(dataset.iterbatches(batch_size)): if ind % log_every_N_batches == 0: print("On batch %d" % ind) feed_dict = {self.features: X_b, self.labels: y_b} fetches = [self.outputs] + [train_op, self.loss] fetched_values = sess.run(fetches, feed_dict=feed_dict) output = fetched_values[:1] loss = fetched_values[-1] avg_loss += loss y_pred = np.squeeze(np.array(output)) y_b = y_b.flatten() n_batches += 1 if epoch % checkpoint_interval == checkpoint_interval - 1: saver.save(sess, self._save_path, global_step=epoch) avg_loss = float(avg_loss) / n_batches print('Ending epoch %d: Average loss %g' % (epoch, avg_loss)) # Always save a final checkpoint when complete. saver.save(sess, self._save_path, global_step=epoch + 1) ############################################################## TIMING time2 = time.time() print("TIMING: model fitting took %0.3f s" % (time2 - time1)) ############################################################## TIMING def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None, **kwargs): """Computes the loss on some input data, batch by batch. Parameters ---------- x: input data, as a Numpy array or list of Numpy arrays (if the model has multiple inputs). y: labels, as a Numpy array. batch_size: integer. Number of samples per gradient update. verbose: verbosity mode, 0 or 1. sample_weight: sample weights, as a Numpy array. Returns ------- Scalar test loss (if the model has no metrics) or list of scalars (if the model computes other metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ if self.model is None: raise RuntimeError('The model needs to be compiled ' 'before being used.') if 'show_accuracy' in kwargs: kwargs.pop('show_accuracy') warnings.warn('The "show_accuracy" argument is deprecated, ' 'instead you should pass the "accuracy" metric to ' 'the model at compile time:\n' '`model.compile(optimizer, loss, ' 'metrics=["accuracy"])`') if kwargs: raise TypeError('Received unknown keyword arguments: ' + str(kwargs)) return self.model.evaluate( x, y, batch_size=batch_size, verbose=verbose, sample_weight=sample_weight) def predict(self, x, batch_size=32, verbose=0): """Generates output predictions for the input samples, processing the samples in a batched way. # Arguments x: the input data, as a Numpy array. batch_size: integer. verbose: verbosity mode, 0 or 1. # Returns A Numpy array of predictions. """ if self.model is None: self.build() return self.model.predict(x, batch_size=batch_size, verbose=verbose) def predict_on_batch(self, x): """Returns predictions for a single batch of samples. """ if self.model is None: self.build() return self.model.predict_on_batch(x) def train_on_batch(self, x, y, class_weight=None, sample_weight=None, **kwargs): """Single gradient update over one batch of samples. # Arguments x: input data, as a Numpy array or list of Numpy arrays (if the model has multiple inputs). y: labels, as a Numpy array. class_weight: dictionary mapping classes to a weight value, used for scaling the loss function (during training only). sample_weight: sample weights, as a Numpy array. # Returns Scalar training loss (if the model has no metrics) or list of scalars (if the model computes other metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ if self.model is None: raise RuntimeError('The model needs to be compiled ' 'before being used.') if 'accuracy' in kwargs: kwargs.pop('accuracy') warnings.warn('The "accuracy" argument is deprecated, ' 'instead you should pass the "accuracy" metric to ' 'the model at compile time:\n' '`model.compile(optimizer, loss, ' 'metrics=["accuracy"])`') if kwargs: raise TypeError('Received unknown keyword arguments: ' + str(kwargs)) return self.model.train_on_batch( x, y, sample_weight=sample_weight, class_weight=class_weight) def test_on_batch(self, x, y, sample_weight=None, **kwargs): """Evaluates the model over a single batch of samples. # Arguments x: input data, as a Numpy array or list of Numpy arrays (if the model has multiple inputs). y: labels, as a Numpy array. sample_weight: sample weights, as a Numpy array. # Returns Scalar test loss (if the model has no metrics) or list of scalars (if the model computes other metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ if self.model is None: raise RuntimeError('The model needs to be compiled ' 'before being used.') if 'accuracy' in kwargs: kwargs.pop('accuracy') warnings.warn('The "accuracy" argument is deprecated, ' 'instead you should pass the "accuracy" metric to ' 'the model at compile time:\n' '`model.compile(optimizer, loss, ' 'metrics=["accuracy"])`') if kwargs: raise TypeError('Received unknown keyword arguments: ' + str(kwargs)) return self.model.test_on_batch(x, y, sample_weight=sample_weight) def predict_proba(self, x, batch_size=32, verbose=1): """Generates class probability predictions for the input samples batch by batch. # Arguments x: input data, as a Numpy array or list of Numpy arrays (if the model has multiple inputs). batch_size: integer. verbose: verbosity mode, 0 or 1. # Returns A Numpy array of probability predictions. """ preds = self.predict(x, batch_size, verbose) if preds.min() < 0. or preds.max() > 1.: warnings.warn('Network returning invalid probability values. ' 'The last layer might not normalize predictions ' 'into probabilities ' '(like softmax or sigmoid would).') return preds
deepchem/models/tensorgraph/layers.py +1 −1 Original line number Diff line number Diff line Loading @@ -1084,7 +1084,7 @@ class SoftMax(Layer): def create_tensor(self, in_layers=None, set_tensors=True, **kwargs): inputs = self._get_input_tensors(in_layers) if len(inputs) != 1: raise ValueError("Must only Softmax single parent") raise ValueError("Softmax must have a single input layer.") parent = inputs[0] out_tensor = tf.contrib.layers.softmax(parent) if set_tensors: Loading
deepchem/models/tensorgraph/models/gan.py +120 −45 Original line number Diff line number Diff line Loading @@ -50,14 +50,33 @@ class GAN(TensorGraph): create_generator_loss() create_discriminator_loss() get_noise_batch() This class allows a GAN to have multiple generators and discriminators, a model known as MIX+GAN. It is described in Arora et al., "Generalization and Equilibrium in Generative Adversarial Nets (GANs)" (https://arxiv.org/abs/1703.00573). This can lead to better models, and is especially useful for reducing mode collapse, since different generators can learn different parts of the distribution. To use this technique, simply specify the number of generators and discriminators when calling the constructor. You can then tell predict_gan_generator() which generator to use for predicting samples. """ def __init__(self, **kwargs): def __init__(self, n_generators=1, n_discriminators=1, **kwargs): """Construct a GAN. This class accepts all the keyword arguments from TensorGraph. In addition to the parameters listed below, this class accepts all the keyword arguments from TensorGraph. Parameters ---------- n_generators: int the number of generators to include n_discriminators: int the number of discriminators to include """ super(GAN, self).__init__(use_queue=False, **kwargs) self.n_generators = n_generators self.n_discriminators = n_discriminators # Create the inputs. Loading @@ -69,40 +88,49 @@ class GAN(TensorGraph): for shape in self.get_conditional_input_shapes(): self.conditional_inputs.append(layers.Feature(shape=shape)) # Create the generator. # Create the generators. self.generator = self.create_generator(self.noise_input, self.generators = [] for i in range(n_generators): generator = self.create_generator(self.noise_input, self.conditional_inputs) if not isinstance(self.generator, Sequence): if not isinstance(generator, Sequence): raise ValueError('create_generator() must return a list of Layers') if len(self.generator) != len(self.data_inputs): if len(generator) != len(self.data_inputs): raise ValueError( 'The number of generator outputs must match the number of data inputs' ) for g, d in zip(self.generator, self.data_inputs): for g, d in zip(generator, self.data_inputs): if g.shape != d.shape: raise ValueError( 'The shapes of the generator outputs must match the shapes of the data inputs' ) for g in self.generator: for g in generator: self.add_output(g) self.generators.append(generator) # Create the discriminator. # Create the discriminators. self.discrim_train = self.create_discriminator(self.data_inputs, self.discrim_train = [] self.discrim_gen = [] for i in range(n_discriminators): discrim_train = self.create_discriminator(self.data_inputs, self.conditional_inputs) self.discrim_train.append(discrim_train) # Make a copy of the discriminator that takes the generator's output as # Make a copy of the discriminator that takes each generator's output as # its input. for generator in self.generators: replacements = {} for g, d in zip(self.generator, self.data_inputs): for g, d in zip(generator, self.data_inputs): replacements[d] = g for c in self.conditional_inputs: replacements[c] = c self.discrim_gen = self.discrim_train.copy(replacements, shared=True) discrim_gen = discrim_train.copy(replacements, shared=True) self.discrim_gen.append(discrim_gen) # Make a list of all layers in the generator and discriminator. # Make a list of all layers in the generators and discriminators. def add_layers_to_set(layer, layers): if layer not in layers: Loading @@ -111,21 +139,63 @@ class GAN(TensorGraph): add_layers_to_set(i, layers) gen_layers = set() for layer in self.generator: for generator in self.generators: for layer in generator: add_layers_to_set(layer, gen_layers) discrim_layers = set() add_layers_to_set(self.discrim_train, discrim_layers) for discriminator in self.discrim_train: add_layers_to_set(discriminator, discrim_layers) discrim_layers -= gen_layers # Create submodels for training the generator and discriminator. # Compute the loss functions. gen_losses = [self.create_generator_loss(d) for d in self.discrim_gen] discrim_losses = [] for i in range(n_discriminators): for j in range(n_generators): discrim_losses.append( self.create_discriminator_loss( self.discrim_train[i], self.discrim_gen[i * n_generators + j])) if n_generators == 1 and n_discriminators == 1: total_gen_loss = gen_losses[0] total_discrim_loss = discrim_losses[0] else: # Create learnable weights for the generators and discriminators. gen_alpha = layers.Variable(np.ones((1, n_generators))) gen_weights = layers.SoftMax(gen_alpha) discrim_alpha = layers.Variable(np.ones((1, n_discriminators))) discrim_weights = layers.SoftMax(discrim_alpha) # Compute the weighted errors weight_products = layers.Reshape( (n_generators * n_discriminators,), in_layers=layers.Reshape( (n_discriminators, 1), in_layers=discrim_weights) * layers.Reshape( (1, n_generators), in_layers=gen_weights)) total_gen_loss = layers.WeightedError((layers.Stack(gen_losses, axis=0), weight_products)) total_discrim_loss = layers.WeightedError((layers.Stack( discrim_losses, axis=0), weight_products)) gen_layers.add(gen_alpha) discrim_layers.add(gen_alpha) discrim_layers.add(discrim_alpha) # Add an entropy term to the loss. entropy = -( layers.ReduceSum(layers.Log(gen_weights)) / n_generators + layers.ReduceSum(layers.Log(discrim_weights)) / n_discriminators) total_discrim_loss += entropy # Create submodels for training the generators and discriminators. gen_loss = self.create_generator_loss(self.discrim_gen) discrim_loss = self.create_discriminator_loss(self.discrim_train, self.discrim_gen) self.generator_submodel = self.create_submodel( layers=gen_layers, loss=gen_loss) layers=gen_layers, loss=total_gen_loss) self.discriminator_submodel = self.create_submodel( layers=discrim_layers, loss=discrim_loss) layers=discrim_layers, loss=total_discrim_loss) def get_noise_input_shape(self): """Get the shape of the generator's noise input layer. Loading Loading @@ -370,7 +440,8 @@ class GAN(TensorGraph): def predict_gan_generator(self, batch_size=1, noise_input=None, conditional_inputs=[]): conditional_inputs=[], generator_index=0): """Use the GAN to generate a batch of samples. Parameters Loading @@ -386,6 +457,9 @@ class GAN(TensorGraph): conditional_inputs: list of arrays the values to use for all conditional inputs. This must be specified if the GAN has any conditional inputs. generator_index: int the index of the generator (between 0 and n_generators-1) to use for generating the samples. Returns ------- Loading @@ -402,7 +476,8 @@ class GAN(TensorGraph): batch[self.noise_input] = noise_input for layer, value in zip(self.conditional_inputs, conditional_inputs): batch[layer] = value return self.predict_on_generator([batch]) return self.predict_on_generator( [batch], outputs=self.generators[generator_index]) def _set_empty_inputs(self, feed_dict, layers): """Set entries in a feed dict corresponding to a batch size of 0.""" Loading
deepchem/models/tensorgraph/models/graph_models.py +127 −115 Original line number Diff line number Diff line Loading @@ -338,6 +338,18 @@ class DTNNTensorGraph(TensorGraph): yield feed_dict def predict(self, dataset, transformers=[], outputs=None): if outputs is None: outputs = self.outputs if transformers != [] and not isinstance(outputs, collections.Sequence): raise ValueError( "DTNN does not support single tensor output with transformers") retval = super(DTNNTensorGraph, self).predict(dataset, outputs=outputs) if not isinstance(outputs, collections.Sequence): return retval retval = np.concatenate(retval, axis=-1) return undo_transforms(retval, transformers) class DAGTensorGraph(TensorGraph): Loading
