Fit works in sequential

__copyright__ = "Copyright 2017, Stanford University"

__license__ = "GPL"

import time

import os

import tempfile

import numpy as np

import tensorflow as tf

from deepchem.models.models import Model

from deepchem.nn import model_ops

  Example

  -------

  >>> model = dc.models.Sequential()

  >>> # first layer must have a defined input shape

  >>> model.add(dc.nn.Dense(32, input_dim=500))

  >>> # afterwards, Keras does automatic shape inference

  >>> model.add(dc.nn.Dense(32))

  >>> # also possible (equivalent to the above):

  >>> model = dc.models.Sequential()

  >>> model.add(dc.nn.Dense(32, input_shape=(500,)))

  >>> model.add(dc.nn.Dense(32))

  >>> # also possible (equivalent to the above):

  >>> model = dc.models.Sequential()

  >>> # here the batch dimension is None,

  >>> # which means any batch size will be accepted by the model.

  >>> model.add(dc.nn.Dense(32, batch_input_shape=(None, 500)))

  >>> model.add(dc.nn.Dense(32))

  >>> # Add features

  >>> model.add_features(dc.nn.Input(shape=(50,)))

  >>> # Add labels

  >>> model.add_features(dc.nn.Input(shape=(1,)))

  >>> model.add(dc.nn.Dense(32, 50))

  >>> model.add(dc.nn.Dense(64, 32))

  """

  def __init__(self, name=None):

  def __init__(self, name=None, logdir=None):

    self.layers = []  # stack of layers

    self.outputs = []  # tensors (length 1)

    self.name = name

    self.graph = tf.Graph()

    config = tf.ConfigProto(allow_soft_placement=True)

    self.session = tf.Session(graph=self.graph, config=config)

    # Path to save checkpoint files

    if logdir is not None:

      if not os.path.exists(logdir):

        os.makedirs(logdir)

    else:

      logdir = tempfile.mkdtemp()

    self.logdir = logdir

    self._save_path = os.path.join(self.logdir, 'model.ckpt')

  def add(self, layer):

    """Adds a layer instance on top of the layer stack.

                      "Found: " + str(layer))

    with self.graph.as_default():

      if not self.layers:

        raise ValueError("Call add_features() before calling add()")

        # first layer in model: check that it is an input layer

        if not isinstance(layer, InputLayer):

          raise ValueError("First layer in sequential model must be InputLayer")

        self.outputs = layer()

      else:

        self.outputs = layer(self.outputs[0])

      self.layers.append(layer)

  def add_features(self, layer):

    """Adds an input layer."""

    if self.layers:

      raise ValueError("add_features() has to be called before layers are added.")

    if not isinstance(layer, InputLayer):

      raise ValueError("First layer in sequential model must be InputLayer")

    with self.graph.as_default():

      self.features = layer()[0]

      self.outputs = [self.features]

      self.layers = [layer]

  def add_labels(self, layer):

    """Adds a layer for labels"""

    with self.graph.as_default():

      self.labels = layer()[0]

  def add_loss(self, loss, inputs=None):

    """Adds a loss to model.

    """

    # Add losses to graph

    with self.graph.as_default():

      self.loss = loss()

      # Loss for each batch element

      batch_loss = loss(self.outputs[0], self.labels)

      # Loss should be a float

      self.loss = tf.reduce_sum(batch_loss)

  @property

  def uses_learning_phase(self):

    return self.uses_learning_phase

  def fit(self, dataset, batch_size=32, nb_epoch=10, verbose=1,

          initial_epoch=0, **kwargs):

  def fit(self, dataset, nb_epoch=10, max_checkpoints_to_keep=5,

          log_every_N_batches=50, learning_rate=.001, batch_size=50):

    """Trains the model for a fixed number of epochs.

    # Arguments

        x: input data, as a Numpy array or list of Numpy arrays

            (if the model has multiple inputs).

        y: labels, as a Numpy array.

    TODO(rbharath0: This is mostly copied from TensorflowGraphModel. Should

    eventually refactor both together.

    Parameters

    ----------

    dataset: dc.data.Dataset

    nb_epoch: 10

      Number of training epochs.

      Dataset object holding training data

        batch_size: integer. Number of samples per gradient update.

        nb_epoch: integer, the number of epochs to train the model.

        verbose: 0 for no logging to stdout,

        initial_epoch: epoch at which to start training

            (useful for resuming a previous training run)

    """

    pass

    #return self.model.fit(x, y,

    #                      batch_size=batch_size,

    #                      nb_epoch=nb_epoch,

    #                      verbose=verbose,

    #                      initial_epoch=initial_epoch)

    ############################################################## TIMING

    time1 = time.time()

    ############################################################## TIMING

    print("Training for %d epochs" % nb_epoch)

    with self.graph.as_default():

      opt = model_ops.optimizer("adam", learning_rate)

      train_op = opt.minimize(self.loss, name='train')

      with self.session as sess:

        sess.run(tf.global_variables_initializer())

        ############################################################ DEBUG

        #print("after global variable initialization")

        #print("[var.name for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)]")

        #print([var.name for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)])

        ############################################################ DEBUG

        saver = tf.train.Saver(max_to_keep=max_checkpoints_to_keep)

        # Save an initial checkpoint.

        saver.save(sess, self._save_path, global_step=0)

        for epoch in range(nb_epoch):

          avg_loss, n_batches = 0., 0

          # TODO(rbharath): Don't support example weighting yet.

          for ind, (X_b, y_b, w_b, ids_b) in enumerate(

              dataset.iterbatches(batch_size)):

            if ind % log_every_N_batches == 0:

              print("On batch %d" % ind)

            feed_dict = {self.features: X_b,

                         self.labels: y_b}

            fetches = self.outputs + [train_op, self.loss]

            fetched_values = sess.run(fetches, feed_dict=feed_dict)

            output = fetched_values[:len(self.outputs)]

            ######################################### DEBUG

            print("fetched_values")

            print(fetched_values)

            ######################################### DEBUG

            loss = fetched_values[-1]

            avg_loss += loss

            y_pred = np.squeeze(np.array(output))

            y_b = y_b.flatten()

            n_batches += 1

          saver.save(sess, self._save_path, global_step=epoch)

          ######################################### DEBUG

          print("avg_loss")

          print(avg_loss)

          ######################################### DEBUG

          avg_loss = float(avg_loss)/n_batches

          print('Ending epoch %d: Average loss %g' % (epoch, avg_loss))

        # Always save a final checkpoint when complete.

        saver.save(sess, self._save_path, global_step=epoch+1)

    ############################################################## TIMING

    time2 = time.time()

    print("TIMING: model fitting took %0.3f s" % (time2-time1))

    ############################################################## TIMING

  def evaluate(self, x, y, batch_size=32, verbose=1,

               sample_weight=None, **kwargs):

                        'into probabilities '

                        '(like softmax or sigmoid would).')

      return preds

  def predict_classes(self, x, batch_size=32, verbose=1):

      """Generate class predictions for the input samples

      batch by batch.

      # Arguments

          x: input data, as a Numpy array or list of Numpy arrays

              (if the model has multiple inputs).

          batch_size: integer.

          verbose: verbosity mode, 0 or 1.

      # Returns

          A numpy array of class predictions.

      """

      proba = self.predict(x, batch_size=batch_size, verbose=verbose)

      if proba.shape[-1] > 1:

          return proba.argmax(axis=-1)

      else:

          return (proba > 0.5).astype('int32')

import numpy as np

import deepchem as dc

class TestAPI(unittest.TestCase):

class TestSequential(unittest.TestCase):

  """

  Test API for sequential models 

  """

  def test_model_construct(self):

    """Test that models can be constructed with Sequential."""

    model = dc.models.Sequential()

    model.add(dc.nn.Input(shape=(32,)))

    model.add_features(dc.nn.Input(shape=(32,)))

    model.add_labels(dc.nn.Input(shape=(1,)))

    model.add(dc.nn.Dense(32, 32))

    model.add(dc.nn.Dense(32, 32))

  def test_model_fit(self):

    """Test that models can be fit"""

    model = dc.models.Sequential()

    model.add(dc.nn.Input(shape=(32,)))

    model.add_features(dc.nn.Input(shape=(32,)))

    model.add_labels(dc.nn.Input(shape=(1,)))

    model.add(dc.nn.Dense(32, 32))

    model.add(dc.nn.Dense(1, 32))

    model.add_loss(dc.nn.mean_squared_error)

    X = np.zeros((10, 32))

    y = np.zeros((10,))

    dataset = dc.data.NumpyDataset(X, y)

from deepchem.nn import initializations

from deepchem.nn import regularizers

from deepchem.nn import activations

from deepchem.nn import constraints

from deepchem.nn import model_ops

def to_list(x):

  object_list = to_list(object_list)

  return ', '.join([str(abs(id(x))) for x in object_list])

class InputSpec(object):

  """This specifies the ndim, dtype and shape of every input to a layer.

  Every layer should expose (if appropriate) an `input_spec` attribute:

  a list of instances of InputSpec (one per input tensor).

  A None entry in a shape is compatible with any dimension,

  a None shape is compatible with any shape.

  """

  def __init__(self, dtype=None, shape=None, ndim=None):

    if isinstance(ndim, str):

      if '+' not in ndim:

          raise ValueError('When passing a str "ndim", '

                           'it should have the form "2+", "3+", etc.')

      int_ndim = ndim[:ndim.find('+')]

      if not int_ndim.isdigit():

          raise ValueError('When passing a str "ndim", '

                           'it should have the form "2+", "3+", etc.')

    if shape is not None:

      self.ndim = len(shape)

    else:

      self.ndim = ndim

    self.dtype = dtype

    self.shape = shape

class Layer(object):

  """Abstract base layer class.

  Attributes

  ----------

  name: String, must be unique within a model.

  input_spec: List of InputSpec class instances

    each entry describes one required input:

        - ndim

        - dtype

    A layer with n input tensors must have

    an input_spec of length n.

  trainable: Boolean, whether the layer weights

      will be updated during training.

  uses_learning_phase: Whether any operation

  input, output: Input/output tensor(s). Note that if the layer is used

    more than once (shared layer), this is ill-defined

    and will raise an exception. In such cases, use

  trainable_weights: List of variables.

  non_trainable_weights: List of variables.

  weights: The concatenation of the lists trainable_weights and

      non_trainable_weights (in this order).

  constraints: Dict mapping weights to constraints.

  Methods

  -------

          - Connect current layer with last layer from tensor:

          - Add layer to tensor history

      If layer is not built:

  count_params()

  get_output_shape_for(input_shape)

  # Internal methods:

  build(input_shape)

  """

  def __init__(self, **kwargs):

    # These properties should have been set

    # by the child class, as appropriate.

    if not hasattr(self, 'input_spec'):

      self.input_spec = None

    if not hasattr(self, 'uses_learning_phase'):

      self.uses_learning_phase = False

    # These properties will be set upon call of self.build(),

    if not hasattr(self, '_trainable_weights'):

      self._trainable_weights = []

    if not hasattr(self, '_non_trainable_weights'):

      self._non_trainable_weights = []

    if not hasattr(self, 'losses'):

      self.losses = []

    if not hasattr(self, 'constraints'):

      self.constraints = {}  # dict {tensor: constraint instance}

    self.built = False

    # These properties should be set by the user via keyword arguments.

    # note that 'input_dtype', 'input_shape' and 'batch_input_shape'

      input_dtype = kwargs.get('input_dtype', tf.float32)

      self.input_dtype = input_dtype

  @property

  def trainable_weights(self):

    trainable = getattr(self, 'trainable', True)

    if trainable:

      return self._trainable_weights

    else:

      return []

  @trainable_weights.setter

  def trainable_weights(self, weights):

    self._trainable_weights = weights

  @property

  def non_trainable_weights(self):

    trainable = getattr(self, 'trainable', True)

    if not trainable:

      return self._trainable_weights + self._non_trainable_weights

    else:

      return self._non_trainable_weights

  @non_trainable_weights.setter

  def non_trainable_weights(self, weights):

    self._non_trainable_weights = weights

  def add_weight(self, shape, initializer, name=None,

                 trainable=True)

  def add_weight(self, shape, initializer, name=None):

    """Adds a weight variable to the layer.

    Parameters

    ----------

    shape: The shape tuple of the weight.

    initializer: An Initializer instance (callable).

    trainable: A boolean, whether the weight should

      be trained via backprop or not (assuming

      that the layer itself is also trainable).

    regularizer: An optional Regularizer instance.

    """

    initializer = initializations.get(initializer)

    weight = initializer(shape, name=name)

    if trainable:

      self._trainable_weights.append(weight)

    else:

      self._non_trainable_weights.append(weight)

    return weight

  def call(self, x):

  def __call__(self, x):

    """Wrapper around self.call(), for handling

    internal Keras references.

    If a tensor is passed:

      - If necessary, we `build` the layer to match

    Parameters

    ----------

    x: Can be a tensor or list/tuple of tensors.

    """

    input_tensors = to_list(x)

    outputs = to_list(self.call(x))

    return outputs

  def get_output_shape_for(self, input_shape):

    """Computes the output shape of the layer given

    an input shape (assumes that the layer will be built

    to match that input shape).

    Parameters

    ----------

    input_shape: Shape tuple (tuple of integers)

      or list of shape tuples (one per output tensor of the layer).

      Shape tuples can include None for free dimensions,

      instead of an integer.

    """

    return input_shape

  @property

  def weights(self):

    return self.trainable_weights + self.non_trainable_weights

class InputLayer(Layer):

  """Layer to be used as an entry point into a graph.

  def __init__(self, input_shape=None, batch_input_shape=None,

               input_dtype=None, name=None):

    self.input_spec = None

    self.uses_learning_phase = False

    self.trainable = False

    self._trainable_weights = []

    self._non_trainable_weights = []

    self.constraints = {}

    if not name:

      prefix = 'input'

  b_regularizer: instance of regularize applied to the bias.

  activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),

    applied to the network output.

  W_constraint: instance of the [constraints](../constraints.md) module

    (eg. maxnorm, nonneg), applied to the main weights matrix.

  b_constraint: instance of the [constraints](../constraints.md) module,

    applied to the bias.

  bias: whether to include a bias

    (i.e. make the layer affine rather than linear).

  input_dim: dimensionality of the input (integer). This argument

  """

  def __init__(self, output_dim, input_dim, init='glorot_uniform',

               activation=None, bias=True, **kwargs):

               activation="relu", bias=True, **kwargs):

    self.init = initializations.get(init)

    self.activation = activations.get(activation)

    self.output_dim = output_dim

    self.input_dim = input_dim

    self.bias = bias

    self.input_spec = [InputSpec(ndim='2+')]

    input_shape = (self.input_dim,)

    if self.input_dim:

      kwargs['input_shape'] = (self.input_dim,)

    super(Dense, self).__init__(**kwargs)

    self.input_dim = input_dim

    self.input_spec = [InputSpec(dtype=tf.float32,

                                 ndim='2+')]

  def call(self, x):

  def __call__(self, x):

    self.W = self.add_weight((self.input_dim, self.output_dim),

                             initializer=self.init,

                             name='{}_W'.format(self.name))

    if self.bias:

      self.b = self.add_weight((self.output_dim,),

                               initializer='zero',

    self.b = self.add_weight((self.output_dim,), initializer='zero',

                                name='{}_b'.format(self.name))

    else:

      self.b = None

    ######################################################## DEBUG

    print("Created variables!")

    print("[var.name for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)]")

    print([var.name for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)])

    ######################################################## DEBUG

    output = model_ops.dot(x, self.W)

    if self.bias:

      output += self.b

    return self.activation(output)

  def get_output_shape_for(self, input_shape):

    assert input_shape and len(input_shape) >= 2

    assert input_shape[-1] and input_shape[-1] == self.input_dim

    output_shape = list(input_shape)

    output_shape[-1] = self.output_dim

    return tuple(output_shape)

    outputs = to_list(self.activation(output))

    return outputs

class Dropout(Layer):

  """Applies Dropout to the input.

    super(BatchNormalization, self).__init__(**kwargs)

  def build(self, input_shape):

    self.input_spec = [InputSpec(shape=input_shape)]

    shape = (input_shape[self.axis],)

    self.gamma = self.add_weight(shape,

                                       name='{}_running_std'.format(self.name),

                                       trainable=False)

    self.built = True

  def call(self, x):

    if self.mode == 0 or self.mode == 2:

      assert self.built, 'Layer must be built before being called'

      input_shape = model_ops.int_shape(x)

      reduction_axes = list(range(len(input_shape)))

import numpy as np

import tensorflow as tf

from deepchem.nn.model_ops import variable

#from deepchem.nn.model_ops import variable

from deepchem.nn.model_ops import random_uniform_variable

from deepchem.nn.model_ops import random_normal_variable

from deepchem.nn.activations import get_from_module

  # Pick the one with the correct shape.

  q = u if u.shape == flat_shape else v

  q = q.reshape(shape)

  return variable(scale * q[:shape[0], :shape[1]], name=name)

  return tf.Variable(scale * q[:shape[0], :shape[1]], dtype=tf.float32,

                     name=name)

def identity(shape, scale=1, name=None):

    raise ValueError('Identity matrix initialization can only be used '

                     'for 2D square matrices.')

  else:

    return variable(scale * np.identity(shape[0]), name=name)

    return tf.Variable(scale * np.identity(shape[0]), dtype=tf.float32,

                       name=name)

def zero(shape, name=None):

  return tf.zeros(shape, name=name)

  return tf.Variable(tf.zeros(shape), dtype=tf.float32, name=name)

def one(shape, name=None):

  return tf.ones(shape, name=name)

  return tf.Variable(tf.ones(shape), dtype=tf.float32, name=name)

def get(identifier, **kwargs):

  return get_from_module(identifier, globals(),

  if dtype is None:

    dtype = tf.float32 

  shape = tuple(map(int, shape))

  return variable(tf.constant_initializer(1., dtype=dtype)(shape),

  return tf.Variable(tf.constant_initializer(1., dtype=dtype)(shape),

                     dtype, name)

def cast_to_floatx(x):

  """

  return 1e-7 

def variable(value, dtype=tf.float32, name=None):

  """Instantiates a variable and returns it.

  Parameters

  ----------

  value: Numpy array, initial value of the tensor.

  dtype: Tensor type.

  name: Optional name string for the tensor.

  Returns

  -------

  A variable instance (with Keras metadata included).

  """

  v = tf.Variable(value, dtype=dtype, name=name)

  if hasattr(value, 'get_shape'):

    v._keras_shape = tuple(map(int, value.get_shape()))

  v._uses_learning_phase = False

  return v

#def variable(value, dtype=tf.float32, name=None):

#  """Instantiates a variable and returns it.

#

#  Parameters

#  ----------