Should have fixed (almost) all tests

from deepchem.models.tensorflow_models import utils as tf_utils

from deepchem.utils.save import log

from deepchem.datasets import pad_features

from tensorflow.contrib.layers.python.layers import batch_norm

def softmax(x):

  """Simple numpy softmax implementation

          # weight decay

          if self.penalty != 0.0:

            penalty = model_ops.WeightDecay(self.penalty_type, self.penalty)

            penalty = model_ops.weight_decay(self.penalty_type, self.penalty)

            loss += penalty

      return loss 

    A training op.

    """

    with graph.as_default():

      opt = model_ops.Optimizer(self.optimizer, self.learning_rate, self.momentum)

      opt = model_ops.optimizer(self.optimizer, self.learning_rate, self.momentum)

      return opt.minimize(loss, name='train')

  def _get_shared_session(self, train):

  """Classification model.

  Subclasses must set the following attributes:

    output: Logits op(s) used for computing classification loss and predicted

    output: logits op(s) used for computing classification loss and predicted

      class probabilities for each task.

  Class attributes:

      prev_layer = self.mol_features

      prev_layer_size = n_features 

      for i in xrange(n_layers):

        layer = tf.nn.relu(model_ops.FullyConnectedLayer(

        layer = tf.nn.relu(model_ops.fully_connected_layer(

            tensor=prev_layer,

            size=layer_sizes[i],

            weight_init=tf.truncated_normal(

                stddev=weight_init_stddevs[i]),

            bias_init=tf.constant(value=bias_init_consts[i],

                                  shape=[layer_sizes[i]])))

        layer = model_ops.Dropout(layer, dropouts[i], training)

        layer = model_ops.dropout(layer, dropouts[i], training)

        prev_layer = layer

        prev_layer_size = layer_sizes[i]

      output = model_ops.MultitaskLogits(

      output = model_ops.multitask_logits(

          layer, self.n_tasks)

    return output

      prev_layer = self.mol_features

      prev_layer_size = n_features 

      for i in xrange(n_layers):

        layer = tf.nn.relu(model_ops.FullyConnectedLayer(

        layer = tf.nn.relu(model_ops.fully_connected_layer(

            tensor=prev_layer,

            size=layer_sizes[i],

            weight_init=tf.truncated_normal(

                stddev=weight_init_stddevs[i]),

            bias_init=tf.constant(value=bias_init_consts[i],

                                  shape=[layer_sizes[i]])))

        layer = model_ops.Dropout(layer, dropouts[i])

        layer = model_ops.dropout(layer, dropouts[i])

        prev_layer = layer

        prev_layer_size = layer_sizes[i]

      output = []

      for task in range(self.n_tasks):

        output.append(tf.squeeze(

            model_ops.FullyConnectedLayer(

            model_ops.fully_connected_layer(

                tensor=prev_layer,

                size=layer_sizes[i],

                weight_init=tf.truncated_normal(

from __future__ import unicode_literals

import tensorflow as tf

from google.protobuf import text_format

from tensorflow.python.platform import gfile

from deepchem.models.tensorflow_models import utils as model_utils

import sys

import traceback

import tensorflow as tf

def AddBias(tensor, init=None, name=None):

def add_bias(tensor, init=None, name=None):

  """Add a bias term to a tensor.

  Args:

    return tf.nn.bias_add(tensor, b)

def BatchNormalize(tensor, convolution, training=True, mask=None,

                   epsilon=0.001, scale_after_normalization=True, decay=0.999,

                   global_step=None, name=None):

  """Batch normalization.

  Normalize, scale, and shift the input tensor to reduce covariate shift.

  NOTE(user): For inference, the mean and variance must be set to fixed

  values derived from the entire training set. This is accomplished by using

  moving_mean and moving_variance during evaluation. Be sure that models run

  the ops in updates during training or the moving averages will not be very

  useful!

  Args:

    tensor: Input tensor (must be 4D).

    convolution: If True, perform normalization across rows and columns as

      well as over batch.

    mask: Mask to apply to tensor.

    epsilon: Small float to avoid dividing by zero.

    scale_after_normalization: If True, multiply by gamma. If False, gamma is

      not used. When the next layer is linear (also e.g. ReLU), this can be

      disabled since the scaling can be done by the next layer.

    decay: Float value for moving average decay.

    global_step: Tensor containing global step for accelerating moving averages

      at the beginning of training.

    name: Name for this op. Defaults to 'batch_norm'.

  Returns:

    A new tensor corresponding to the batch normalized input.

  Raises:

    ValueError: If the input tensor is not 4D.

  """

  if len(tensor.get_shape()) != 4:

    raise ValueError('Input tensor must be 4D, not %dD'

                     % len(tensor.get_shape()))

  if convolution:

    axes = [0, 1, 2]

    shape = tensor.get_shape()[3:]

  else:

    axes = [0]

    shape = tensor.get_shape()[1:]

  with tf.op_scope([tensor], None, 'BatchNormalize'):

    if mask is not None:

      mean, variance = model_utils.Moment(

          2, tensor, reduction_indices=axes, mask=mask)

    else:

      mean, variance = tf.nn.moments(tensor, axes)

    # Keep track of moving averages for mean and variance. During eval, use the

    # moving averages from training.

    mean_moving_average = MovingAverage(mean, global_step, decay)

    variance_moving_average = MovingAverage(variance, global_step, decay)

    if training:

      mean = mean_moving_average

      variance = variance_moving_average

    beta = tf.Variable(tf.zeros(shape), name='beta')

    gamma = tf.Variable(tf.constant(1.0, shape=shape), name='gamma')

    if convolution:

      batch_norm = tf.nn.batch_norm_with_global_normalization(

          tensor, mean, variance, beta, gamma, epsilon,

          scale_after_normalization)

    else:

      batch_norm = (tensor - mean) * tf.rsqrt(variance + epsilon)

      if scale_after_normalization:

        batch_norm *= gamma

      batch_norm += beta

    if mask is not None:

      batch_norm = model_utils.Mask(batch_norm, mask)

    return batch_norm

def MovingAverage(tensor, global_step, decay=0.999):

  """Create a variable that contains the moving average of a tensor.

  Adds a tf.identity and special namescope to ensure the tensor

  is colocated with its Variable on the parameter server.

  See http://g/tensorflow-users/PAAXYLlybNs/xA0z-x1qEwAJ

  and replicated_model.py#NameScopeDevicePicker for context.

  Args:

    tensor: Tensor to calculate moving average of.

    global_step: Variable containing the number of global steps.

    decay: Float for exponential decay of moving average.

  Returns:

    A tf.Variable containing the moving average of the input tensor.

  """

  exponential_moving_average = tf.train.ExponentialMovingAverage(

      decay=decay, num_updates=global_step)

  update_op = exponential_moving_average.apply([tensor])

  tf.get_default_graph().add_to_collection('updates', update_op)

  return exponential_moving_average.average(tensor)

def Dropout(tensor, dropout_prob, training=True, training_only=True):

def dropout(tensor, dropout_prob, training=True, training_only=True):

  """Random dropout.

  This implementation supports "always-on" dropout (training_only=False), which

  return tensor

def FullyConnectedLayer(tensor, size, weight_init=None, bias_init=None,

def fully_connected_layer(tensor, size, weight_init=None, bias_init=None,

                          name=None):

  """Fully connected layer.

    b = tf.Variable(bias_init, name='b', dtype=tf.float32)

    return tf.nn.xw_plus_b(tensor, w, b)

'''

def is_training():

  """Determine whether the default graph is in training mode.

  Returns:

    A boolean value indicating whether the default graph is in training mode.

  Raises:

    ValueError: If the 'train' collection in the default graph does not contain

      exactly one element.

  """

  #traceback.print_stack(file=sys.stdout) 

  train = tf.get_collection("train")

  if not train:

    raise ValueError('Training mode is not set. Please call set_training.')

  elif len(train) > 1:

    raise ValueError('Training mode has more than one setting.')

  return train[0]

'''

def WeightDecay(penalty_type, penalty):

def weight_decay(penalty_type, penalty):

  """Add weight decay.

  Args:

    tf.scalar_summary('Weight Decay Cost', cost)

  return cost

def set_training(train):

  """Set the training mode of the default graph.

  This operation may only be called once for a given graph.

  Args:

    graph: Tensorflow graph. 

    train: If True, graph is in training mode.

  Raises:

    AssertionError: If the default graph already has this value set.

  """

  if tf.get_collection('train'):

    raise AssertionError('Training mode already set: %s' %

                         graph.get_collection('train'))

  tf.add_to_collection('train', train)

def MultitaskLogits(features, num_tasks, num_classes=2, weight_init=None,

                    bias_init=None, dropout=None, name=None):

def multitask_logits(features, num_tasks, num_classes=2, weight_init=None,

                     bias_init=None, dropout_prob=None, name=None):

  """Create a logit tensor for each classification task.

  Args:

    num_classes: Number of classes for each task.

    weight_init: Weight initializer.

    bias_init: Bias initializer.

    dropout: Float giving dropout probability for weights (NOT keep

    dropout_prob: Float giving dropout probability for weights (NOT keep

      probability).

    name: Name for this op. Defaults to 'multitask_logits'.

  Returns:

    A list of logit tensors; one for each classification task.

  """

  logits = []

  logits_list = []

  with tf.name_scope('multitask_logits'):

    for task_idx in range(num_tasks):

      with tf.op_scope([features], name,

                       ('task' + str(task_idx).zfill(len(str(num_tasks))))):

        logits.append(

            Logits(features, num_classes, weight_init=weight_init,

                   bias_init=bias_init, dropout=dropout))

  return logits

        logits_list.append(

            logits(features, num_classes, weight_init=weight_init,

                   bias_init=bias_init, dropout_prob=dropout_prob))

  return logits_list

def Logits(features, num_classes=2, weight_init=None, bias_init=None,

           dropout=None, name=None):

def logits(features, num_classes=2, weight_init=None, bias_init=None,

           dropout_prob=None, name=None):

  """Create a logits tensor for a single classification task.

  You almost certainly don't want dropout on there -- it's like randomly setting

    num_classes: Number of classes for each task.

    weight_init: Weight initializer.

    bias_init: Bias initializer.

    dropout: Float giving dropout probability for weights (NOT keep

    dropout_prob: Float giving dropout probability for weights (NOT keep

      probability).

    name: Name for this op.

    A logits tensor with shape batch_size x num_classes.

  """

  with tf.op_scope([features], name, 'logits') as name:

    return Dropout(

        FullyConnectedLayer(features, num_classes, weight_init=weight_init,

    return dropout(

        fully_connected_layer(features, num_classes, weight_init=weight_init,

                              bias_init=bias_init, name=name),

        dropout)

        dropout_prob)

def SoftmaxN(tensor, name=None):

def softmax_N(tensor, name=None):

  """Apply softmax across last dimension of a tensor.

  Args:

    tensor: Input tensor.

    name: Name for this op. If None, defaults to 'SoftmaxN'.

    name: Name for this op. If None, defaults to 'softmax_N'.

  Returns:

    A tensor with softmax-normalized values on the last dimension.

  """

  with tf.op_scope([tensor], name, 'SoftmaxN'):

  with tf.op_scope([tensor], name, 'softmax_N'):

    exp_tensor = tf.exp(tensor)

    reduction_indices = [tensor.get_shape().ndims - 1]

    return tf.div(exp_tensor,

                                reduction_indices=reduction_indices,

                                keep_dims=True))

def Transform(tensor, transform, convolution=True, mask=None):

  """Apply a transform to a tensor.

  Args:

    tensor: Input tensor.

    transform: String description of transform. Supported values are 'bias'

      and 'batch_norm'.

    convolution: If True, assume tensor is the output of a convolution.

    mask: Mask to apply to tensor.

  Returns:

    A tensor with the same shape as the input tensor.

  Raises:

    ValueError: If the input tensor is not 3D or 4D.

  """

  if len(tensor.get_shape()) not in [2, 3, 4]:

    raise ValueError('Input tensor must be 2D, 3D or 4D, not %dD.'

                     % len(tensor.get_shape()))

  with tensor.graph.as_default():

    if transform == 'batch_norm':

      # batch normalization requires 4D input

      if len(tensor.get_shape()) != 4:

        # 3D case: add one extra dimension

        if len(tensor.get_shape()) == 3:

          squeeze = [2]

          tensor = tf.expand_dims(tensor, 2)

          if mask is not None:

            mask = tf.expand_dims(mask, -1)

        # 2D case: add two extra dimensions

        else:

          squeeze = [1, 2]

          tensor = tf.expand_dims(tf.expand_dims(tensor, -2), -2)

          if mask is not None:

            mask = tf.expand_dims(tf.expand_dims(mask, -1), -1)

        tensor = BatchNormalize(tensor, convolution=convolution, mask=mask)

        tensor = tf.squeeze(tensor, squeeze)

      else:

        tensor = BatchNormalize(tensor, convolution=convolution, mask=mask)

    elif transform == 'bias':

      tensor = AddBias(tensor, init=tf.constant(

          1.0, shape=[tensor.get_shape()[-1].value]))

      if mask is not None:

        tensor = model_utils.Mask(tensor, mask)

  return tensor

def Optimizer(optimizer="adam", learning_rate=.001, momentum=.9):

def optimizer(optimizer="adam", learning_rate=.001, momentum=.9):

  """Create model optimizer.

  Args:

import os

import numpy as np

import tensorflow as tf

from google.protobuf import text_format

from tensorflow.python.framework import ops

from tensorflow.python.framework import test_util

from tensorflow.python.platform import flags

from tensorflow.python.platform import gfile

from tensorflow.python.platform import googletest

from tensorflow.python.training import checkpoint_state_pb2 as cspb

from deepchem.models.tensorflow_models import model_ops

FLAGS = flags.FLAGS

FLAGS.test_random_seed = 20151102

class ModelOpsTest(test_util.TensorFlowTestCase):

class TestModelOps(test_util.TensorFlowTestCase):

  def setUp(self):

    super(ModelOpsTest, self).setUp()

    super(TestModelOps, self).setUp()

    self.root = '/tmp'

  def testAddBias(self):

  def test_add_bias(self):

    with self.test_session() as sess:

      w_t = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], shape=[2, 3])

      w_biased_t = model_ops.AddBias(w_t, init=tf.constant(5.0, shape=[3]))

      w_biased_t = model_ops.add_bias(w_t, init=tf.constant(5.0, shape=[3]))

      sess.run(tf.initialize_all_variables())

      w, w_biased, bias = sess.run([w_t, w_biased_t] + tf.trainable_variables())

      self.assertAllEqual(w, [[1.0, 2.0, 3.0],

                                     [9.0, 10.0, 11.0]])

      self.assertAllEqual(bias, [5.0, 5.0, 5.0])

  def testFullyConnectedLayer(self):

  def test_fully_connected_layer(self):

    with self.test_session() as sess:

      features = np.random.random((128, 100))

      features_t = tf.constant(features, dtype=tf.float32)

      dense_t = model_ops.FullyConnectedLayer(features_t, 50)

      dense_t = model_ops.fully_connected_layer(features_t, 50)

      sess.run(tf.initialize_all_variables())

      features, dense, w, b = sess.run(

          [features_t, dense_t] + tf.trainable_variables())

      expected = np.dot(features, w) + b

      self.assertAllClose(dense, expected)

  def testMultitaskLogits(self):

  def test_multitask_logits(self):

    with self.test_session() as sess:

      num_tasks = 3

      np.random.seed(FLAGS.test_random_seed)

      features = np.random.random((5, 100))

      logits_t = model_ops.MultitaskLogits(

      logits_t = model_ops.multitask_logits(

          tf.constant(features,

                      dtype=tf.float32),

          num_tasks)

        expected = np.dot(features, w[i]) + b[i]

        self.assertAllClose(logits[i], expected, rtol=1e-5, atol=1e-5)

  def GetModel(self, train=True):

    model_ops.set_training(train)

    # dummy variable for testing Restore

    tf.Variable(tf.constant(10.0, shape=[1]), name='v0')

  def _CheckBatchNormalization(self, features, convolution, mean, variance,

                               mask=None):

    model_ops.set_training(True)

    epsilon = 0.001

    with self.test_session() as sess:

      features_t = tf.constant(features, dtype=tf.float32)

      batch_norm_t = model_ops.BatchNormalize(

          features_t, convolution=convolution, epsilon=epsilon, mask=mask)

      sess.run(tf.initialize_all_variables())

      batch_norm, beta, gamma = sess.run(

          [batch_norm_t] + tf.trainable_variables())

      expected = gamma * (features - mean) / np.sqrt(variance + epsilon) + beta

      self.assertAllClose(batch_norm, np.ma.filled(expected, 0),

                          rtol=1e-5, atol=1e-5)

  def CheckBatchNormalization(self, features, convolution):

    if convolution:

      axis = (0, 1, 2)

    else:

      axis = 0

    mean = features.mean(axis=axis)

    variance = features.var(axis=axis)

    self._CheckBatchNormalization(features, convolution, mean, variance)

  def CheckBatchNormalizationWithMask(self, features, convolution, mask):

    # convert features to a masked array

    # masked array must be created with a mask of the same shape as features

    expanded_mask = np.logical_not(

        np.ones_like(features) * np.expand_dims(mask, -1))

    features = np.ma.array(features, mask=expanded_mask)

    if convolution:

      axis = (0, 1, 2)

      # masked arrays don't support mean/variance with tuple for axis

      count = np.logical_not(features.mask).sum(axis=axis)

      mean = features.sum(axis=axis) / count

      variance = np.square(features - mean).sum(axis=axis) / count

    else:

      axis = 0

      mean = features.mean(axis=axis)

      variance = features.var(axis=axis)

    mask_t = tf.constant(mask, dtype=tf.float32)

    self._CheckBatchNormalization(features, convolution, mean, variance,

                                  mask=mask_t)

  def testBatchNormalization(self):

    # no convolution: norm over batch (first axis)

    self.CheckBatchNormalization(

        features=np.random.random((2, 3, 2, 3)), convolution=False)

  def testBatchNormalizationWithConv(self):

    # convolution: norm over first three axes

    self.CheckBatchNormalization(

        features=np.random.random((2, 3, 2, 3)), convolution=True)

  def testBatchNormalizationInference(self):

    # create a simple batch-normalized model

    model_ops.set_training(True)

    epsilon = 0.001

    decay = 0.95

    checkpoint = os.path.join(self.root, 'my-checkpoint')

    with self.test_session() as sess:

      features = np.random.random((2, 3, 2, 3))

      features_t = tf.constant(features, dtype=tf.float32)

      # create variables for beta, gamma, and moving mean and variance

      model_ops.BatchNormalize(

          features_t, convolution=False, epsilon=epsilon, decay=decay)

      sess.run(tf.initialize_all_variables())

      updates = tf.group(*tf.get_default_graph().get_collection('updates'))

      sess.run(updates)  # update moving mean and variance

      expected_mean, expected_variance, _, _ = tf.all_variables()

      expected_mean = expected_mean.eval()

      expected_variance = expected_variance.eval()

      # save a checkpoint

      saver = tf.train.Saver()

      saver.save(sess, checkpoint)

    super(ModelOpsTest, self).setUp()  # reset the default graph

    # check that the moving mean and variance are used for evaluation

    # get a new set of features to verify that the correct mean and var are used

    model_ops.set_training(False)

    with self.test_session() as sess:

      new_features = np.random.random((2, 3, 2, 3))

      new_features_t = tf.constant(new_features, dtype=tf.float32)

      batch_norm_t = model_ops.BatchNormalize(

          new_features_t, convolution=False, epsilon=epsilon, decay=decay)

      saver = tf.train.Saver()

      saver.restore(sess, checkpoint)

      batch_norm, mean, variance, beta, gamma = sess.run(

          [batch_norm_t] + tf.all_variables())

      self.assertAllClose(mean, expected_mean)

      self.assertAllClose(variance, expected_variance)

      expected = (gamma * (new_features - mean) /

                  np.sqrt(variance + epsilon) + beta)

      self.assertAllClose(batch_norm, expected)

  def testBatchNormalizationWithMask(self):

    features = np.random.random((2, 3, 2, 3))

    mask = np.asarray(

        [[[1, 0],

          [1, 1],

          [1, 0]],

         [[0, 1],

          [0, 0],

          [0, 1]]],

        dtype=float)

    self.CheckBatchNormalizationWithMask(

        features=features, convolution=False, mask=mask)

  def testBatchNormalizationWithMaskAndConv(self):

    features = np.random.random((2, 3, 2, 3))