Started untangling tf graphs from tf models.

    for i, (X, y, w, ids) in enumerate(self._itershards()):

      log("Iterating on shard-%s/epoch-%s" % (str(i+1), str(epoch+1)),

          self.verbosity)

      print("np.shape(X)")

      print(np.shape(X))

      print("np.shape(y)")

      print(np.shape(y))

      print("np.shape(w)")

      print(np.shape(w))

      nb_sample = np.shape(X)[0]

      interval_points = np.linspace(

          0, nb_sample, np.ceil(float(nb_sample)/batch_size)+1, dtype=int)

      print("interval_points")

      print(interval_points)

      for j in range(len(interval_points)-1):

        log("Iterating on batch-%s/shard-%s/epoch-%s" %

            (str(j+1), str(i+1), str(epoch+1)), self.verbosity)

    Due to rounding issues, some batches will not have exactly batch_size

    elements. Handle these batches by zero padding all arrays.

    """

    print("_pad_batch")

    print("np.shape(X_b)")

    print(np.shape(X_b))

    n, feature_shape = np.shape(X_b)[0], np.shape(X_b)[1:]

    _, num_tasks = np.shape(y_b)

    if n == batch_size:

class TensorflowMultiTaskClassifier(TensorflowClassifier):

  """Implements an icml model as configured in a model_config.proto."""

  def build(self):

  def build(self, tf_graph):

    """Constructs the graph architecture as specified in its config.

    This method creates the following Placeholders:

      mol_features: Molecule descriptor (e.g. fingerprint) tensor with shape

        batch_size x num_features.

    """

    assert len(self.model_params["data_shape"]) == 1

    num_features = self.model_params["data_shape"][0]

    with self.graph.as_default():

      with tf.name_scope(self.placeholder_scope):

        self.mol_features = tf.placeholder(

    assert len(tf_graph.model_params["data_shape"]) == 1

    num_features = tf_graph.model_params["data_shape"][0]

    with tf_graph.graph.as_default():

      with tf.name_scope(tf_graph.placeholder_scope):

        tf_graph.mol_features = tf.placeholder(

            tf.float32,

            shape=[self.model_params["batch_size"],

            shape=[tf_graph.model_params["batch_size"],

                   num_features],

            name='mol_features')

      layer_sizes = self.model_params["layer_sizes"]

      weight_init_stddevs = self.model_params["weight_init_stddevs"]

      bias_init_consts = self.model_params["bias_init_consts"]

      dropouts = self.model_params["dropouts"]

      layer_sizes = tf_graph.model_params["layer_sizes"]

      weight_init_stddevs = tf_graph.model_params["weight_init_stddevs"]

      bias_init_consts = tf_graph.model_params["bias_init_consts"]

      dropouts = tf_graph.model_params["dropouts"]

      lengths_set = {

          len(layer_sizes),

          len(weight_init_stddevs),

      num_layers = lengths_set.pop()

      assert num_layers > 0, 'Must have some layers defined.'

      prev_layer = self.mol_features

      prev_layer = tf_graph.mol_features

      prev_layer_size = num_features 

      for i in xrange(num_layers):

        layer = tf.nn.relu(model_ops.FullyConnectedLayer(

        prev_layer = layer

        prev_layer_size = layer_sizes[i]

      self.output = model_ops.MultitaskLogits(

          layer, self.model_params["num_classification_tasks"])

  # TODO(rbharath): Copying this out for now. Ensure this isn't harmful

  #def add_labels_and_weights(self):

  #  """Parse Label protos and create tensors for labels and weights.

  #  This method creates the following Placeholders in the graph:

  #    labels: Tensor with shape batch_size x num_tasks containing serialized

  #      Label protos.

  #  """

  #  config = self.config

  #  with tf.name_scope(self.placeholder_scope):

  #    labels = tf.placeholder(

  #        tf.string,

  #        shape=[config.batch_size, config.num_classification_tasks],

  #        name='labels')

  #  self.labels = label_ops.MultitaskLabelClasses(labels, config.num_classes)

  #  self.weights = label_ops.MultitaskLabelWeights(labels)

      tf_graph.output = model_ops.MultitaskLogits(

          layer, tf_graph.model_params["num_classification_tasks"])

  def construct_feed_dict(self, X_b, y_b=None, w_b=None, ids_b=None):

    """Construct a feed dictionary from minibatch data.

  """

  #traceback.print_stack(file=sys.stdout) 

  train = tf.get_collection("train")

  print("is_training()")

  print("train")

  print(train)

  if not train:

    raise ValueError('Training mode is not set. Please call set_training.')

  elif len(train) > 1:

    raise ValueError('Training mode has more than one setting.')

  return train[0]

def set_training(train):

  """Set the training mode of the default graph.

  This operation may only be called once for a given graph.

def WeightDecay(model_params):

  """Add weight decay.

  Args:

    graph: Tensorflow graph. 

    train: If True, graph is in training mode.

    model_params: dictionary.

  Returns:

    A scalar tensor containing the weight decay cost.

  Raises:

    AssertionError: If the default graph already has this value set.

    NotImplementedError: If an unsupported penalty type is requested.

  """

  if tf.get_collection('train'):

    raise AssertionError('Training mode already set: %s' %

                         graph.get_collection('train'))

  tf.add_to_collection('train', train)

  variables = []

  # exclude bias variables

  for v in tf.trainable_variables():

    if v.get_shape().ndims == 2:

      variables.append(v)

  with tf.name_scope('weight_decay'):

    if model_params["penalty_type"] == 'l1':

      cost = tf.add_n([tf.reduce_sum(tf.Abs(v)) for v in variables])

    elif model_params["penalty_type"] == 'l2':

      cost = tf.add_n([tf.nn.l2_loss(v) for v in variables])

    else:

      raise NotImplementedError('Unsupported penalty_type %s' %

                                model_params["penalty_type"])

    cost *= model_params["penalty"]

    tf.scalar_summary('Weight Decay Cost', cost)

  return cost

#def set_training(train):

#  """Set the training mode of the default graph.

#

#  This operation may only be called once for a given graph.

#

#  Args:

#    graph: Tensorflow graph. 

#    train: If True, graph is in training mode.

#

#  Raises:

#    AssertionError: If the default graph already has this value set.

#  """

#  if tf.get_collection('train'):

#    raise AssertionError('Training mode already set: %s' %

#                         graph.get_collection('train'))

#  tf.add_to_collection('train', train)

def MultitaskLogits(features, num_tasks, num_classes=2, weight_init=None,