Copy layers, initialize variables

import random

import string

from collections import Sequence

from copy import deepcopy

import tensorflow as tf

import numpy as np

    self.in_layers = in_layers

    self.op_type = "gpu"

    self.variable_scope = ''

    self.variable_values = None

    self.rnn_initial_states = []

    self.rnn_final_states = []

    self.rnn_zero_states = []

    else:

      self.variable_scope = local_scope

  def set_variable_initial_values(self, values):

    """Set the initial values of all variables.

    This takes a list, which contains the initial values to use for all of

    this layer's values (in the same order retured by

    TensorGraph.get_layer_variables()).  When this layer is used in a

    TensorGraph, it will automatically initialize each variable to the value

    specified in the list.  Note that some layers also have separate mechanisms

    for specifying variable initializers; this method overrides them. The

    purpose of this method is to let a Layer object represent a pre-trained

    layer, complete with trained values for its variables."""

    self.variable_values = values

  def set_summary(self, summary_op, summary_description=None, collections=None):

    """Annotates a tensor with a tf.summary operation

    Collects data from self.out_tensor by default but can be changed by setting

    elif self.summary_op == 'histogram':

      tf.summary.histogram(self.name, self.tb_input, self.collections)

  def copy(self, replacements={}, variables_graph=None):

    """Duplicate this Layer and all its inputs.

    This creates and returns a clone of this layer.  It also recursively calls

    copy() on all of this layer's inputs to clone the entire hierarchy of layers.

    In the process, you can optionally tell it to replace particular layers with

    specific existing ones.  For example, you can clone a stack of layers, while

    connecting the topmost ones to different inputs.

    Parameters

    ----------

    replacements: map

      specifies existing layers, and the layers to replace them with (instead of

      cloning them).  This argument serves two purposes.  First, you can pass in

      a list of replacements to control which layers get cloned.  In addition,

      as each layer is cloned, it is added to this map.  On exit, it therefore

      contains a complete record of all layers that were copied, and a reference

      to the copy of each one.

    variables_graph: TensorGraph

      an optional TensorGraph from which to take variables.  If this is specified,

      the current value of each variable in each layer is recorded, and the copy

      has that value specified as its initial value.  This allows a piece of a

      pre-trained model to be copied to another model.

    """

    if self in replacements:

      return replacements[self]

    copied_inputs = [

        layer.copy(replacements, variables_graph) for layer in self.in_layers

    ]

    saved_inputs = self.in_layers

    self.in_layers = []

    saved_tensors = self.none_tensors()

    copy = deepcopy(self)

    self.in_layers = saved_inputs

    self.set_tensors(saved_tensors)

    copy.in_layers = copied_inputs

    if variables_graph is not None:

      variables = variables_graph.get_layer_variables(self)

      if len(variables) > 0:

        with variables_graph._get_tf("Graph").as_default():

          values = variables_graph.session.run(variables)

          copy.set_variable_initial_values(values)

    return copy

  def _as_graph_element(self):

    if '_as_graph_element' in dir(self.out_tensor):

      return self.out_tensor._as_graph_element()

      self.session.run(tf.global_variables_initializer())

      if restore:

        self.restore()

      else:

        # Initialize variables that have pre-trained values.

        for layer in self.layers.values():

          if layer.variable_values is not None:

            variables = self.get_layer_variables(layer)

            for var, val in zip(variables, layer.variable_values):

              self.session.run(var.assign(val))

      avg_loss, n_batches = 0.0, 0.0

      coord = tf.train.Coordinator()

      n_samples = 0

import deepchem as dc

from deepchem.data import NumpyDataset

from deepchem.data.datasets import Databag

from deepchem.models.tensorgraph.layers import Dense, SoftMaxCrossEntropy, ReduceMean, SoftMax, Constant

from deepchem.models.tensorgraph.layers import Dense, SoftMaxCrossEntropy, ReduceMean, SoftMax, Constant, Variable

from deepchem.models.tensorgraph.layers import Feature, Label

from deepchem.models.tensorgraph.layers import ReduceSquareDifference

from deepchem.models.tensorgraph.layers import ReduceSquareDifference, Add

from deepchem.models.tensorgraph.tensor_graph import TensorGraph

from deepchem.models.tensorgraph.optimizers import GradientDescent, ExponentialDecay

    for o, e in zip(tg.outputs, expected):

      value = tg.predict_on_batch(np.array([0]), outputs=o)

      assert np.array_equal(e, value)

  def test_initialize_variable(self):

    """Test methods for initializing a variable."""

    tg = dc.models.TensorGraph(use_queue=False)

    features = Feature(shape=(None, 1))

    tg.set_loss(Dense(1, in_layers=features))

    var = Variable([10.0])

    tg.add_output(var)

    tg.fit_generator([])

    assert tg.predict_on_batch(np.zeros((1, 1))) == [10.0]

    var.set_variable_initial_values([[15.0]])

    tg.fit_generator([])

    assert tg.predict_on_batch(np.zeros((1, 1))) == [15.0]

  def test_copy_layers(self):

    """Test copying layers."""

    tg = dc.models.TensorGraph()

    features = Feature(shape=(None, 10))

    dense = Dense(

        10, in_layers=features, biases_initializer=tf.random_normal_initializer)

    constant = Constant(10.0)

    output = dense + constant

    tg.add_output(output)

    tg.set_loss(output)

    tg.fit_generator([])

    replacements = {constant: Constant(20.0)}

    copy = output.copy(replacements, tg)

    assert isinstance(copy, Add)

    assert isinstance(copy.in_layers[0], Dense)

    assert isinstance(copy.in_layers[0].in_layers[0], Feature)

    assert copy.in_layers[1] == replacements[constant]

    variables = tg.get_layer_variables(dense)

    with tg._get_tf("Graph").as_default():

      values = tg.session.run(variables)

    for v1, v2 in zip(values, copy.in_layers[0].variable_values):

      assert np.array_equal(v1, v2)