Continuing to implement A3C algorithm

  layer_number_dict = {}

  def __init__(self, in_layers=None, **kwargs):

    if "name" not in kwargs:

      self.name = "%s_%s" % (self.__class__.__name__, self._get_layer_number())

    else:

    if "name" in kwargs:

      self.name = kwargs['name']

    else:

      self.name = None

    if "tensorboard" not in kwargs:

      self.tensorboard = False

    else:

    self.weights_initializer = weights_initializer

    self.time_series = time_series

    self.reuse = reuse

    if scope_name is None:

      scope_name = self.name

    self.scope_name = scope_name

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):

      biases_initializer = None

    else:

      biases_initializer = self.biases_initializer()

    if self.scope_name is None:

      scope_name = self.name

    else:

      scope_name = self.scope_name

    if not self.time_series:

      self.out_tensor = tf.contrib.layers.fully_connected(

          parent.out_tensor,

          activation_fn=self.activation_fn,

          biases_initializer=biases_initializer,

          weights_initializer=self.weights_initializer(),

          scope=self.scope_name,

          scope=scope_name,

          reuse=self.reuse,

          trainable=True)

      return self.out_tensor

               random_seed=None,

               use_queue=True,

               mode="regression",

               graph=None,

               **kwargs):

    """

    TODO(LESWING) allow a model to change its learning rate

      "regression" or "classification".  "classification" models on

      predict will do an argmax(axis=2) to determine the class of the

      prediction.

    graph: tensorflow.Graph

      the Graph in which to create Tensorflow objects.  If None, a new Graph

      is created.

    kwargs

    """

    # See TensorGraph._get_tf() for more details on lazy construction

    self.tensor_objects = {

        "FileWriter": None,

        "Graph": tf.Graph(),

        "Graph": graph,

        "train_op": None,

        "summary_op": None,

    }

    self.model_class = None

  def _add_layer(self, layer):

    if layer.name is None:

      layer.name = "%s_%s" % (layer.__class__.__name__, len(self.layers)+1)

    if layer.name in self.layers:

      return

    if isinstance(layer, Feature):

    self.nxgraph.add_node(layer.name)

    self.layers[layer.name] = layer

    for in_layer in layer.in_layers:

      self.nxgraph.add_edge(in_layer.name, layer.name)

      self._add_layer(in_layer)

      self.nxgraph.add_edge(in_layer.name, layer.name)

  def fit(self,

          dataset,

        tf.set_random_seed(self.random_seed)

      self._install_queue()

      order = self.topsort()

      print(order)

      for node in order:

        with tf.name_scope(node):

          node_layer = self.layers[node]

      self.tensor_objects['Graph'] = tf.Graph()

    elif obj == "FileWriter":

      self.tensor_objects['FileWriter'] = tf.summary.FileWriter(self.model_dir)

    elif obj == 'Optimizer':

      self.tensor_objects['Optimizer'] = self.optimizer()

    elif obj == 'train_op':

      self.tensor_objects['train_op'] = self.optimizer().minimize(

      self.tensor_objects['train_op'] = self._get_tf('Optimizer').minimize(

          self.loss.out_tensor)

    elif obj == 'summary_op':

      self.tensor_objects['summary_op'] = tf.summary.merge_all(

  is concerned, they are simply arbitrary objects.  The environment also computes

  a reward for each action, and reports when the task has been terminated

  (meaning that no more actions may be taken.

  Environment objects should be written to support pickle and deepcopy operations.

  Many algorithms involve creating multiple copies of the Environment, possibly

  running in different processes or even on different computers.

  """

  def __init__(self, state_shape, n_actions):

class GymEnvironment(Environment):

  """This is a convenient class for working with environments from OpenAI Gym."""

  """This is a convenience class for working with environments from OpenAI Gym."""

  def __init__(self, name):

    """Create an Environment wrapping the OpenAI Gym environment with a specified name."""

    import gym

    self.env = gym.make(name)

    self.name = name

    super().__init__(self.env.observation_space.shape, self.env.action_space.n)

  def reset(self):

    self._state, reward, self._terminated, info = self.env.step(action)

    return reward

  def __deepcopy__(self, memo):

    return GymEnvironment(self.name)

class Policy(object):

  """A policy for taking actions within an environment.

from deepchem.models.tensorgraph.layers import Feature, Weights, Label, Layer

import numpy as np

import tensorflow as tf

import copy

import multiprocessing

import threading

class A3CLoss(Layer):

  """This layer computes the loss function for A3C."""

  def __init__(self, value_weight, entropy_weight, **kwargs):

    super(A3CLoss, self).__init__(**kwargs)

    self.value_weight = value_weight

    log_prob = tf.log(prob)

    policy_loss = -tf.reduce_sum((reward-value)*tf.reduce_sum(action*log_prob))

    value_loss = tf.reduce_sum(tf.square(reward-value))

    entropy = tf.reduce_sum(prob*log_prob)

    entropy = -tf.reduce_sum(prob*log_prob)

    self.out_tensor = policy_loss + self.value_weight*value_loss - self.entropy_weight*entropy

    return self.out_tensor

  3. An entropy term to encourage exploration.

  """

  def __init__(self, env, policy, max_rollout_length, discount_factor=0.99, value_weight=0.25, entropy_weight=0.01):

  def __init__(self, env, policy, max_rollout_length=20, discount_factor=0.99, value_weight=0.25, entropy_weight=0.01):

    """Create an object for optimizing a policy.

    Parameters

    self.entropy_weight = entropy_weight

    self.optimizer = TFWrapper(tf.train.AdamOptimizer, learning_rate=0.001, beta1=0.9, beta2=0.999)

  def _build_graph(self):

  def _build_graph(self, tf_graph, scope):

    """Construct a TensorGraph containing the policy and loss calculations."""

    features = Feature(shape=[None]+list(self._env.state_shape))

    policy_layers = self._policy.create_layers(features)

    rewards = Weights(shape=(None, 1))

    actions = Label(shape=(None, self._env.n_actions))

    loss = A3CLoss(self.value_weight, self.entropy_weight, in_layers=[rewards, actions, action_prob, value])

    graph = TensorGraph(batch_size=self.max_rollout_length, use_queue=False)

    graph = TensorGraph(batch_size=self.max_rollout_length, use_queue=False, graph=tf_graph)

    graph.add_output(action_prob)

    graph.add_output(value)

    graph.set_loss(loss)

    graph.set_optimizer(self.optimizer)

    with graph._get_tf("Graph").as_default():

      with tf.variable_scope(scope):

        graph.build()

    return graph, features, rewards, actions, action_prob, value

  def _create_rollout(self, sess, features, action_prob):

  def fit(self, total_steps):

    (graph, features, rewards, actions, action_prob, value) = self._build_graph(None, 'global')

    with graph._get_tf("Graph").as_default():

      train_op = graph._get_tf('train_op')

      with tf.Session() as sess:

        sess.run(tf.global_variables_initializer())

        step_count = [0]

        workers = []

        threads = []

        for i in range(multiprocessing.cpu_count()):

          workers.append(Worker(self, graph, i))

        for worker in workers:

          thread = threading.Thread(name=worker.scope, target=lambda: worker.run(sess, step_count, total_steps))

          threads.append(thread)

          thread.start()

        for thread in threads:

          thread.join()

class Worker(object):

  def __init__(self, a3c, global_graph, index):

    self.a3c = a3c

    self.index = index

    self.scope = 'worker%d' % index

    self.env = copy.deepcopy(a3c._env)

    self.env.reset()

    self.graph, self.features, self.rewards, self.actions, self.action_prob, self.value = a3c._build_graph(global_graph._get_tf('Graph'), self.scope)

    local_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, self.scope)

    global_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'global')

    gradients = tf.gradients(self.graph.loss.out_tensor, local_vars)

    grads_and_vars = list(zip(gradients, global_vars))

    self.train_op = global_graph._get_tf('Optimizer').apply_gradients(grads_and_vars)

    self.update_local_variables = tf.group(*[tf.assign(v1, v2) for v1, v2 in zip(local_vars, global_vars)])

  def run(self, sess, step_count, total_steps):

    with self.graph._get_tf("Graph").as_default():

      while step_count[0] < total_steps:

        sess.run(self.update_local_variables)

        episode_states, episode_actions, episode_rewards = self.create_rollout(sess)

        feed_dict = {}

        feed_dict[self.features.out_tensor] = episode_states

        feed_dict[self.rewards.out_tensor] = episode_rewards

        feed_dict[self.actions.out_tensor] = episode_actions

        sess.run(self.train_op, feed_dict=feed_dict)

        step_count[0] += len(episode_states)

  def create_rollout(self, sess):

    """Generate a rollout."""

    n_actions = self._env.n_actions

    self._env.reset()

    n_actions = self.env.n_actions

    states = []

    actions = []

    scores = []

    for i in range(self.max_rollout_length):

      if self._env.terminated:

    for i in range(self.a3c.max_rollout_length):

      if self.env.terminated:

        break

      states.append(self._env.state)

      feed_dict = {features.out_tensor: np.expand_dims(self._env.state, axis=0)}

      probabilities = sess.run(action_prob.out_tensor, feed_dict=feed_dict)

      states.append(self.env.state)

      feed_dict = {self.features.out_tensor: np.expand_dims(self.env.state, axis=0)}

      probabilities = sess.run(self.action_prob.out_tensor, feed_dict=feed_dict)

      action = np.random.choice([0,1], p=probabilities[0])

      actions.append(np.zeros(n_actions))

      actions[i][action] = 1.0

      scores.append(self._env.step(action))

      scores.append(self.env.step(action))

    if not self.env.terminated:

      # Add an estimate of the reward for the rest of the episode.

      feed_dict = {self.features.out_tensor: np.expand_dims(self.env.state, axis=0)}

      scores[-1] += sess.run(self.value.out_tensor, feed_dict)

    for j in range(len(scores)-1, 0, -1):

      scores[j-1] += self.discount_factor*scores[j]

      scores[j-1] += self.a3c.discount_factor*scores[j]

    if self.env.terminated:

      self.env.reset()

    return np.array(states), np.array(actions), np.array(scores).reshape((len(scores), 1))

  def fit(self, n_episodes):

    (graph, features, rewards, actions, action_prob, value) = self._build_graph()

    with graph._get_tf("Graph").as_default():

      train_op = graph._get_tf('train_op')

      with tf.Session() as sess:

        sess.run(tf.global_variables_initializer())

        for i in range(n_episodes):

          episode_states, episode_actions, episode_rewards = self._create_rollout(sess, features, action_prob)

          feed_dict = {}

          feed_dict[features.out_tensor] = episode_states

          feed_dict[rewards.out_tensor] = episode_rewards

          feed_dict[actions.out_tensor] = episode_actions

          sess.run(train_op, feed_dict=feed_dict)