yapf

def main():

  env = TicTacToeEnvironment()

  policy = TicTacToePolicy()

  a3c = dc.rl.A3C(

      env, policy, entropy_weight=0, value_weight=0.25)

  a3c = dc.rl.A3C(env, policy, entropy_weight=0, value_weight=0.25)

  a3c.optimizer = dc.models.tensorgraph.TFWrapper(

      tf.train.AdamOptimizer, learning_rate=0.01)

  a3c.fit(100000)

    self.entropy_weight = entropy_weight

  def create_tensor(self, **kwargs):

    reward, action, prob, value, advantages = [layer.out_tensor for layer in self.in_layers]

    reward, action, prob, value, advantages = [

        layer.out_tensor for layer in self.in_layers

    ]

    log_prob = tf.log(prob + 0.0001)

    policy_loss = -tf.reduce_sum(

        advantages * tf.reduce_sum(action * log_prob))

    policy_loss = -tf.reduce_sum(advantages * tf.reduce_sum(action * log_prob))

    value_loss = tf.reduce_sum(tf.square(reward - value))

    entropy = -tf.reduce_sum(prob * log_prob)

    self.out_tensor = policy_loss + self.value_weight * value_loss - self.entropy_weight * entropy

    self.optimizer = TFWrapper(

        tf.train.AdamOptimizer, learning_rate=0.001, beta1=0.9, beta2=0.999)

    (self._graph, self._features, rewards, actions, self._action_prob,

     self._value, self._advantages) = self._build_graph(None, 'global', model_dir)

     self._value,

     self._advantages) = self._build_graph(None, 'global', model_dir)

    with self._graph._get_tf("Graph").as_default():

      self._session = tf.Session()

      session = self.a3c._session

      while step_count[0] < total_steps:

        session.run(self.update_local_variables)

        episode_states, episode_actions, episode_rewards, episode_advantages = self.create_rollout()

        episode_states, episode_actions, episode_rewards, episode_advantages = self.create_rollout(

        )

        feed_dict = {}

        for f, s in zip(self.features, episode_states):

          feed_dict[f.out_tensor] = s

        states[j].append(state[j])

      feed_dict = _create_feed_dict(self.features, state)

      probabilities, value = session.run(

          [self.action_prob.out_tensor, self.value.out_tensor], feed_dict=feed_dict)

          [self.action_prob.out_tensor, self.value.out_tensor],

          feed_dict=feed_dict)

      action = np.random.choice(np.arange(n_actions), p=probabilities[0])

      actions.append(np.zeros(n_actions))

      actions[i][action] = 1.0