Merge pull request #697 from peastman/ppo

"""Interface for reinforcement learning."""

from deepchem.rl.a3c import A3C

from deepchem.rl.ppo import PPO

class Environment(object):

    ]

    prob = prob + np.finfo(np.float32).eps

    log_prob = tf.log(prob)

    policy_loss = -tf.reduce_mean(advantage * tf.reduce_sum(action * log_prob))

    policy_loss = -tf.reduce_mean(

        advantage * tf.reduce_sum(action * log_prob, axis=1))

    value_loss = tf.reduce_mean(tf.square(reward - value))

    entropy = -tf.reduce_mean(tf.reduce_sum(prob * log_prob, axis=1))

    self.out_tensor = policy_loss + self.value_weight * value_loss - self.entropy_weight * entropy

      the maximum length of rollouts to generate

    discount_factor: float

      the discount factor to use when computing rewards

    advantage_lambda: float

      the parameter for trading bias vs. variance in Generalized Advantage Estimation

    value_weight: float

      a scale factor for the value loss term in the loss function

    entropy_weight: float

        env,

        TestPolicy(),

        use_hindsight=True,

        entropy_weight=0.2,

        optimizer=dc.models.tensorgraph.TFWrapper(

            tf.train.AdamOptimizer, learning_rate=0.0005))

    a3c.fit(2000000)

    # Try running it a few times and see if it succeeds.

    pass_count = 0

    for i in range(5):

      env.reset()

      while not env.terminated:

        env.step(a3c.select_action(env.state))

      assert np.array_equal(env.state[:2], env.state[2:])

      if np.array_equal(env.state[:2], env.state[2:]):

        pass_count += 1

    assert pass_count >= 3