Loading deepchem/rl/__init__.py +14 −14 Original line number Diff line number Diff line Loading @@ -6,12 +6,12 @@ from deepchem.rl.a3c import A3C class Environment(object): """An environment in which an actor performs actions to accomplish a task. An environment has a current state, which is represented as a list of NumPy arrays. When an action is taken, that causes the state to be updated. Exactly what is meant by an "action" is defined by each subclass. As far as this interface is concerned, it is simply an arbitrary object. The environment also computes a reward for each action, and reports when the task has been terminated (meaning that no more actions may be taken). An environment has a current state, which is represented as either a single NumPy array, or optionally a list of NumPy arrays. When an action is taken, that causes the state to be updated. Exactly what is meant by an "action" is defined by each subclass. As far as this interface is concerned, it is simply an arbitrary object. The environment also computes a reward for each action, and reports when the task has been terminated (meaning that no more actions may be taken). Environment objects should be written to support pickle and deepcopy operations. Many algorithms involve creating multiple copies of the Environment, possibly Loading @@ -27,7 +27,7 @@ class Environment(object): @property def state(self): """The current state of the environment, represented as a list of NumPy arrays. """The current state of the environment, represented as either a NumPy array or list of arrays. If reset() has not yet been called at least once, this is undefined. """ Loading @@ -45,7 +45,9 @@ class Environment(object): def state_shape(self): """The shape of the arrays that describe a state. This returns a list of tuples, where each tuple is the shape of one array. If the state is a single array, this returns a tuple giving the shape of that array. If the state is a list of arrays, this returns a list of tuples where each tuple is the shape of one array. """ return self._state_shape Loading Loading @@ -74,7 +76,7 @@ class Environment(object): Returns ------- the reward earned by taking the action, represented as a float point number the reward earned by taking the action, represented as a floating point number (higher values are better) """ raise NotImplemented("Subclasses must implement this") Loading @@ -88,17 +90,15 @@ class GymEnvironment(Environment): import gym self.env = gym.make(name) self.name = name super(GymEnvironment, self).__init__([self.env.observation_space.shape], super(GymEnvironment, self).__init__(self.env.observation_space.shape, self.env.action_space.n) def reset(self): state = self.env.reset() self._state = [state] self._state = self.env.reset() self._terminated = False def step(self, action): state, reward, self._terminated, info = self.env.step(action) self._state = [state] self._state, reward, self._terminated, info = self.env.step(action) return reward def __deepcopy__(self, memo): Loading deepchem/rl/a3c.py +18 −1 Original line number Diff line number Diff line Loading @@ -5,6 +5,7 @@ from deepchem.models.tensorgraph import TFWrapper from deepchem.models.tensorgraph.layers import Feature, Weights, Label, Layer import numpy as np import tensorflow as tf import collections import copy import multiprocessing import os Loading Loading @@ -119,6 +120,7 @@ class A3C(object): self.value_weight = value_weight self.entropy_weight = entropy_weight self.use_hindsight = use_hindsight self._state_is_list = isinstance(env.state_shape[0], collections.Sequence) if optimizer is None: self._optimizer = TFWrapper( tf.train.AdamOptimizer, learning_rate=0.001, beta1=0.9, beta2=0.999) Loading @@ -133,7 +135,10 @@ class A3C(object): def _build_graph(self, tf_graph, scope, model_dir): """Construct a TensorGraph containing the policy and loss calculations.""" features = [Feature(shape=[None] + list(s)) for s in self._env.state_shape] state_shape = self._env.state_shape if not self._state_is_list: state_shape = [state_shape] features = [Feature(shape=[None] + list(s)) for s in state_shape] policy_layers = self._policy.create_layers(features) action_prob = policy_layers['action_prob'] value = policy_layers['value'] Loading Loading @@ -235,6 +240,8 @@ class A3C(object): ------- the array of action probabilities, and the estimated value function """ if not self._state_is_list: state = [state] with self._graph._get_tf("Graph").as_default(): feed_dict = self._create_feed_dict(state, use_saved_states) tensors = [self._action_prob.out_tensor, self._value.out_tensor] Loading Loading @@ -277,6 +284,8 @@ class A3C(object): ------- the index of the selected action """ if not self._state_is_list: state = [state] with self._graph._get_tf("Graph").as_default(): feed_dict = self._create_feed_dict(state, use_saved_states) tensors = [self._action_prob.out_tensor] Loading Loading @@ -424,10 +433,13 @@ class _Worker(object): # Rearrange the states into the proper set of arrays. if self.a3c._state_is_list: state_arrays = [[] for i in range(len(self.features))] for state in states: for j in range(len(state)): state_arrays[j].append(state[j]) else: state_arrays = [states] # Build the feed dict and apply gradients. Loading @@ -446,10 +458,13 @@ class _Worker(object): """Create a new rollout by applying hindsight to an existing one, then train the network.""" hindsight_states, rewards = self.env.apply_hindsight( states, actions, states[-1]) if self.a3c._state_is_list: state_arrays = [[] for i in range(len(self.features))] for state in hindsight_states: for j in range(len(state)): state_arrays[j].append(state[j]) else: state_arrays = [hindsight_states] feed_dict = {} for placeholder, value in zip(self.graph.rnn_initial_states, initial_rnn_states): Loading @@ -464,6 +479,8 @@ class _Worker(object): def create_feed_dict(self, state): """Create a feed dict for use during a rollout.""" if not self.a3c._state_is_list: state = [state] feed_dict = dict((f.out_tensor, np.expand_dims(s, axis=0)) for f, s in zip(self.features, state)) for (placeholder, value) in zip(self.graph.rnn_initial_states, Loading deepchem/rl/tests/test_a3c.py +12 −12 Original line number Diff line number Diff line Loading @@ -95,11 +95,11 @@ class TestA3C(unittest.TestCase): class TestEnvironment(dc.rl.Environment): def __init__(self): super(TestEnvironment, self).__init__([(10,)], 10) self._state = [np.random.random(10)] super(TestEnvironment, self).__init__((10,), 10) self._state = np.random.random(10) def step(self, action): self._state = [np.random.random(10)] self._state = np.random.random(10) return 0.0 def reset(self): Loading Loading @@ -156,18 +156,18 @@ class TestA3C(unittest.TestCase): class TestEnvironment(dc.rl.Environment): def __init__(self): super(TestEnvironment, self).__init__([(4,)], 4) super(TestEnvironment, self).__init__((4,), 4) self.moves = [(-1, 0), (1, 0), (0, -1), (0, 1)] def reset(self): self._state = [np.concatenate([[0, 0], np.random.randint(-50, 50, 2)])] self._state = np.concatenate([[0, 0], np.random.randint(-50, 50, 2)]) self._terminated = False self.count = 0 def step(self, action): new_state = self._state[0].copy() new_state = self._state.copy() new_state[:2] += self.moves[action] self._state = [new_state] self._state = new_state self.count += 1 reward = 0 if np.array_equal(new_state[:2], new_state[2:]): Loading @@ -180,11 +180,11 @@ class TestA3C(unittest.TestCase): def apply_hindsight(self, states, actions, goal): new_states = [] rewards = [] goal_pos = goal[0][:2] goal_pos = goal[:2] for state, action in zip(states, actions): new_state = state[0].copy() new_state = state.copy() new_state[2:] = goal_pos new_states.append([new_state]) new_states.append(new_state) pos_after_action = new_state[:2] + self.moves[action] if np.array_equal(pos_after_action, goal_pos): rewards.append(1) Loading Loading @@ -215,7 +215,7 @@ class TestA3C(unittest.TestCase): env, TestPolicy(), use_hindsight=True, entropy_weight=0.1, entropy_weight=0.2, optimizer=dc.models.tensorgraph.TFWrapper( tf.train.AdamOptimizer, learning_rate=0.0005)) a3c.fit(2000000) Loading @@ -226,4 +226,4 @@ class TestA3C(unittest.TestCase): env.reset() while not env.terminated: env.step(a3c.select_action(env.state)) assert np.array_equal(env.state[0][:2], env.state[0][2:]) assert np.array_equal(env.state[:2], env.state[2:]) Loading
deepchem/rl/__init__.py +14 −14 Original line number Diff line number Diff line Loading @@ -6,12 +6,12 @@ from deepchem.rl.a3c import A3C class Environment(object): """An environment in which an actor performs actions to accomplish a task. An environment has a current state, which is represented as a list of NumPy arrays. When an action is taken, that causes the state to be updated. Exactly what is meant by an "action" is defined by each subclass. As far as this interface is concerned, it is simply an arbitrary object. The environment also computes a reward for each action, and reports when the task has been terminated (meaning that no more actions may be taken). An environment has a current state, which is represented as either a single NumPy array, or optionally a list of NumPy arrays. When an action is taken, that causes the state to be updated. Exactly what is meant by an "action" is defined by each subclass. As far as this interface is concerned, it is simply an arbitrary object. The environment also computes a reward for each action, and reports when the task has been terminated (meaning that no more actions may be taken). Environment objects should be written to support pickle and deepcopy operations. Many algorithms involve creating multiple copies of the Environment, possibly Loading @@ -27,7 +27,7 @@ class Environment(object): @property def state(self): """The current state of the environment, represented as a list of NumPy arrays. """The current state of the environment, represented as either a NumPy array or list of arrays. If reset() has not yet been called at least once, this is undefined. """ Loading @@ -45,7 +45,9 @@ class Environment(object): def state_shape(self): """The shape of the arrays that describe a state. This returns a list of tuples, where each tuple is the shape of one array. If the state is a single array, this returns a tuple giving the shape of that array. If the state is a list of arrays, this returns a list of tuples where each tuple is the shape of one array. """ return self._state_shape Loading Loading @@ -74,7 +76,7 @@ class Environment(object): Returns ------- the reward earned by taking the action, represented as a float point number the reward earned by taking the action, represented as a floating point number (higher values are better) """ raise NotImplemented("Subclasses must implement this") Loading @@ -88,17 +90,15 @@ class GymEnvironment(Environment): import gym self.env = gym.make(name) self.name = name super(GymEnvironment, self).__init__([self.env.observation_space.shape], super(GymEnvironment, self).__init__(self.env.observation_space.shape, self.env.action_space.n) def reset(self): state = self.env.reset() self._state = [state] self._state = self.env.reset() self._terminated = False def step(self, action): state, reward, self._terminated, info = self.env.step(action) self._state = [state] self._state, reward, self._terminated, info = self.env.step(action) return reward def __deepcopy__(self, memo): Loading
deepchem/rl/a3c.py +18 −1 Original line number Diff line number Diff line Loading @@ -5,6 +5,7 @@ from deepchem.models.tensorgraph import TFWrapper from deepchem.models.tensorgraph.layers import Feature, Weights, Label, Layer import numpy as np import tensorflow as tf import collections import copy import multiprocessing import os Loading Loading @@ -119,6 +120,7 @@ class A3C(object): self.value_weight = value_weight self.entropy_weight = entropy_weight self.use_hindsight = use_hindsight self._state_is_list = isinstance(env.state_shape[0], collections.Sequence) if optimizer is None: self._optimizer = TFWrapper( tf.train.AdamOptimizer, learning_rate=0.001, beta1=0.9, beta2=0.999) Loading @@ -133,7 +135,10 @@ class A3C(object): def _build_graph(self, tf_graph, scope, model_dir): """Construct a TensorGraph containing the policy and loss calculations.""" features = [Feature(shape=[None] + list(s)) for s in self._env.state_shape] state_shape = self._env.state_shape if not self._state_is_list: state_shape = [state_shape] features = [Feature(shape=[None] + list(s)) for s in state_shape] policy_layers = self._policy.create_layers(features) action_prob = policy_layers['action_prob'] value = policy_layers['value'] Loading Loading @@ -235,6 +240,8 @@ class A3C(object): ------- the array of action probabilities, and the estimated value function """ if not self._state_is_list: state = [state] with self._graph._get_tf("Graph").as_default(): feed_dict = self._create_feed_dict(state, use_saved_states) tensors = [self._action_prob.out_tensor, self._value.out_tensor] Loading Loading @@ -277,6 +284,8 @@ class A3C(object): ------- the index of the selected action """ if not self._state_is_list: state = [state] with self._graph._get_tf("Graph").as_default(): feed_dict = self._create_feed_dict(state, use_saved_states) tensors = [self._action_prob.out_tensor] Loading Loading @@ -424,10 +433,13 @@ class _Worker(object): # Rearrange the states into the proper set of arrays. if self.a3c._state_is_list: state_arrays = [[] for i in range(len(self.features))] for state in states: for j in range(len(state)): state_arrays[j].append(state[j]) else: state_arrays = [states] # Build the feed dict and apply gradients. Loading @@ -446,10 +458,13 @@ class _Worker(object): """Create a new rollout by applying hindsight to an existing one, then train the network.""" hindsight_states, rewards = self.env.apply_hindsight( states, actions, states[-1]) if self.a3c._state_is_list: state_arrays = [[] for i in range(len(self.features))] for state in hindsight_states: for j in range(len(state)): state_arrays[j].append(state[j]) else: state_arrays = [hindsight_states] feed_dict = {} for placeholder, value in zip(self.graph.rnn_initial_states, initial_rnn_states): Loading @@ -464,6 +479,8 @@ class _Worker(object): def create_feed_dict(self, state): """Create a feed dict for use during a rollout.""" if not self.a3c._state_is_list: state = [state] feed_dict = dict((f.out_tensor, np.expand_dims(s, axis=0)) for f, s in zip(self.features, state)) for (placeholder, value) in zip(self.graph.rnn_initial_states, Loading
deepchem/rl/tests/test_a3c.py +12 −12 Original line number Diff line number Diff line Loading @@ -95,11 +95,11 @@ class TestA3C(unittest.TestCase): class TestEnvironment(dc.rl.Environment): def __init__(self): super(TestEnvironment, self).__init__([(10,)], 10) self._state = [np.random.random(10)] super(TestEnvironment, self).__init__((10,), 10) self._state = np.random.random(10) def step(self, action): self._state = [np.random.random(10)] self._state = np.random.random(10) return 0.0 def reset(self): Loading Loading @@ -156,18 +156,18 @@ class TestA3C(unittest.TestCase): class TestEnvironment(dc.rl.Environment): def __init__(self): super(TestEnvironment, self).__init__([(4,)], 4) super(TestEnvironment, self).__init__((4,), 4) self.moves = [(-1, 0), (1, 0), (0, -1), (0, 1)] def reset(self): self._state = [np.concatenate([[0, 0], np.random.randint(-50, 50, 2)])] self._state = np.concatenate([[0, 0], np.random.randint(-50, 50, 2)]) self._terminated = False self.count = 0 def step(self, action): new_state = self._state[0].copy() new_state = self._state.copy() new_state[:2] += self.moves[action] self._state = [new_state] self._state = new_state self.count += 1 reward = 0 if np.array_equal(new_state[:2], new_state[2:]): Loading @@ -180,11 +180,11 @@ class TestA3C(unittest.TestCase): def apply_hindsight(self, states, actions, goal): new_states = [] rewards = [] goal_pos = goal[0][:2] goal_pos = goal[:2] for state, action in zip(states, actions): new_state = state[0].copy() new_state = state.copy() new_state[2:] = goal_pos new_states.append([new_state]) new_states.append(new_state) pos_after_action = new_state[:2] + self.moves[action] if np.array_equal(pos_after_action, goal_pos): rewards.append(1) Loading Loading @@ -215,7 +215,7 @@ class TestA3C(unittest.TestCase): env, TestPolicy(), use_hindsight=True, entropy_weight=0.1, entropy_weight=0.2, optimizer=dc.models.tensorgraph.TFWrapper( tf.train.AdamOptimizer, learning_rate=0.0005)) a3c.fit(2000000) Loading @@ -226,4 +226,4 @@ class TestA3C(unittest.TestCase): env.reset() while not env.terminated: env.step(a3c.select_action(env.state)) assert np.array_equal(env.state[0][:2], env.state[0][2:]) assert np.array_equal(env.state[:2], env.state[2:])
