Loading deepchem/models/tensorgraph/optimizers.py +58 −0 Original line number Diff line number Diff line Loading @@ -101,6 +101,29 @@ class GradientDescent(Optimizer): return tf.train.GradientDescentOptimizer(learning_rate=learning_rate) class PowerSign(Optimizer): """The powersign optimization algorithm""" def __init__(self, learning_rate=0.1, beta=0.9): """construct a powerSign optimizer Parameters ---------- learning_rate : float or LearningRateSchedule the learning rate to use for optimization beta : A float Parameter for the PowerSign algorithm """ self.learning_rate = learning_rate self.beta = beta def _create_optimizer(self, global_step): if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) else: learning_rate = self.learning_rate return tf.contrib.opt.PowerSignOptimizer( learning_rate=learning_rate, beta=self.beta) class ExponentialDecay(LearningRateSchedule): """A learning rate that decreases exponentially with the number of training steps.""" Loading Loading @@ -168,3 +191,38 @@ class PolynomialDecay(LearningRateSchedule): global_step=global_step, decay_steps=self.decay_steps, power=self.power) class LinearCosineDecay(LearningRateSchedule): """Applies linear cosine decay to the learning rate""" def __init__(self, initial_rate, decay_steps, alpha=0.0, beta=0.001, num_periods=0.5): """ Parameters ---------- learning_rate : float initial learning rate decay_steps : int number of steps to decay over num_periods : number of periods in the cosine part of the decay """ self.initial_rate = initial_rate self.decay_steps = decay_steps self.alpha = alpha self.beta = beta self.num_periods = num_periods def _create_tensor(self, global_step): return tf.train.linear_cosine_decay( learning_rate=self.initial_rate, global_step=global_step, decay_steps=self.decay_steps, alpha=self.alpha, beta=self.beta, num_periods=self.num_periods) deepchem/models/tensorgraph/tests/test_optimizers.py +16 −0 Original line number Diff line number Diff line Loading @@ -22,6 +22,14 @@ class TestLayers(test_util.TensorFlowTestCase): tfopt = opt._create_optimizer(global_step) assert isinstance(tfopt, tf.train.GradientDescentOptimizer) def test_power_sign(self): """Test creating a power sign optimizer.""" opt = optimizers.PowerSign(learning_rate=0.01) with self.test_session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) assert isinstance(tfopt, tf.contrib.opt.PowerSignOptimizer) def test_exponential_decay(self): """Test creating an optimizer with an exponentially decaying learning rate.""" rate = optimizers.ExponentialDecay( Loading @@ -39,3 +47,11 @@ class TestLayers(test_util.TensorFlowTestCase): with self.test_session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) def test_linearCosine_decay(self): """test creating an optimizer with a linear cosine decay to the learning rate""" rate = optimizers.LinearCosineDecay(initial_rate=0.1, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) with self.test_session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) Loading
deepchem/models/tensorgraph/optimizers.py +58 −0 Original line number Diff line number Diff line Loading @@ -101,6 +101,29 @@ class GradientDescent(Optimizer): return tf.train.GradientDescentOptimizer(learning_rate=learning_rate) class PowerSign(Optimizer): """The powersign optimization algorithm""" def __init__(self, learning_rate=0.1, beta=0.9): """construct a powerSign optimizer Parameters ---------- learning_rate : float or LearningRateSchedule the learning rate to use for optimization beta : A float Parameter for the PowerSign algorithm """ self.learning_rate = learning_rate self.beta = beta def _create_optimizer(self, global_step): if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) else: learning_rate = self.learning_rate return tf.contrib.opt.PowerSignOptimizer( learning_rate=learning_rate, beta=self.beta) class ExponentialDecay(LearningRateSchedule): """A learning rate that decreases exponentially with the number of training steps.""" Loading Loading @@ -168,3 +191,38 @@ class PolynomialDecay(LearningRateSchedule): global_step=global_step, decay_steps=self.decay_steps, power=self.power) class LinearCosineDecay(LearningRateSchedule): """Applies linear cosine decay to the learning rate""" def __init__(self, initial_rate, decay_steps, alpha=0.0, beta=0.001, num_periods=0.5): """ Parameters ---------- learning_rate : float initial learning rate decay_steps : int number of steps to decay over num_periods : number of periods in the cosine part of the decay """ self.initial_rate = initial_rate self.decay_steps = decay_steps self.alpha = alpha self.beta = beta self.num_periods = num_periods def _create_tensor(self, global_step): return tf.train.linear_cosine_decay( learning_rate=self.initial_rate, global_step=global_step, decay_steps=self.decay_steps, alpha=self.alpha, beta=self.beta, num_periods=self.num_periods)
deepchem/models/tensorgraph/tests/test_optimizers.py +16 −0 Original line number Diff line number Diff line Loading @@ -22,6 +22,14 @@ class TestLayers(test_util.TensorFlowTestCase): tfopt = opt._create_optimizer(global_step) assert isinstance(tfopt, tf.train.GradientDescentOptimizer) def test_power_sign(self): """Test creating a power sign optimizer.""" opt = optimizers.PowerSign(learning_rate=0.01) with self.test_session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) assert isinstance(tfopt, tf.contrib.opt.PowerSignOptimizer) def test_exponential_decay(self): """Test creating an optimizer with an exponentially decaying learning rate.""" rate = optimizers.ExponentialDecay( Loading @@ -39,3 +47,11 @@ class TestLayers(test_util.TensorFlowTestCase): with self.test_session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) def test_linearCosine_decay(self): """test creating an optimizer with a linear cosine decay to the learning rate""" rate = optimizers.LinearCosineDecay(initial_rate=0.1, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) with self.test_session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step)
