Loading deepchem/metalearning/maml.py +2 −2 Original line number Diff line number Diff line Loading @@ -131,9 +131,9 @@ class MAML(object): # Create the optimizers for meta-optimization and task optimization. self._global_step = tf.Variable(0, trainable=False) self._tf_optimizer = optimizer._create_optimizer(self._global_step) self._tf_optimizer = optimizer._create_tf_optimizer(self._global_step) task_optimizer = GradientDescent(learning_rate=self.learning_rate) self._tf_task_optimizer = task_optimizer._create_optimizer( self._tf_task_optimizer = task_optimizer._create_tf_optimizer( self._global_step) # Create a Checkpoint for saving. Loading deepchem/models/keras_model.py +1 −1 Original line number Diff line number Diff line Loading @@ -227,7 +227,7 @@ class KerasModel(Model): return self._built = True self._global_step = tf.Variable(0, trainable=False) self._tf_optimizer = self.optimizer._create_optimizer(self._global_step) self._tf_optimizer = self.optimizer._create_tf_optimizer(self._global_step) self._checkpoint = tf.train.Checkpoint( optimizer=self._tf_optimizer, model=self.model) Loading deepchem/models/optimizers.py +148 −33 Original line number Diff line number Diff line """Optimizers and related classes for use with TensorGraph.""" import tensorflow as tf import math from typing import Union class Optimizer(object): """An algorithm for optimizing a TensorGraph based model. """An algorithm for optimizing a model. This is an abstract class. Subclasses represent specific optimization algorithms. """ def _create_optimizer(self, global_step): """Construct the TensorFlow optimizer. def _create_tf_optimizer(self, global_step): """Construct a TensorFlow optimizer. Parameters ---------- Loading @@ -23,6 +25,20 @@ class Optimizer(object): """ raise NotImplemented("Subclasses must implement this") def _create_pytorch_optimizer(self, params): """Construct a PyTorch optimizer. Parameters ---------- params: Iterable the model parameters to optimize Returns ------- a new PyTorch optimizer implementing the algorithm """ raise NotImplemented("Subclasses must implement this") class LearningRateSchedule(object): """A schedule for changing the learning rate over the course of optimization. Loading @@ -30,7 +46,7 @@ class LearningRateSchedule(object): This is an abstract class. Subclasses represent specific schedules. """ def _create_tensor(self, global_step): def _create_tf_tensor(self, global_step): """Construct a tensor that equals the learning rate. Parameters Loading @@ -44,6 +60,20 @@ class LearningRateSchedule(object): """ raise NotImplemented("Subclasses must implement this") def _create_pytorch_schedule(self, optimizer): """Construct a PyTorch learning rate scheduler. Parameters ---------- optimizer: torch.optim.Optimizer the Optimizer whose learning rate will be modified Returns ------- a PyTorch scheduler implementing the schedule """ raise NotImplemented("Subclasses must implement this") class AdaGrad(Optimizer): """The AdaGrad optimization algorithm. Loading @@ -61,9 +91,9 @@ learning research 12.7 (2011). """ def __init__(self, learning_rate=0.001, initial_accumulator_value=0.1, epsilon=1e-07): learning_rate: Union[float, LearningRateSchedule] = 0.001, initial_accumulator_value: float = 0.1, epsilon: float = 1e-07): """Construct an AdaGrad optimizer. Parameters ---------- Loading @@ -79,9 +109,10 @@ learning research 12.7 (2011). self.initial_accumulator_value = initial_accumulator_value self.epsilon = epsilon def _create_optimizer(self, global_step): def _create_tf_optimizer(self, global_step): import tensorflow as tf if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) learning_rate = self.learning_rate._create_tf_tensor(global_step) else: learning_rate = self.learning_rate return tf.keras.optimizers.Adagrad( Loading @@ -89,12 +120,27 @@ learning research 12.7 (2011). initial_accumulator_value=self.initial_accumulator_value, epsilon=self.epsilon) def _create_pytorch_optimizer(self, params): import torch if isinstance(self.learning_rate, LearningRateSchedule): lr = self.learning_rate.initial_rate else: lr = self.learning_rate return torch.optim.Adagrad( params, lr, initial_accumulator_value=self.initial_accumulator_value, eps=self.epsilon) class Adam(Optimizer): """The Adam optimization algorithm.""" def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08): def __init__(self, learning_rate: Union[float, LearningRateSchedule] = 0.001, beta1: float = 0.9, beta2: float = 0.999, epsilon: float = 1e-08): """Construct an Adam optimizer. Parameters Loading @@ -113,9 +159,10 @@ class Adam(Optimizer): self.beta2 = beta2 self.epsilon = epsilon def _create_optimizer(self, global_step): def _create_tf_optimizer(self, global_step): import tensorflow as tf if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) learning_rate = self.learning_rate._create_tf_tensor(global_step) else: learning_rate = self.learning_rate return tf.keras.optimizers.Adam( Loading @@ -124,15 +171,23 @@ class Adam(Optimizer): beta_2=self.beta2, epsilon=self.epsilon) def _create_pytorch_optimizer(self, params): import torch if isinstance(self.learning_rate, LearningRateSchedule): lr = self.learning_rate.initial_rate else: lr = self.learning_rate return torch.optim.Adam(params, lr, (self.beta1, self.beta2), self.epsilon) class RMSProp(Optimizer): """RMSProp Optimization algorithm.""" def __init__(self, learning_rate=0.001, momentum=0.0, decay=0.9, epsilon=1e-10): learning_rate: Union[float, LearningRateSchedule] = 0.001, momentum: float = 0.0, decay: float = 0.9, epsilon: float = 1e-10): """Construct an RMSProp Optimizer. Parameters Loading @@ -151,9 +206,10 @@ class RMSProp(Optimizer): self.decay = decay self.epsilon = epsilon def _create_optimizer(self, global_step): def _create_tf_optimizer(self, global_step): import tensorflow as tf if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) learning_rate = self.learning_rate._create_tf_tensor(global_step) else: learning_rate = self.learning_rate return tf.keras.optimizers.RMSprop( Loading @@ -162,11 +218,20 @@ class RMSProp(Optimizer): rho=self.decay, epsilon=self.epsilon) def _create_pytorch_optimizer(self, params): import torch if isinstance(self.learning_rate, LearningRateSchedule): lr = self.learning_rate.initial_rate else: lr = self.learning_rate return torch.optim.RMSprop( params, lr, alpha=self.decay, eps=self.epsilon, momentum=self.momentum) class GradientDescent(Optimizer): """The gradient descent optimization algorithm.""" def __init__(self, learning_rate=0.001): def __init__(self, learning_rate: Union[float, LearningRateSchedule] = 0.001): """Construct a gradient descent optimizer. Parameters Loading @@ -176,18 +241,31 @@ class GradientDescent(Optimizer): """ self.learning_rate = learning_rate def _create_optimizer(self, global_step): def _create_tf_optimizer(self, global_step): import tensorflow as tf if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) learning_rate = self.learning_rate._create_tf_tensor(global_step) else: learning_rate = self.learning_rate return tf.keras.optimizers.SGD(learning_rate=learning_rate) def _create_pytorch_optimizer(self, params): import torch if isinstance(self.learning_rate, LearningRateSchedule): lr = self.learning_rate.initial_rate else: lr = self.learning_rate return torch.optim.SGD(params, lr) class ExponentialDecay(LearningRateSchedule): """A learning rate that decreases exponentially with the number of training steps.""" def __init__(self, initial_rate, decay_rate, decay_steps, staircase=True): def __init__(self, initial_rate: float, decay_rate: float, decay_steps: int, staircase: bool = True): """Create an exponentially decaying learning rate. The learning rate starts as initial_rate. Every decay_steps training steps, it is multiplied by decay_rate. Loading @@ -209,18 +287,31 @@ class ExponentialDecay(LearningRateSchedule): self.decay_steps = decay_steps self.staircase = staircase def _create_tensor(self, global_step): def _create_tf_tensor(self, global_step): import tensorflow as tf return tf.keras.optimizers.schedules.ExponentialDecay( initial_learning_rate=self.initial_rate, decay_rate=self.decay_rate, decay_steps=self.decay_steps, staircase=self.staircase)(global_step) def _create_pytorch_schedule(self, optimizer): import torch if self.staircase: return torch.optim.lr_scheduler.StepLR(optimizer, self.decay_steps, self.decay_rate) return torch.optim.lr_scheduler.ExponentialLR( optimizer, math.pow(self.decay_rate, 1 / self.decay_steps)) class PolynomialDecay(LearningRateSchedule): """A learning rate that decreases from an initial value to a final value over a fixed number of training steps.""" def __init__(self, initial_rate, final_rate, decay_steps, power=1.0): def __init__(self, initial_rate: float, final_rate: float, decay_steps: int, power: float = 1.0): """Create a smoothly decaying learning rate. The learning rate starts as initial_rate. It smoothly decreases to final_rate over decay_steps training steps. Loading @@ -243,23 +334,34 @@ class PolynomialDecay(LearningRateSchedule): self.decay_steps = decay_steps self.power = power def _create_tensor(self, global_step): def _create_tf_tensor(self, global_step): import tensorflow as tf return tf.keras.optimizers.schedules.PolynomialDecay( initial_learning_rate=self.initial_rate, end_learning_rate=self.final_rate, decay_steps=self.decay_steps, power=self.power)(global_step) def _create_pytorch_schedule(self, optimizer): def f(step): t = min(step, self.decay_steps) / self.decay_steps return ((self.initial_rate - self.final_rate) * (1 - t)**self.power) + self.final_rate import torch return torch.optim.lr_scheduler.LambdaLR(optimizer, f) class LinearCosineDecay(LearningRateSchedule): """Applies linear cosine decay to the learning rate""" def __init__(self, initial_rate, decay_steps, alpha=0.0, beta=0.001, num_periods=0.5): initial_rate: float, decay_steps: int, alpha: float = 0.0, beta: float = 0.001, num_periods: float = 0.5): """ Parameters ---------- Loading @@ -276,7 +378,8 @@ class LinearCosineDecay(LearningRateSchedule): self.beta = beta self.num_periods = num_periods def _create_tensor(self, global_step): def _create_tf_tensor(self, global_step): import tensorflow as tf return tf.compat.v1.train.linear_cosine_decay( learning_rate=self.initial_rate, global_step=global_step, Loading @@ -284,3 +387,15 @@ class LinearCosineDecay(LearningRateSchedule): alpha=self.alpha, beta=self.beta, num_periods=self.num_periods) def _create_pytorch_schedule(self, optimizer): def f(step): t = min(step, self.decay_steps) / self.decay_steps linear_decay = 1 - t cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * self.num_periods * t)) decayed = (self.alpha + linear_decay) * cosine_decay + self.beta return self.initial_rate * decayed import torch return torch.optim.lr_scheduler.LambdaLR(optimizer, f) deepchem/models/tests/test_optimizers.py +110 −30 Original line number Diff line number Diff line import deepchem.models.optimizers as optimizers import unittest try: import tensorflow as tf from tensorflow.python.framework import test_util has_tensorflow = True except: has_tensorflow = False try: import torch has_pytorch = True except: has_pytorch = False class TestLayers(test_util.TensorFlowTestCase): class TestLayers(unittest.TestCase): """Test optimizers and related classes.""" def test_adam(self): @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_adam_tf(self): """Test creating an Adam optimizer.""" opt = optimizers.Adam(learning_rate=0.01) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) assert isinstance(tfopt, tf.keras.optimizers.Adam) def test_rmsprop(self): @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_adam_pytorch(self): """Test creating an Adam optimizer.""" opt = optimizers.Adam(learning_rate=0.01) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) assert isinstance(torchopt, torch.optim.Adam) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_adagrad_tf(self): """Test creating an AdaGrad optimizer.""" opt = optimizers.AdaGrad(learning_rate=0.01) global_step = tf.Variable(0) tfopt = opt._create_tf_optimizer(global_step) assert isinstance(tfopt, tf.keras.optimizers.Adagrad) @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_adagrad_pytorch(self): """Test creating an AdaGrad optimizer.""" opt = optimizers.AdaGrad(learning_rate=0.01) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) assert isinstance(torchopt, torch.optim.Adagrad) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_rmsprop_tf(self): """Test creating an RMSProp Optimizer.""" opt = optimizers.RMSProp(learning_rate=0.01) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) assert isinstance(tfopt, tf.keras.optimizers.RMSprop) def test_gradient_descent(self): @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_rmsprop_pytorch(self): """Test creating an RMSProp Optimizer.""" opt = optimizers.RMSProp(learning_rate=0.01) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) assert isinstance(torchopt, torch.optim.RMSprop) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_gradient_descent_tf(self): """Test creating a Gradient Descent optimizer.""" opt = optimizers.GradientDescent(learning_rate=0.01) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) assert isinstance(tfopt, tf.keras.optimizers.SGD) def test_exponential_decay(self): @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_gradient_descent_pytorch(self): """Test creating a Gradient Descent optimizer.""" opt = optimizers.GradientDescent(learning_rate=0.01) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) assert isinstance(torchopt, torch.optim.SGD) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_exponential_decay_tf(self): """Test creating an optimizer with an exponentially decaying learning rate.""" rate = optimizers.ExponentialDecay( initial_rate=0.001, decay_rate=0.99, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) def test_polynomial_decay(self): @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_exponential_decay_pytorch(self): """Test creating an optimizer with an exponentially decaying learning rate.""" rate = optimizers.ExponentialDecay( initial_rate=0.001, decay_rate=0.99, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) schedule = rate._create_pytorch_schedule(torchopt) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_polynomial_decay_tf(self): """Test creating an optimizer with a polynomially decaying learning rate.""" rate = optimizers.PolynomialDecay( initial_rate=0.001, final_rate=0.0001, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_polynomial_decay_pytorch(self): """Test creating an optimizer with a polynomially decaying learning rate.""" rate = optimizers.PolynomialDecay( initial_rate=0.001, final_rate=0.0001, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) schedule = rate._create_pytorch_schedule(torchopt) def test_linearCosine_decay(self): @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_linearCosine_decay_tf(self): """test creating an optimizer with a linear cosine decay to the learning rate""" rate = optimizers.LinearCosineDecay(initial_rate=0.1, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_linearCosine_decay_pytorch(self): """test creating an optimizer with a linear cosine decay to the learning rate""" rate = optimizers.LinearCosineDecay(initial_rate=0.1, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) schedule = rate._create_pytorch_schedule(torchopt) Loading
deepchem/metalearning/maml.py +2 −2 Original line number Diff line number Diff line Loading @@ -131,9 +131,9 @@ class MAML(object): # Create the optimizers for meta-optimization and task optimization. self._global_step = tf.Variable(0, trainable=False) self._tf_optimizer = optimizer._create_optimizer(self._global_step) self._tf_optimizer = optimizer._create_tf_optimizer(self._global_step) task_optimizer = GradientDescent(learning_rate=self.learning_rate) self._tf_task_optimizer = task_optimizer._create_optimizer( self._tf_task_optimizer = task_optimizer._create_tf_optimizer( self._global_step) # Create a Checkpoint for saving. Loading
deepchem/models/keras_model.py +1 −1 Original line number Diff line number Diff line Loading @@ -227,7 +227,7 @@ class KerasModel(Model): return self._built = True self._global_step = tf.Variable(0, trainable=False) self._tf_optimizer = self.optimizer._create_optimizer(self._global_step) self._tf_optimizer = self.optimizer._create_tf_optimizer(self._global_step) self._checkpoint = tf.train.Checkpoint( optimizer=self._tf_optimizer, model=self.model) Loading
deepchem/models/optimizers.py +148 −33 Original line number Diff line number Diff line """Optimizers and related classes for use with TensorGraph.""" import tensorflow as tf import math from typing import Union class Optimizer(object): """An algorithm for optimizing a TensorGraph based model. """An algorithm for optimizing a model. This is an abstract class. Subclasses represent specific optimization algorithms. """ def _create_optimizer(self, global_step): """Construct the TensorFlow optimizer. def _create_tf_optimizer(self, global_step): """Construct a TensorFlow optimizer. Parameters ---------- Loading @@ -23,6 +25,20 @@ class Optimizer(object): """ raise NotImplemented("Subclasses must implement this") def _create_pytorch_optimizer(self, params): """Construct a PyTorch optimizer. Parameters ---------- params: Iterable the model parameters to optimize Returns ------- a new PyTorch optimizer implementing the algorithm """ raise NotImplemented("Subclasses must implement this") class LearningRateSchedule(object): """A schedule for changing the learning rate over the course of optimization. Loading @@ -30,7 +46,7 @@ class LearningRateSchedule(object): This is an abstract class. Subclasses represent specific schedules. """ def _create_tensor(self, global_step): def _create_tf_tensor(self, global_step): """Construct a tensor that equals the learning rate. Parameters Loading @@ -44,6 +60,20 @@ class LearningRateSchedule(object): """ raise NotImplemented("Subclasses must implement this") def _create_pytorch_schedule(self, optimizer): """Construct a PyTorch learning rate scheduler. Parameters ---------- optimizer: torch.optim.Optimizer the Optimizer whose learning rate will be modified Returns ------- a PyTorch scheduler implementing the schedule """ raise NotImplemented("Subclasses must implement this") class AdaGrad(Optimizer): """The AdaGrad optimization algorithm. Loading @@ -61,9 +91,9 @@ learning research 12.7 (2011). """ def __init__(self, learning_rate=0.001, initial_accumulator_value=0.1, epsilon=1e-07): learning_rate: Union[float, LearningRateSchedule] = 0.001, initial_accumulator_value: float = 0.1, epsilon: float = 1e-07): """Construct an AdaGrad optimizer. Parameters ---------- Loading @@ -79,9 +109,10 @@ learning research 12.7 (2011). self.initial_accumulator_value = initial_accumulator_value self.epsilon = epsilon def _create_optimizer(self, global_step): def _create_tf_optimizer(self, global_step): import tensorflow as tf if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) learning_rate = self.learning_rate._create_tf_tensor(global_step) else: learning_rate = self.learning_rate return tf.keras.optimizers.Adagrad( Loading @@ -89,12 +120,27 @@ learning research 12.7 (2011). initial_accumulator_value=self.initial_accumulator_value, epsilon=self.epsilon) def _create_pytorch_optimizer(self, params): import torch if isinstance(self.learning_rate, LearningRateSchedule): lr = self.learning_rate.initial_rate else: lr = self.learning_rate return torch.optim.Adagrad( params, lr, initial_accumulator_value=self.initial_accumulator_value, eps=self.epsilon) class Adam(Optimizer): """The Adam optimization algorithm.""" def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08): def __init__(self, learning_rate: Union[float, LearningRateSchedule] = 0.001, beta1: float = 0.9, beta2: float = 0.999, epsilon: float = 1e-08): """Construct an Adam optimizer. Parameters Loading @@ -113,9 +159,10 @@ class Adam(Optimizer): self.beta2 = beta2 self.epsilon = epsilon def _create_optimizer(self, global_step): def _create_tf_optimizer(self, global_step): import tensorflow as tf if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) learning_rate = self.learning_rate._create_tf_tensor(global_step) else: learning_rate = self.learning_rate return tf.keras.optimizers.Adam( Loading @@ -124,15 +171,23 @@ class Adam(Optimizer): beta_2=self.beta2, epsilon=self.epsilon) def _create_pytorch_optimizer(self, params): import torch if isinstance(self.learning_rate, LearningRateSchedule): lr = self.learning_rate.initial_rate else: lr = self.learning_rate return torch.optim.Adam(params, lr, (self.beta1, self.beta2), self.epsilon) class RMSProp(Optimizer): """RMSProp Optimization algorithm.""" def __init__(self, learning_rate=0.001, momentum=0.0, decay=0.9, epsilon=1e-10): learning_rate: Union[float, LearningRateSchedule] = 0.001, momentum: float = 0.0, decay: float = 0.9, epsilon: float = 1e-10): """Construct an RMSProp Optimizer. Parameters Loading @@ -151,9 +206,10 @@ class RMSProp(Optimizer): self.decay = decay self.epsilon = epsilon def _create_optimizer(self, global_step): def _create_tf_optimizer(self, global_step): import tensorflow as tf if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) learning_rate = self.learning_rate._create_tf_tensor(global_step) else: learning_rate = self.learning_rate return tf.keras.optimizers.RMSprop( Loading @@ -162,11 +218,20 @@ class RMSProp(Optimizer): rho=self.decay, epsilon=self.epsilon) def _create_pytorch_optimizer(self, params): import torch if isinstance(self.learning_rate, LearningRateSchedule): lr = self.learning_rate.initial_rate else: lr = self.learning_rate return torch.optim.RMSprop( params, lr, alpha=self.decay, eps=self.epsilon, momentum=self.momentum) class GradientDescent(Optimizer): """The gradient descent optimization algorithm.""" def __init__(self, learning_rate=0.001): def __init__(self, learning_rate: Union[float, LearningRateSchedule] = 0.001): """Construct a gradient descent optimizer. Parameters Loading @@ -176,18 +241,31 @@ class GradientDescent(Optimizer): """ self.learning_rate = learning_rate def _create_optimizer(self, global_step): def _create_tf_optimizer(self, global_step): import tensorflow as tf if isinstance(self.learning_rate, LearningRateSchedule): learning_rate = self.learning_rate._create_tensor(global_step) learning_rate = self.learning_rate._create_tf_tensor(global_step) else: learning_rate = self.learning_rate return tf.keras.optimizers.SGD(learning_rate=learning_rate) def _create_pytorch_optimizer(self, params): import torch if isinstance(self.learning_rate, LearningRateSchedule): lr = self.learning_rate.initial_rate else: lr = self.learning_rate return torch.optim.SGD(params, lr) class ExponentialDecay(LearningRateSchedule): """A learning rate that decreases exponentially with the number of training steps.""" def __init__(self, initial_rate, decay_rate, decay_steps, staircase=True): def __init__(self, initial_rate: float, decay_rate: float, decay_steps: int, staircase: bool = True): """Create an exponentially decaying learning rate. The learning rate starts as initial_rate. Every decay_steps training steps, it is multiplied by decay_rate. Loading @@ -209,18 +287,31 @@ class ExponentialDecay(LearningRateSchedule): self.decay_steps = decay_steps self.staircase = staircase def _create_tensor(self, global_step): def _create_tf_tensor(self, global_step): import tensorflow as tf return tf.keras.optimizers.schedules.ExponentialDecay( initial_learning_rate=self.initial_rate, decay_rate=self.decay_rate, decay_steps=self.decay_steps, staircase=self.staircase)(global_step) def _create_pytorch_schedule(self, optimizer): import torch if self.staircase: return torch.optim.lr_scheduler.StepLR(optimizer, self.decay_steps, self.decay_rate) return torch.optim.lr_scheduler.ExponentialLR( optimizer, math.pow(self.decay_rate, 1 / self.decay_steps)) class PolynomialDecay(LearningRateSchedule): """A learning rate that decreases from an initial value to a final value over a fixed number of training steps.""" def __init__(self, initial_rate, final_rate, decay_steps, power=1.0): def __init__(self, initial_rate: float, final_rate: float, decay_steps: int, power: float = 1.0): """Create a smoothly decaying learning rate. The learning rate starts as initial_rate. It smoothly decreases to final_rate over decay_steps training steps. Loading @@ -243,23 +334,34 @@ class PolynomialDecay(LearningRateSchedule): self.decay_steps = decay_steps self.power = power def _create_tensor(self, global_step): def _create_tf_tensor(self, global_step): import tensorflow as tf return tf.keras.optimizers.schedules.PolynomialDecay( initial_learning_rate=self.initial_rate, end_learning_rate=self.final_rate, decay_steps=self.decay_steps, power=self.power)(global_step) def _create_pytorch_schedule(self, optimizer): def f(step): t = min(step, self.decay_steps) / self.decay_steps return ((self.initial_rate - self.final_rate) * (1 - t)**self.power) + self.final_rate import torch return torch.optim.lr_scheduler.LambdaLR(optimizer, f) class LinearCosineDecay(LearningRateSchedule): """Applies linear cosine decay to the learning rate""" def __init__(self, initial_rate, decay_steps, alpha=0.0, beta=0.001, num_periods=0.5): initial_rate: float, decay_steps: int, alpha: float = 0.0, beta: float = 0.001, num_periods: float = 0.5): """ Parameters ---------- Loading @@ -276,7 +378,8 @@ class LinearCosineDecay(LearningRateSchedule): self.beta = beta self.num_periods = num_periods def _create_tensor(self, global_step): def _create_tf_tensor(self, global_step): import tensorflow as tf return tf.compat.v1.train.linear_cosine_decay( learning_rate=self.initial_rate, global_step=global_step, Loading @@ -284,3 +387,15 @@ class LinearCosineDecay(LearningRateSchedule): alpha=self.alpha, beta=self.beta, num_periods=self.num_periods) def _create_pytorch_schedule(self, optimizer): def f(step): t = min(step, self.decay_steps) / self.decay_steps linear_decay = 1 - t cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * self.num_periods * t)) decayed = (self.alpha + linear_decay) * cosine_decay + self.beta return self.initial_rate * decayed import torch return torch.optim.lr_scheduler.LambdaLR(optimizer, f)
deepchem/models/tests/test_optimizers.py +110 −30 Original line number Diff line number Diff line import deepchem.models.optimizers as optimizers import unittest try: import tensorflow as tf from tensorflow.python.framework import test_util has_tensorflow = True except: has_tensorflow = False try: import torch has_pytorch = True except: has_pytorch = False class TestLayers(test_util.TensorFlowTestCase): class TestLayers(unittest.TestCase): """Test optimizers and related classes.""" def test_adam(self): @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_adam_tf(self): """Test creating an Adam optimizer.""" opt = optimizers.Adam(learning_rate=0.01) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) assert isinstance(tfopt, tf.keras.optimizers.Adam) def test_rmsprop(self): @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_adam_pytorch(self): """Test creating an Adam optimizer.""" opt = optimizers.Adam(learning_rate=0.01) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) assert isinstance(torchopt, torch.optim.Adam) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_adagrad_tf(self): """Test creating an AdaGrad optimizer.""" opt = optimizers.AdaGrad(learning_rate=0.01) global_step = tf.Variable(0) tfopt = opt._create_tf_optimizer(global_step) assert isinstance(tfopt, tf.keras.optimizers.Adagrad) @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_adagrad_pytorch(self): """Test creating an AdaGrad optimizer.""" opt = optimizers.AdaGrad(learning_rate=0.01) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) assert isinstance(torchopt, torch.optim.Adagrad) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_rmsprop_tf(self): """Test creating an RMSProp Optimizer.""" opt = optimizers.RMSProp(learning_rate=0.01) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) assert isinstance(tfopt, tf.keras.optimizers.RMSprop) def test_gradient_descent(self): @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_rmsprop_pytorch(self): """Test creating an RMSProp Optimizer.""" opt = optimizers.RMSProp(learning_rate=0.01) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) assert isinstance(torchopt, torch.optim.RMSprop) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_gradient_descent_tf(self): """Test creating a Gradient Descent optimizer.""" opt = optimizers.GradientDescent(learning_rate=0.01) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) assert isinstance(tfopt, tf.keras.optimizers.SGD) def test_exponential_decay(self): @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_gradient_descent_pytorch(self): """Test creating a Gradient Descent optimizer.""" opt = optimizers.GradientDescent(learning_rate=0.01) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) assert isinstance(torchopt, torch.optim.SGD) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_exponential_decay_tf(self): """Test creating an optimizer with an exponentially decaying learning rate.""" rate = optimizers.ExponentialDecay( initial_rate=0.001, decay_rate=0.99, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) def test_polynomial_decay(self): @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_exponential_decay_pytorch(self): """Test creating an optimizer with an exponentially decaying learning rate.""" rate = optimizers.ExponentialDecay( initial_rate=0.001, decay_rate=0.99, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) schedule = rate._create_pytorch_schedule(torchopt) @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_polynomial_decay_tf(self): """Test creating an optimizer with a polynomially decaying learning rate.""" rate = optimizers.PolynomialDecay( initial_rate=0.001, final_rate=0.0001, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_polynomial_decay_pytorch(self): """Test creating an optimizer with a polynomially decaying learning rate.""" rate = optimizers.PolynomialDecay( initial_rate=0.001, final_rate=0.0001, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) schedule = rate._create_pytorch_schedule(torchopt) def test_linearCosine_decay(self): @unittest.skipIf(not has_tensorflow, 'TensorFlow is not installed') def test_linearCosine_decay_tf(self): """test creating an optimizer with a linear cosine decay to the learning rate""" rate = optimizers.LinearCosineDecay(initial_rate=0.1, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) with self.session() as sess: global_step = tf.Variable(0) tfopt = opt._create_optimizer(global_step) tfopt = opt._create_tf_optimizer(global_step) @unittest.skipIf(not has_pytorch, 'PyTorch is not installed') def test_linearCosine_decay_pytorch(self): """test creating an optimizer with a linear cosine decay to the learning rate""" rate = optimizers.LinearCosineDecay(initial_rate=0.1, decay_steps=10000) opt = optimizers.Adam(learning_rate=rate) params = [torch.nn.parameter.Parameter(torch.Tensor([1.0]))] torchopt = opt._create_pytorch_optimizer(params) schedule = rate._create_pytorch_schedule(torchopt)
