PulseFocusPlatform/static/ppdet/optimizer.py

267 lines
8.2 KiB
Python

# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import math
import logging
from paddle import fluid
import paddle.fluid.optimizer as optimizer
import paddle.fluid.regularizer as regularizer
from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
from paddle.fluid.layers.ops import cos
from ppdet.core.workspace import register, serializable
__all__ = ['LearningRate', 'OptimizerBuilder']
logger = logging.getLogger(__name__)
@serializable
class PiecewiseDecay(object):
"""
Multi step learning rate decay
Args:
gamma (float | list): decay factor
milestones (list): steps at which to decay learning rate
"""
def __init__(self, gamma=[0.1, 0.1], milestones=[60000, 80000],
values=None):
super(PiecewiseDecay, self).__init__()
if type(gamma) is not list:
self.gamma = []
for i in range(len(milestones)):
self.gamma.append(gamma / 10**i)
else:
self.gamma = gamma
self.milestones = milestones
self.values = values
def __call__(self, base_lr=None, learning_rate=None):
if self.values is not None:
return fluid.layers.piecewise_decay(self.milestones, self.values)
assert base_lr is not None, "either base LR or values should be provided"
values = [base_lr]
for g in self.gamma:
new_lr = base_lr * g
values.append(new_lr)
return fluid.layers.piecewise_decay(self.milestones, values)
@serializable
class PolynomialDecay(object):
"""
Applies polynomial decay to the initial learning rate.
Args:
max_iter (int): The learning rate decay steps.
end_lr (float): End learning rate.
power (float): Polynomial attenuation coefficient
"""
def __init__(self, max_iter=180000, end_lr=0.0001, power=1.0):
super(PolynomialDecay).__init__()
self.max_iter = max_iter
self.end_lr = end_lr
self.power = power
def __call__(self, base_lr=None, learning_rate=None):
assert base_lr is not None, "either base LR or values should be provided"
lr = fluid.layers.polynomial_decay(base_lr, self.max_iter, self.end_lr,
self.power)
return lr
@serializable
class ExponentialDecay(object):
"""
Applies exponential decay to the learning rate.
Args:
max_iter (int): The learning rate decay steps.
decay_rate (float): The learning rate decay rate.
"""
def __init__(self, max_iter, decay_rate):
super(ExponentialDecay).__init__()
self.max_iter = max_iter
self.decay_rate = decay_rate
def __call__(self, base_lr=None, learning_rate=None):
assert base_lr is not None, "either base LR or values should be provided"
lr = fluid.layers.exponential_decay(base_lr, self.max_iter,
self.decay_rate)
return lr
@serializable
class CosineDecay(object):
"""
Cosine learning rate decay
Args:
max_iters (float): max iterations for the training process.
if you commbine cosine decay with warmup, it is recommended that
the max_iter is much larger than the warmup iter
"""
def __init__(self, max_iters=180000):
self.max_iters = max_iters
def __call__(self, base_lr=None, learning_rate=None):
assert base_lr is not None, "either base LR or values should be provided"
lr = fluid.layers.cosine_decay(base_lr, 1, self.max_iters)
return lr
@serializable
class CosineDecayWithSkip(object):
"""
Cosine decay, with explicit support for warm up
Args:
total_steps (int): total steps over which to apply the decay
skip_steps (int): skip some steps at the beginning, e.g., warm up
"""
def __init__(self, total_steps, skip_steps=None):
super(CosineDecayWithSkip, self).__init__()
assert (not skip_steps or skip_steps > 0), \
"skip steps must be greater than zero"
assert total_steps > 0, "total step must be greater than zero"
assert (not skip_steps or skip_steps < total_steps), \
"skip steps must be smaller than total steps"
self.total_steps = total_steps
self.skip_steps = skip_steps
def __call__(self, base_lr=None, learning_rate=None):
steps = _decay_step_counter()
total = self.total_steps
if self.skip_steps is not None:
total -= self.skip_steps
lr = fluid.layers.tensor.create_global_var(
shape=[1],
value=base_lr,
dtype='float32',
persistable=True,
name="learning_rate")
def decay():
cos_lr = base_lr * .5 * (cos(steps * (math.pi / total)) + 1)
fluid.layers.tensor.assign(input=cos_lr, output=lr)
if self.skip_steps is None:
decay()
else:
skipped = steps >= self.skip_steps
fluid.layers.cond(skipped, decay)
return lr
@serializable
class LinearWarmup(object):
"""
Warm up learning rate linearly
Args:
steps (int): warm up steps
start_factor (float): initial learning rate factor
"""
def __init__(self, steps=500, start_factor=1. / 3):
super(LinearWarmup, self).__init__()
self.steps = steps
self.start_factor = start_factor
def __call__(self, base_lr, learning_rate):
start_lr = base_lr * self.start_factor
return fluid.layers.linear_lr_warmup(
learning_rate=learning_rate,
warmup_steps=self.steps,
start_lr=start_lr,
end_lr=base_lr)
@register
class LearningRate(object):
"""
Learning Rate configuration
Args:
base_lr (float): base learning rate
schedulers (list): learning rate schedulers
"""
__category__ = 'optim'
def __init__(self,
base_lr=0.01,
schedulers=[PiecewiseDecay(), LinearWarmup()]):
super(LearningRate, self).__init__()
self.base_lr = base_lr
self.schedulers = schedulers
def __call__(self):
lr = None
for sched in self.schedulers:
lr = sched(self.base_lr, lr)
return lr
@register
class OptimizerBuilder():
"""
Build optimizer handles
Args:
regularizer (object): an `Regularizer` instance
optimizer (object): an `Optimizer` instance
"""
__category__ = 'optim'
def __init__(self,
clip_grad_by_norm=None,
regularizer={'type': 'L2',
'factor': .0001},
optimizer={'type': 'Momentum',
'momentum': .9}):
self.clip_grad_by_norm = clip_grad_by_norm
self.regularizer = regularizer
self.optimizer = optimizer
def __call__(self, learning_rate):
if self.clip_grad_by_norm is not None:
fluid.clip.set_gradient_clip(
clip=fluid.clip.GradientClipByGlobalNorm(
clip_norm=self.clip_grad_by_norm))
if self.regularizer:
reg_type = self.regularizer['type'] + 'Decay'
reg_factor = self.regularizer['factor']
regularization = getattr(regularizer, reg_type)(reg_factor)
else:
regularization = None
optim_args = self.optimizer.copy()
optim_type = optim_args['type']
del optim_args['type']
op = getattr(optimizer, optim_type)
return op(learning_rate=learning_rate,
regularization=regularization,
**optim_args)