forked from PulseFocusPlatform/PulseFocusPlatform
267 lines
8.2 KiB
Python
267 lines
8.2 KiB
Python
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import math
|
|
import logging
|
|
|
|
from paddle import fluid
|
|
|
|
import paddle.fluid.optimizer as optimizer
|
|
import paddle.fluid.regularizer as regularizer
|
|
from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
|
|
from paddle.fluid.layers.ops import cos
|
|
|
|
from ppdet.core.workspace import register, serializable
|
|
|
|
__all__ = ['LearningRate', 'OptimizerBuilder']
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@serializable
|
|
class PiecewiseDecay(object):
|
|
"""
|
|
Multi step learning rate decay
|
|
|
|
Args:
|
|
gamma (float | list): decay factor
|
|
milestones (list): steps at which to decay learning rate
|
|
"""
|
|
|
|
def __init__(self, gamma=[0.1, 0.1], milestones=[60000, 80000],
|
|
values=None):
|
|
super(PiecewiseDecay, self).__init__()
|
|
if type(gamma) is not list:
|
|
self.gamma = []
|
|
for i in range(len(milestones)):
|
|
self.gamma.append(gamma / 10**i)
|
|
else:
|
|
self.gamma = gamma
|
|
self.milestones = milestones
|
|
self.values = values
|
|
|
|
def __call__(self, base_lr=None, learning_rate=None):
|
|
if self.values is not None:
|
|
return fluid.layers.piecewise_decay(self.milestones, self.values)
|
|
assert base_lr is not None, "either base LR or values should be provided"
|
|
values = [base_lr]
|
|
for g in self.gamma:
|
|
new_lr = base_lr * g
|
|
values.append(new_lr)
|
|
return fluid.layers.piecewise_decay(self.milestones, values)
|
|
|
|
|
|
@serializable
|
|
class PolynomialDecay(object):
|
|
"""
|
|
Applies polynomial decay to the initial learning rate.
|
|
Args:
|
|
max_iter (int): The learning rate decay steps.
|
|
end_lr (float): End learning rate.
|
|
power (float): Polynomial attenuation coefficient
|
|
"""
|
|
|
|
def __init__(self, max_iter=180000, end_lr=0.0001, power=1.0):
|
|
super(PolynomialDecay).__init__()
|
|
self.max_iter = max_iter
|
|
self.end_lr = end_lr
|
|
self.power = power
|
|
|
|
def __call__(self, base_lr=None, learning_rate=None):
|
|
assert base_lr is not None, "either base LR or values should be provided"
|
|
lr = fluid.layers.polynomial_decay(base_lr, self.max_iter, self.end_lr,
|
|
self.power)
|
|
return lr
|
|
|
|
|
|
@serializable
|
|
class ExponentialDecay(object):
|
|
"""
|
|
Applies exponential decay to the learning rate.
|
|
Args:
|
|
max_iter (int): The learning rate decay steps.
|
|
decay_rate (float): The learning rate decay rate.
|
|
"""
|
|
|
|
def __init__(self, max_iter, decay_rate):
|
|
super(ExponentialDecay).__init__()
|
|
self.max_iter = max_iter
|
|
self.decay_rate = decay_rate
|
|
|
|
def __call__(self, base_lr=None, learning_rate=None):
|
|
assert base_lr is not None, "either base LR or values should be provided"
|
|
lr = fluid.layers.exponential_decay(base_lr, self.max_iter,
|
|
self.decay_rate)
|
|
return lr
|
|
|
|
|
|
@serializable
|
|
class CosineDecay(object):
|
|
"""
|
|
Cosine learning rate decay
|
|
|
|
Args:
|
|
max_iters (float): max iterations for the training process.
|
|
if you commbine cosine decay with warmup, it is recommended that
|
|
the max_iter is much larger than the warmup iter
|
|
"""
|
|
|
|
def __init__(self, max_iters=180000):
|
|
self.max_iters = max_iters
|
|
|
|
def __call__(self, base_lr=None, learning_rate=None):
|
|
assert base_lr is not None, "either base LR or values should be provided"
|
|
lr = fluid.layers.cosine_decay(base_lr, 1, self.max_iters)
|
|
return lr
|
|
|
|
|
|
@serializable
|
|
class CosineDecayWithSkip(object):
|
|
"""
|
|
Cosine decay, with explicit support for warm up
|
|
|
|
Args:
|
|
total_steps (int): total steps over which to apply the decay
|
|
skip_steps (int): skip some steps at the beginning, e.g., warm up
|
|
"""
|
|
|
|
def __init__(self, total_steps, skip_steps=None):
|
|
super(CosineDecayWithSkip, self).__init__()
|
|
assert (not skip_steps or skip_steps > 0), \
|
|
"skip steps must be greater than zero"
|
|
assert total_steps > 0, "total step must be greater than zero"
|
|
assert (not skip_steps or skip_steps < total_steps), \
|
|
"skip steps must be smaller than total steps"
|
|
self.total_steps = total_steps
|
|
self.skip_steps = skip_steps
|
|
|
|
def __call__(self, base_lr=None, learning_rate=None):
|
|
steps = _decay_step_counter()
|
|
total = self.total_steps
|
|
if self.skip_steps is not None:
|
|
total -= self.skip_steps
|
|
|
|
lr = fluid.layers.tensor.create_global_var(
|
|
shape=[1],
|
|
value=base_lr,
|
|
dtype='float32',
|
|
persistable=True,
|
|
name="learning_rate")
|
|
|
|
def decay():
|
|
cos_lr = base_lr * .5 * (cos(steps * (math.pi / total)) + 1)
|
|
fluid.layers.tensor.assign(input=cos_lr, output=lr)
|
|
|
|
if self.skip_steps is None:
|
|
decay()
|
|
else:
|
|
skipped = steps >= self.skip_steps
|
|
fluid.layers.cond(skipped, decay)
|
|
return lr
|
|
|
|
|
|
@serializable
|
|
class LinearWarmup(object):
|
|
"""
|
|
Warm up learning rate linearly
|
|
|
|
Args:
|
|
steps (int): warm up steps
|
|
start_factor (float): initial learning rate factor
|
|
"""
|
|
|
|
def __init__(self, steps=500, start_factor=1. / 3):
|
|
super(LinearWarmup, self).__init__()
|
|
self.steps = steps
|
|
self.start_factor = start_factor
|
|
|
|
def __call__(self, base_lr, learning_rate):
|
|
start_lr = base_lr * self.start_factor
|
|
|
|
return fluid.layers.linear_lr_warmup(
|
|
learning_rate=learning_rate,
|
|
warmup_steps=self.steps,
|
|
start_lr=start_lr,
|
|
end_lr=base_lr)
|
|
|
|
|
|
@register
|
|
class LearningRate(object):
|
|
"""
|
|
Learning Rate configuration
|
|
|
|
Args:
|
|
base_lr (float): base learning rate
|
|
schedulers (list): learning rate schedulers
|
|
"""
|
|
__category__ = 'optim'
|
|
|
|
def __init__(self,
|
|
base_lr=0.01,
|
|
schedulers=[PiecewiseDecay(), LinearWarmup()]):
|
|
super(LearningRate, self).__init__()
|
|
self.base_lr = base_lr
|
|
self.schedulers = schedulers
|
|
|
|
def __call__(self):
|
|
lr = None
|
|
for sched in self.schedulers:
|
|
lr = sched(self.base_lr, lr)
|
|
return lr
|
|
|
|
|
|
@register
|
|
class OptimizerBuilder():
|
|
"""
|
|
Build optimizer handles
|
|
|
|
Args:
|
|
regularizer (object): an `Regularizer` instance
|
|
optimizer (object): an `Optimizer` instance
|
|
"""
|
|
__category__ = 'optim'
|
|
|
|
def __init__(self,
|
|
clip_grad_by_norm=None,
|
|
regularizer={'type': 'L2',
|
|
'factor': .0001},
|
|
optimizer={'type': 'Momentum',
|
|
'momentum': .9}):
|
|
self.clip_grad_by_norm = clip_grad_by_norm
|
|
self.regularizer = regularizer
|
|
self.optimizer = optimizer
|
|
|
|
def __call__(self, learning_rate):
|
|
if self.clip_grad_by_norm is not None:
|
|
fluid.clip.set_gradient_clip(
|
|
clip=fluid.clip.GradientClipByGlobalNorm(
|
|
clip_norm=self.clip_grad_by_norm))
|
|
if self.regularizer:
|
|
reg_type = self.regularizer['type'] + 'Decay'
|
|
reg_factor = self.regularizer['factor']
|
|
regularization = getattr(regularizer, reg_type)(reg_factor)
|
|
else:
|
|
regularization = None
|
|
optim_args = self.optimizer.copy()
|
|
optim_type = optim_args['type']
|
|
del optim_args['type']
|
|
op = getattr(optimizer, optim_type)
|
|
return op(learning_rate=learning_rate,
|
|
regularization=regularization,
|
|
**optim_args)
|