p32761584
/
tensorlayer3
forked from TensorLayer/tensorlayer3
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity
tensorlayer3/tensorlayer/layers/activation.py

624 lines
21 KiB
Python
Raw Permalink Blame History

#! /usr/bin/python
# -*- coding: utf-8 -*-
from tensorlayer import logging
import tensorlayer as tl
from tensorlayer.initializers import truncated_normal
from tensorlayer.layers.core import Module
__all__ = [
'PRelu', 'PRelu6', 'PTRelu6', 'LeakyReLU', 'LeakyReLU6', 'LeakyTwiceRelu6', 'Ramp', 'Swish', 'HardTanh', 'Mish'
]
class PRelu(Module):
"""
The :class:`PRelu` class is Parametric Rectified Linear layer.
It follows f(x) = alpha * x for x < 0, f(x) = x for x >= 0,
where alpha is a learned array with the same shape as x.
Parameters
----------
channel_shared : boolean
If True, single weight is shared by all channels.
in_channels: int
The number of channels of the previous layer.
If None, it will be automatically detected when the layer is forwarded for the first time.
a_init : initializer
The initializer for initializing the alpha(s).
name : None or str
A unique layer name.
Examples
-----------
>>> inputs = tl.layers.Input([10, 5])
>>> prelulayer = tl.layers.PRelu(channel_shared=True, in_channels=5)(inputs)
References
-----------
- `Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification <http://arxiv.org/abs/1502.01852>`__
- `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
"""
def __init__(
self, channel_shared=False, in_channels=None, a_init=truncated_normal(mean=0.0, stddev=0.05), name=None,
data_format='channels_last', dim=2
):
super(PRelu, self).__init__(name)
self.channel_shared = channel_shared
self.in_channels = in_channels
self.a_init = a_init
self.data_format = data_format
self.dim = dim
if self.channel_shared:
self.build((None, ))
self._built = True
elif self.in_channels is not None:
self.build((None, self.in_channels))
self._built = True
logging.info("PRelu %s: channel_shared: %s" % (self.name, self.channel_shared))
def __repr__(self):
s = ('{classname}(')
s += 'channel_shared={channel_shared},'
s += 'in_channels={in_channels},'
s += 'name={name}'
s += ')'
return s.format(classname=self.__class__.__name__, **self.__dict__)
def build(self, inputs_shape):
if self.channel_shared:
w_shape = (1, )
elif self.data_format == 'channels_last':
w_shape = (self.in_channels, )
elif self.data_format == 'channels_first':
if self.dim == 2:
w_shape = (1, self.in_channels, 1, 1)
elif self.dim == 1:
w_shape = (1, self.in_channels, 1)
elif self.dim == 3:
w_shape = (1, self.in_channels, 1, 1, 1)
else:
raise Exception("Dim should be equal to 1, 2 or 3")
self.alpha_var = self._get_weights("alpha", shape=w_shape, init=self.a_init)
self.relu = tl.ops.ReLU()
self.sigmoid = tl.ops.Sigmoid()
def forward(self, inputs):
if self._forward_state == False:
if self._built == False:
self.build(tl.get_tensor_shape(inputs))
self._built = True
self._forward_state = True
pos = self.relu(inputs)
self.alpha_var_constrained = self.sigmoid(self.alpha_var)
neg = -self.alpha_var_constrained * self.relu(-inputs)
return pos + neg
class PRelu6(Module):
"""
The :class:`PRelu6` class is Parametric Rectified Linear layer integrating ReLU6 behaviour.
This Layer is a modified version of the :class:`PRelu`.
This activation layer use a modified version :func:`tl.act.leaky_relu` introduced by the following paper:
`Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
This activation function also use a modified version of the activation function :func:`tf.nn.relu6` introduced by the following paper:
`Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
This activation layer push further the logic by adding `leaky` behaviour both below zero and above six.
The function return the following results:
- When x < 0: ``f(x) = alpha_low * x``.
- When x in [0, 6]: ``f(x) = x``.
- When x > 6: ``f(x) = 6``.
Parameters
----------
channel_shared : boolean
If True, single weight is shared by all channels.
in_channels: int
The number of channels of the previous layer.
If None, it will be automatically detected when the layer is forwarded for the first time.
a_init : initializer
The initializer for initializing the alpha(s).
name : None or str
A unique layer name.
Examples
-----------
>>> inputs = tl.layers.Input([10, 5])
>>> prelulayer = tl.layers.PRelu6(channel_shared=True, in_channels=5)(inputs)
References
-----------
- `Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification <http://arxiv.org/abs/1502.01852>`__
- `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
- `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
"""
def __init__(
self,
channel_shared=False,
in_channels=None,
a_init=truncated_normal(mean=0.0, stddev=0.05),
name=None, # "prelu6"
data_format='channels_last',
dim=2
):
super(PRelu6, self).__init__(name)
self.channel_shared = channel_shared
self.in_channels = in_channels
self.a_init = a_init
self.data_format = data_format
self.dim = dim
if self.channel_shared:
self.build((None, ))
self._built = True
elif self.in_channels is not None:
self.build((None, self.in_channels))
self._built = True
logging.info("PRelu6 %s: channel_shared: %s" % (self.name, self.channel_shared))
def __repr__(self):
s = ('{classname}(')
s += 'channel_shared={channel_shared},'
s += 'in_channels={in_channels},'
s += 'name={name}'
s += ')'
return s.format(classname=self.__class__.__name__, **self.__dict__)
def build(self, inputs_shape):
if self.channel_shared:
w_shape = (1, )
elif self.data_format == 'channels_last':
w_shape = (self.in_channels, )
elif self.data_format == 'channels_first':
if self.dim == 2:
w_shape = (1, self.in_channels, 1, 1)
elif self.dim == 1:
w_shape = (1, self.in_channels, 1)
elif self.dim == 3:
w_shape = (1, self.in_channels, 1, 1, 1)
else:
raise Exception("Dim should be equal to 1, 2 or 3")
self.alpha_var = self._get_weights("alpha", shape=w_shape, init=self.a_init)
self.sigmoid = tl.ops.Sigmoid()
self.relu = tl.ops.ReLU()
# @tf.function
def forward(self, inputs):
if self._forward_state == False:
if self._built == False:
self.build(tl.get_tensor_shape(inputs))
self._built = True
self._forward_state = True
alpha_var_constrained = self.sigmoid(self.alpha_var)
pos = self.relu(inputs)
pos_6 = -self.relu(inputs - 6)
neg = -alpha_var_constrained * self.relu(-inputs)
return pos + pos_6 + neg
class PTRelu6(Module):
"""
The :class:`PTRelu6` class is Parametric Rectified Linear layer integrating ReLU6 behaviour.
This Layer is a modified version of the :class:`PRelu`.
This activation layer use a modified version :func:`tl.act.leaky_relu` introduced by the following paper:
`Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
This activation function also use a modified version of the activation function :func:`tf.nn.relu6` introduced by the following paper:
`Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
This activation layer push further the logic by adding `leaky` behaviour both below zero and above six.
The function return the following results:
- When x < 0: ``f(x) = alpha_low * x``.
- When x in [0, 6]: ``f(x) = x``.
- When x > 6: ``f(x) = 6 + (alpha_high * (x-6))``.
This version goes one step beyond :class:`PRelu6` by introducing leaky behaviour on the positive side when x > 6.
Parameters
----------
channel_shared : boolean
If True, single weight is shared by all channels.
in_channels: int
The number of channels of the previous layer.
If None, it will be automatically detected when the layer is forwarded for the first time.
a_init : initializer
The initializer for initializing the alpha(s).
name : None or str
A unique layer name.
Examples
-----------
>>> inputs = tl.layers.Input([10, 5])
>>> prelulayer = tl.layers.PTRelu6(channel_shared=True, in_channels=5)(inputs)
References
-----------
- `Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification <http://arxiv.org/abs/1502.01852>`__
- `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
- `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
"""
def __init__(
self,
channel_shared=False,
in_channels=None,
data_format='channels_last',
a_init=truncated_normal(mean=0.0, stddev=0.05),
name=None # "ptrelu6"
):
super(PTRelu6, self).__init__(name)
self.channel_shared = channel_shared
self.in_channels = in_channels
self.data_format = data_format
self.a_init = a_init
if self.channel_shared:
self.build((None, ))
self._built = True
elif self.in_channels:
self.build((None, self.in_channels))
self._built = True
logging.info("PTRelu6 %s: channel_shared: %s" % (self.name, self.channel_shared))
def __repr__(self):
s = ('{classname}(')
s += 'channel_shared={channel_shared},'
s += 'in_channels={in_channels},'
s += 'name={name}'
s += ')'
return s.format(classname=self.__class__.__name__, **self.__dict__)
def build(self, inputs_shape):
if self.channel_shared:
w_shape = (1, )
elif self.data_format == 'channels_last':
w_shape = (self.in_channels, )
elif self.data_format == 'channels_first':
if self.dim == 2:
w_shape = (1, self.in_channels, 1, 1)
elif self.dim == 1:
w_shape = (1, self.in_channels, 1)
elif self.dim == 3:
w_shape = (1, self.in_channels, 1, 1, 1)
else:
raise Exception("Dim should be equal to 1, 2 or 3")
# Alpha for outputs lower than zeros
self.alpha_low = self._get_weights("alpha_low", shape=w_shape, init=self.a_init)
self.sigmoid = tl.ops.Sigmoid()
self.relu = tl.ops.ReLU()
# Alpha for outputs higher than 6
self.alpha_high = self._get_weights("alpha_high", shape=w_shape, init=self.a_init)
# @tf.function
def forward(self, inputs):
if self._forward_state == False:
if self._built == False:
self.build(tl.get_tensor_shape(inputs))
self._built = True
self._forward_state = True
alpha_low_constrained = self.sigmoid(self.alpha_low)
alpha_high_constrained = self.sigmoid(self.alpha_high)
pos = self.relu(inputs)
pos_6 = -self.relu(inputs - 6) + alpha_high_constrained * self.relu(inputs - 6)
neg = -alpha_low_constrained * self.relu(-inputs)
return pos + pos_6 + neg
class Ramp(Module):
"""Ramp activation function.
Reference: [tf.clip_by_value]<https://www.tensorflow.org/api_docs/python/tf/clip_by_value>
Parameters
----------
x : Tensor
input.
v_min : float
cap input to v_min as a lower bound.
v_max : float
cap input to v_max as a upper bound.
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
Examples
-----------
>>> inputs = tl.layers.Input([10, 5])
>>> prelulayer = tl.layers.Ramp()(inputs)
"""
def __init__(self, v_min=0, v_max=1):
super(Ramp, self).__init__()
self._built = True
self.v_min = v_min
self.v_max = v_max
def forward(self, x):
return tl.ops.clip_by_value(x, clip_value_min=self.v_min, clip_value_max=self.v_max)
class LeakyReLU(Module):
"""
This function is a modified version of ReLU, introducing a nonzero gradient for negative input. Introduced by the paper:
`Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
The function return the following results:
- When x < 0: ``f(x) = alpha_low * x``.
- When x >= 0: ``f(x) = x``.
Parameters
----------
x : Tensor
Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``.
alpha : float
Slope.
name : str
The function name (optional).
Examples
--------
>>> net = tl.layers.Input([10, 200])
>>> net = tl.layers.LeakyReLU(alpha=0.5)(net)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
References
----------
- `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
"""
def __init__(self, alpha=0.2):
super(LeakyReLU, self).__init__()
self._built = True
self.alpha = alpha
self._leakyrelu = tl.ops.LeakyReLU(alpha=alpha)
def forward(self, x):
return self._leakyrelu(x)
class LeakyReLU6(Module):
"""
This activation function is a modified version :func:`leaky_relu` introduced by the following paper:
`Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
This activation function also follows the behaviour of the activation function :func:`tf.ops.relu6` introduced by the following paper:
`Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
The function return the following results:
- When x < 0: ``f(x) = alpha_low * x``.
- When x in [0, 6]: ``f(x) = x``.
- When x > 6: ``f(x) = 6``.
Parameters
----------
x : Tensor
Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``.
alpha : float
Slope.
name : str
The function name (optional).
Examples
--------
>>> net = tl.layers.Input([10, 200])
>>> net = tl.layers.LeakyReLU6(alpha=0.5)(net)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
References
----------
- `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
- `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
"""
def __init__(self, alpha=0.2):
super(LeakyReLU6, self).__init__()
self._built = True
if not (0 < alpha <= 1):
raise ValueError("`alpha` value must be in [0, 1]`")
self.alpha = alpha
self.minimum = tl.ops.Minimum()
self.maximum = tl.ops.Maximum()
def forward(self, x):
return self.minimum(self.maximum(x, self.alpha * x), 6)
class LeakyTwiceRelu6(Module):
"""
This activation function is a modified version :func:`leaky_relu` introduced by the following paper:
`Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
This activation function also follows the behaviour of the activation function :func:`tf.ops.relu6` introduced by the following paper:
`Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
This function push further the logic by adding `leaky` behaviour both below zero and above six.
The function return the following results:
- When x < 0: ``f(x) = alpha_low * x``.
- When x in [0, 6]: ``f(x) = x``.
- When x > 6: ``f(x) = 6 + (alpha_high * (x-6))``.
Parameters
----------
x : Tensor
Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``.
alpha_low : float
Slope for x < 0: ``f(x) = alpha_low * x``.
alpha_high : float
Slope for x < 6: ``f(x) = 6 (alpha_high * (x-6))``.
name : str
The function name (optional).
Examples
--------
>>> net = tl.layers.Input([10, 200])
>>> net = tl.layers.LeakyTwiceRelu6(alpha_low=0.5, alpha_high=0.2)(net)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
References
----------
- `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__
- `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
"""
def __init__(self, alpha_low=0.2, alpha_high=0.2):
super(LeakyTwiceRelu6, self).__init__()
self._built = True
if not (0 < alpha_high <= 1):
raise ValueError("`alpha_high` value must be in [0, 1]`")
if not (0 < alpha_low <= 1):
raise ValueError("`alpha_low` value must be in [0, 1]`")
self.alpha_low = alpha_low
self.alpha_high = alpha_high
self.minimum = tl.ops.Minimum()
self.maximum = tl.ops.Maximum()
def forward(self, x):
x_is_above_0 = self.minimum(x, 6 * (1 - self.alpha_high) + self.alpha_high * x)
x_is_below_0 = self.minimum(self.alpha_low * x, 0)
return self.maximum(x_is_above_0, x_is_below_0)
class Swish(Module):
"""Swish function.
See `Swish: a Self-Gated Activation Function <https://arxiv.org/abs/1710.05941>`__.
Parameters
----------
x : Tensor
input.
name: str
function name (optional).
Examples
--------
>>> net = tl.layers.Input([10, 200])
>>> net = tl.layers.Swish()(net)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
"""
def __init__(self):
super(Swish, self).__init__()
self.sigmoid = tl.ops.Sigmoid()
self._built = True
def forward(self, x):
return self.sigmoid(x) * x
class HardTanh(Module):
"""Hard tanh activation function.
Which is a ramp function with low bound of -1 and upper bound of 1, shortcut is `htanh`.
Parameters
----------
x : Tensor
input.
name : str
The function name (optional).
Examples
--------
>>> net = tl.layers.Input([10, 200])
>>> net = tl.layers.HardTanh()(net)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
"""
def __init__(self):
super(HardTanh, self).__init__()
self._built = True
def forward(self, x):
return tl.ops.clip_by_value(x, -1, 1)
class Mish(Module):
"""Mish activation function.
Reference: [Mish: A Self Regularized Non-Monotonic Neural Activation Function .Diganta Misra, 2019]<https://arxiv.org/abs/1908.08681>
Parameters
----------
x : Tensor
input.
Examples
--------
>>> net = tl.layers.Input([10, 200])
>>> net = tl.layers.Mish()(net)
Returns
-------
Tensor
A ``Tensor`` in the same type as ``x``.
"""
def __init__(self):
super(Mish, self).__init__()
self._tanh = tl.ops.Tanh()
self._softplus = tl.ops.Softplus()
self._built = True
def forward(self, x):
return x * self._tanh(self._softplus(x))
Reference in New Issue View Git Blame Copy Permalink