Activation Methods. More...

#include <activators.hpp>

Public Member Functions
double	sigmoid (double z)
	Computes the sigmoid activation function. More...

double	sigmoid_derivative (double z)
	Computes the derivative of the sigmoid activation function. More...

double	relu (double z)
	Computes the ReLU (Rectified Linear Unit) activation function. More...

double	relu_derivative (double z)
	Computes the derivative of the ReLU (Rectified Linear Unit) activation function. More...

std::vector< double >	softmax (const std::vector< double > &inputs)
	Computes the softmax activation function. More...

std::vector< std::vector< double > >	softmax_derivative (const std::vector< double > &inputs)
	Computes the derivative of the softmax activation function. More...

double	binary_step (double z)
	Computes the binary step activation function. More...

double	tanh (double z)
	Computes the hyperbolic tangent (tanh) activation function. More...

double	tanh_derivative (double z)
	Computes the derivative of the hyperbolic tangent (tanh) activation function. More...

double	smht (double z)
	Computes the Soboleva modified hyperbolic tangent (smht) activation function. More...

double	smht_derivative (double z)
	Computes the derivative of the Soboleva modified hyperbolic tangent (smht) activation function. More...

double	gelu (double z)
	Computes the Gaussian Error Linear Unit (GELU) activation function. More...

double	gelu_derivative (double z)
	Computes the derivative of the Gaussian Error Linear Unit (GELU) activation function. More...

double	softplus (double z)
	Computes the softplus activation function. More...

double	elu (double z, double alpha=10)
	Computes the Exponential Linear Unit (ELU) activation function. More...

double	elu_derivative (double z, double alpha=10)
	Computes the derivative of the Exponential Linear Unit (ELU) activation function. More...

double	selu (double z, double alpha=167326, double scale=10507)
	Computes the Scaled Exponential Linear Unit (SELU) activation function. More...

double	selu_derivative (double z, double alpha=167326, double scale=10507)
	Computes the derivative of the Scaled Exponential Linear Unit (SELU) activation function. More...

double	leaky_relu (double z, double alpha=001)
	Computes the Leaky Rectified Linear Unit (Leaky ReLU) activation function. More...

double	leaky_relu_derivative (double z, double alpha=001)
	Computes the derivative of the Leaky Rectified Linear Unit (Leaky ReLU) activation function. More...

double	prelu (double z, double alpha)
	Computes the Parametric Rectified Linear Unit (PReLU) activation function. More...

double	prelu_derivative (double z, double alpha)
	Computes the derivative of the Parametric Rectified Linear Unit (PReLU) activation function. More...

double	silu (double z)
	Computes the Sigmoid Linear Unit (SiLU or Swish) activation function. More...

double	silu_derivative (double z)
	Computes the derivative of the Sigmoid Linear Unit (SiLU or Swish) activation function. More...

Detailed Description

Activation Methods.

Definition at line 47 of file activators.hpp.

Member Function Documentation

◆ binary_step()

double gpmp::ml::Activation::binary_step ( double z )

Computes the binary step activation function.

Parameters

z	The input value

Returns: 1 if z >= 0, otherwise 0

Definition at line 84 of file activators.cpp.

                                              {
     return z >= 0 ? 1 : 0;
 }

◆ elu()

double gpmp::ml::Activation::elu	(	double	z,
		double	alpha = `10`
	)

Computes the Exponential Linear Unit (ELU) activation function.

Parameters

z	The input value
alpha	The ELU parameter (default: 10)

Returns: The ELU of z

Definition at line 126 of file activators.cpp.

                                                    {
     return z >= 0 ? z : alpha * (exp(z) - 1);
 }

◆ elu_derivative()

double gpmp::ml::Activation::elu_derivative	(	double	z,
		double	alpha = `10`
	)

Computes the derivative of the Exponential Linear Unit (ELU) activation function.

Parameters

z	The input value
alpha	The ELU parameter (default: 10)

Returns: The derivative of ELU at z

Definition at line 130 of file activators.cpp.

                                                               {
     return z >= 0 ? 1 : elu(z, alpha) + alpha;
 }

◆ gelu()

double gpmp::ml::Activation::gelu ( double z )

Computes the Gaussian Error Linear Unit (GELU) activation function.

Parameters

z	The input value

Returns: The GELU of z

Definition at line 106 of file activators.cpp.

                                       {
     return 0.5 * z *
            (1.0 + tanh(sqrt(2.0 / M_PI) * (z + 0.044715 * z * z * z)));
 }

◆ gelu_derivative()

double gpmp::ml::Activation::gelu_derivative ( double z )

Computes the derivative of the Gaussian Error Linear Unit (GELU) activation function.

Parameters

z	The input value

Returns: The derivative of GELU at z

Definition at line 111 of file activators.cpp.

                                                  {
     double cdf =
         0.5 * (1.0 + tanh(sqrt(2.0 / M_PI) * (0.044715 * z * z * z + 3 * z)));
     double pdf = exp(-0.5 * z * z) / sqrt(2.0 * M_PI);
     return 0.5 * (1.0 + cdf +
                   z * pdf * (1.0 / M_PI) *
                       (0.5 * (1.0 + tanh(sqrt(2.0 / M_PI) *
                                          (0.044715 * z * z * z + 3 * z)))) +
                   (1.0 / M_PI) * (1.0 - cdf * cdf));
 }

◆ leaky_relu()

double gpmp::ml::Activation::leaky_relu	(	double	z,
		double	alpha = `001`
	)

Computes the Leaky Rectified Linear Unit (Leaky ReLU) activation function.

Parameters

z	The input value
alpha	The Leaky ReLU parameter (default: 001)

Returns: The Leaky ReLU of z

Definition at line 143 of file activators.cpp.

                                                           {
     return z >= 0 ? z : alpha * z;
 }

◆ leaky_relu_derivative()

double gpmp::ml::Activation::leaky_relu_derivative	(	double	z,
		double	alpha = `001`
	)

Computes the derivative of the Leaky Rectified Linear Unit (Leaky ReLU) activation function.

Parameters

z	The input value
alpha	The Leaky ReLU parameter (default: 001)

Returns: The derivative of Leaky ReLU at z

Definition at line 147 of file activators.cpp.

                                                                      {
     return z >= 0 ? 1 : alpha;
 }

◆ prelu()

double gpmp::ml::Activation::prelu	(	double	z,
		double	alpha
	)

Computes the Parametric Rectified Linear Unit (PReLU) activation function.

Parameters

z	The input value
alpha	The PReLU parameter (slope)

Returns: The PReLU of z

Definition at line 151 of file activators.cpp.

                                                      {
     return z >= 0 ? z : alpha * z;
 }

◆ prelu_derivative()

double gpmp::ml::Activation::prelu_derivative	(	double	z,
		double	alpha
	)

Computes the derivative of the Parametric Rectified Linear Unit (PReLU) activation function.

Parameters

z	The input value
alpha	The PReLU parameter (slope)

Returns: The derivative of PReLU at z

Definition at line 155 of file activators.cpp.

                                                                 {
     return z >= 0 ? 1 : alpha;
 }

◆ relu()

double gpmp::ml::Activation::relu ( double z )

Computes the ReLU (Rectified Linear Unit) activation function.

Parameters

z	The input value

Returns: The ReLU of z

Definition at line 45 of file activators.cpp.

                                       {
     return z > 0 ? z : 0;
 }

◆ relu_derivative()

double gpmp::ml::Activation::relu_derivative ( double z )

Computes the derivative of the ReLU (Rectified Linear Unit) activation function.

Parameters

z	The input value

Returns: The derivative of ReLU at z

Definition at line 49 of file activators.cpp.

                                                  {
     return z > 0 ? 1 : 0;
 }

◆ selu()

double gpmp::ml::Activation::selu	(	double	z,
		double	alpha = `167326`,
		double	scale = `10507`
	)

Computes the Scaled Exponential Linear Unit (SELU) activation function.

Parameters

z	The input value
alpha	The SELU parameter (default: 167326)
scale	The SELU parameter (default: 10507)

Returns: The SELU of z

Definition at line 134 of file activators.cpp.

                                                                   {
     return scale * (z >= 0 ? z : alpha * (exp(z) - 1));
 }

◆ selu_derivative()

double gpmp::ml::Activation::selu_derivative	(	double	z,
		double	alpha = `167326`,
		double	scale = `10507`
	)

Computes the derivative of the Scaled Exponential Linear Unit (SELU) activation function.

Parameters

z	The input value
alpha	The SELU parameter (default: 167326)
scale	The SELU parameter (default: 10507)

Returns: The derivative of SELU at z

Definition at line 139 of file activators.cpp.

                                                                       {
     return scale * (z >= 0 ? 1 : alpha * exp(z));
 }

◆ sigmoid()

double gpmp::ml::Activation::sigmoid ( double z )

Computes the sigmoid activation function.

Parameters

z	The input value

Returns: The sigmoid of z

Definition at line 36 of file activators.cpp.

                                          {
     return 1.0 / (1.0 + exp(-z));
 }

◆ sigmoid_derivative()

double gpmp::ml::Activation::sigmoid_derivative ( double z )

Computes the derivative of the sigmoid activation function.

Parameters

z	The input value

Returns: The derivative of sigmoid at z

Definition at line 40 of file activators.cpp.

                                                     {
     double sig = sigmoid(z);
     return sig * (1.0 - sig);
 }

◆ silu()

double gpmp::ml::Activation::silu ( double z )

Computes the Sigmoid Linear Unit (SiLU or Swish) activation function.

Parameters

z	The input value

Returns: The SiLU of z

Definition at line 159 of file activators.cpp.

                                       {
     return z / (1 + exp(-z));
 }

◆ silu_derivative()

double gpmp::ml::Activation::silu_derivative ( double z )

Computes the derivative of the Sigmoid Linear Unit (SiLU or Swish) activation function.

Parameters

z	The input value

Returns: The derivative of SiLU at z

Definition at line 163 of file activators.cpp.

                                                  {
     double sig_z = sigmoid(z);
     return sig_z + (1 - sig_z) * z * sig_z;
 }

◆ smht()

double gpmp::ml::Activation::smht ( double z )

Computes the Soboleva modified hyperbolic tangent (smht) activation function.

Parameters

z	The input value

Returns: The smht of z

Definition at line 97 of file activators.cpp.

                                       {
     return z / (1.0 + exp(-z));
 }

◆ smht_derivative()

double gpmp::ml::Activation::smht_derivative ( double z )

Computes the derivative of the Soboleva modified hyperbolic tangent (smht) activation function.

Parameters

z	The input value

Returns: The derivative of smht at z

Definition at line 101 of file activators.cpp.

                                                  {
     double smht_z = smht(z);
     return smht_z * (1.0 - smht_z);
 }

◆ softmax()

std::vector< double > gpmp::ml::Activation::softmax ( const std::vector< double > & inputs )

Computes the softmax activation function.

Parameters

inputs The input values

Returns: The softmax of inputs

Definition at line 54 of file activators.cpp.

                                                          {
     std::vector<double> result;
     double sum_exp = 0.0;
     for (double input : inputs) {
         sum_exp += exp(input);
     }
     for (double input : inputs) {
         result.push_back(exp(input) / sum_exp);
     }
     return result;
 }

◆ softmax_derivative()

std::vector< std::vector< double > > gpmp::ml::Activation::softmax_derivative ( const std::vector< double > & inputs )

Computes the derivative of the softmax activation function.

Parameters

inputs The input values

Returns: The derivatives of softmax at inputs

Definition at line 67 of file activators.cpp.

                                                                     {
     std::vector<std::vector<double>> result(
         inputs.size(),
         std::vector<double>(inputs.size(), 0.0));
     std::vector<double> softmax_values = softmax(inputs);
     for (size_t i = 0; i < inputs.size(); ++i) {
         for (size_t j = 0; j < inputs.size(); ++j) {
             if (i == j) {
                 result[i][j] = softmax_values[i] * (1.0 - softmax_values[i]);
             } else {
                 result[i][j] = -softmax_values[i] * softmax_values[j];
             }
         }
     }
     return result;
 }

◆ softplus()

double gpmp::ml::Activation::softplus ( double z )

Computes the softplus activation function.

Parameters

z	The input value

Returns: The softplus of z

Definition at line 122 of file activators.cpp.

                                           {
     return log(1.0 + exp(z));
 }

◆ tanh()

double gpmp::ml::Activation::tanh ( double z )

Computes the hyperbolic tangent (tanh) activation function.

Parameters

z	The input value

Returns: The tanh of z

Definition at line 88 of file activators.cpp.

                                       {
     return std::tanh(z);
 }

◆ tanh_derivative()

double gpmp::ml::Activation::tanh_derivative ( double z )

Computes the derivative of the hyperbolic tangent (tanh) activation function.

Parameters

z	The input value

Returns: The derivative of tanh at z

Definition at line 92 of file activators.cpp.

                                                  {
     double tanh_z = std::tanh(z);
     return 1.0 - tanh_z * tanh_z;
 }

The documentation for this class was generated from the following files:

include/openGPMP/ml/activators.hpp
modules/ml/activators.cpp

Public Member Functions

Detailed Description

Member Function Documentation

◆ binary_step()

◆ elu()

◆ elu_derivative()

◆ gelu()

◆ gelu_derivative()

◆ leaky_relu()

◆ leaky_relu_derivative()

◆ prelu()

◆ prelu_derivative()

◆ relu()

◆ relu_derivative()

◆ selu()

◆ selu_derivative()

◆ sigmoid()

◆ sigmoid_derivative()

◆ silu()

◆ silu_derivative()

◆ smht()

◆ smht_derivative()

◆ softmax()

◆ softmax_derivative()

◆ softplus()

◆ tanh()

◆ tanh_derivative()