Activation functions

# -*- coding: utf-8 -*-
"""activation-functions-for-deep-learning.ipynb
 
Automatically generated by Colaboratory.
 
Original file is located at
    https://colab.research.google.com/drive/1VgnoWZiglcLr_3JI2obyb73Nb3j8ioVy
 
# Most popular activation functions for deep learning
 
Source code for creating the figure for each activation function. 
More detail on this page: [Most popular activation functions for deep learning](https://lucidar.me/en/neural-networks/most-popular-activation-functions-for-deep-learning/).
"""
 
import numpy as np
import math
import matplotlib.pyplot as plt
 
def plot (x,y,dfdx, title):
  # Plot function
  fig, ax = plt.subplots()
  f = ax.plot (x, y, label=title, linewidth=3)
  df = ax.plot (x, dfdx, '--', label='Derivative - df/dt', linewidth=2.5)
 
 
  fig = plt.gcf()
  plt.grid()
  # Set figure size in inches
  fig.set_size_inches(16, 9)
  leg = ax.legend(loc='upper left');
 
  plt.title(title)
  plt.xlabel('x')
  plt.ylabel('y')
  plt.figure(facecolor="white")
 
  ax.axhline(y=0, color='k')
  ax.axvline(x=0, color='k')
 
  # Set x data
x = np.linspace(-6,6,100)
 
# Linear activation function
def linear_function(x):
  return x
 
# Derivative of the linear activation function
def linear_derivative(x):
  return [1] * len(x)
 
plot (x,linear_function(x), linear_derivative(x), 'Linear activation function')
 
# Set x data
x = np.linspace(-8,8,100)
 
# Sigmoid activation function
def sigmoid_function(x):
  return 1/(1+np.exp(-x))
 
# Derivative of the sigmoid activation function
def sigmoid_derivative(x):
  return np.exp(-x) / (1+ np.exp(-x))**2
 
plot (x,sigmoid_function(x), sigmoid_derivative(x), 'Sigmoid activation function')
 
# Set x data
x = np.linspace(-6,6,100)
 
# Tanh activation function
def tanh_function(x):
  return np.tanh(x)
 
# Derivative of the tanh activation function
def tanh_derivative(x):
  return 1 - np.tanh(x)**2
 
plot (x,tanh_function(x), tanh_derivative(x), 'Tanh activation function')
 
    # Set x data
x = np.linspace(-2,2,200)
 
# ReLU activation function
def ReLU_function(x):
  return np.where(x <= 0, 0, x)
 
# Derivative of the ReLU activation function
def ReLU_derivative(x):
  return np.where(x <= 0, 0, 1)
 
plot (x, ReLU_function(x), ReLU_derivative(x), 'ReLU activation function')
 
# Set x data
x = np.linspace(-5,2,200)
 
# Leaky ReLU activation function
def leakyReLU_function(x):
  return np.where(x <= 0, 0.01*x, x)
 
# Derivative of the leaky ReLU activation function
def leakyReLU_derivative(x):
  return np.where(x <= 0, 0.01, 1)
 
plot (x, leakyReLU_function(x), leakyReLU_derivative(x), 'Leaky ReLU activation function')
 
# Set x data
x = np.linspace(-5,2,200)
 
# Parameterised ReLU activation function
def parameterised_ReLU_function(x,a):
  return np.where(x <= 0, a*x, x)
 
# Derivative of the parameterised ReLU activation function
def parameterised_ReLU_derivative(x,a):
  return np.where(x <= 0, a, 1)
 
plot (x, parameterised_ReLU_function(x, 0.2), parameterised_ReLU_derivative(x, 0.2), 'Parameterised ReLU activation function (a=0.2)')
 
# Set x data
x = np.linspace(-5,2,200)
 
# ELU activation function
def ELU_function(x,a):
  return np.where(x <= 0, a*(np.exp(x) - 1), x)
 
# Derivative of the ELU activation function
def ELU_derivative(x,a):
  return np.where(x <= 0, a*np.exp(x), 1)
 
plot (x, ELU_function(x, 1), ELU_derivative(x, 1), 'ELU activation function (a=1)')