Universal approximation

# Downloading dependencies:
import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.optim as optim
 
### Setting function to approximate: *******************   #
 
# Actual Function here, Relationship: y = x^2
x = np.linspace(-30,30,100)
y = x**2
 
# Feel free to change Actual Function whatever u want to try: Ex: y=sin(x)
#x = np.linspace(0,180,200)
#y = np.sin(np.deg2rad(x))
 
 
#   ****************************************************   #
 
### Setting up the feedfoward neural network  **********   #
 
# Hyperparamters to tune: Number of Neurons and Hidden Layers, Learning Rate and Epochs.
n_neurons = 20  # number of neurons/nodes
learning_rate = 5e-3 # learning rate
 
 
model = nn.Sequential(     
          nn.Linear(1, n_neurons),
          nn.ReLU(),
          #nn.Linear(n_neurons,n_neurons),
          #nn.ReLU(),        
          nn.Linear(n_neurons,1),
          nn.ReLU()
          )
 
 
# Set up  : Input (1 Node) -> Hidden (10 nodes) -> Output (1 Node) 
# Set up 2: Input (1 Node) -> Hidden (10 nodes) -> Hidden (10 nodes) -> Output (1 Node)
 
# Important Note: If you increase the number of neurons or use a harder function to approximate, try tuning the learning rate.
#                 Tuning the learning rate is vital to properly train the network.
 
optimizer = optim.RMSprop(model.parameters(), lr=learning_rate) # define optimizer
#optimizer = optim.SGD(model.parameters(), lr=learning_rate)
 
criterion = nn.MSELoss() # define loss function
 
# ******************************************************  #
 
### Training: ******************************************  #
 
# Convert to tensor form with batch for PyTorch model.
inputs = torch.tensor(x).view(-1,1)
labels = torch.tensor(y).view(-1,1)
 
# Important Note 2: Change epochs
epochs = 20000
for epoch in range(epochs):  # loop over the data multiple times
 
    # zero the parameter gradients
    optimizer.zero_grad()
 
    # forward + backward + optimize
    outputs = model(inputs.float())
    loss = criterion(outputs, labels.float())
    loss.backward()
    optimizer.step()
 
# ******************************************************  #
 
### Running Inference over the trained model ***********  #
with torch.no_grad():
    test_inputs = torch.tensor(x).view(len(x),-1).float()
    y_hat = model(test_inputs)
    y_hat = y_hat.detach().numpy()
 
# ******************************************************  #
 
### Plot results: Actual vs Model Prediction ***********  #
plt.scatter(x,y,label='Actual Function')
plt.scatter(x,y_hat,label="Predicted Function")
plt.title(f'Number of neurons : {n_neurons} - Number of epoch : {epochs}')
plt.xlabel('Input Variable (x)')
plt.ylabel('Output Variable (y)')
plt.legend()
plt.show()
# ******************************************************  #
 

# Universal Approximation
 
import numpy as np
import math
import matplotlib.pyplot as plt
import random
import torch
 
from tqdm import tqdm
 
dtype = torch.float
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Using device:', device)
print()
 
#While working with GPUs
if device.type == 'cuda':
    print(torch.cuda.get_device_name(0))
    print('Memory Usage:')
    print('Allocated:', round(torch.cuda.memory_allocated(0)/1024**3,1), 'GB')
    print('Cached:   ', round(torch.cuda.memory_cached(0)/1024**3,1), 'GB')
 
# Model
 
def parameter_initialize(hidden): 
    '''Initialize parameters W and b'''
    #Input Layer to Hidden 
    W_1 = torch.randn(hidden,1, dtype = dtype, device=device) - 0.5
    b_1 = torch.randn(hidden,1, dtype = dtype, device=device) - 0.5
 
    #Hidden Layer to output
    W_2 = torch.randn(1,hidden, dtype = dtype, device=device) - 0.5
    b_2 = torch.randn(1,1, dtype = dtype, device=device) - 0.5
    return W_1, b_1, W_2, b_2
 
def forward(a_0, W_1, W_2, b_1, b_2): 
    '''Forward Propagation Function'''
    n_1 = W_1.mm(a_0) + b_1
    a_1 = []
 
    for i in range(n_1.shape[0]):
          a_1.append(float(1)/(1+math.exp(-n_1[i])))
 
    a_1 = torch.tensor(a_1, device=device).float().reshape(-1,1)
 
    n_2 = W_2.mm(a_1) + b_2
    a_2 = n_2
 
    return a_2,n_2,a_1,n_1
 
def backward(error, a_1, W_2): 
    '''Backpropagation Function'''
    s_2 = -2*1*error
    F_1 = []
    for i in range(a_1.shape[0]):
        F_1.append((1-a_1[i][0])*a_1[i][0])
    F_1 = torch.tensor(F_1, device=device).float().reshape(-1)
    F_1 = torch.diag(F_1)
 
    s_1 = F_1.mm(W_2.t())
    s_1 = s_1.mm(s_2)
    return s_1, s_2
 
def Update(W_1, W_2, b_1, b_2, lr, s_1, s_2, a_1, a_0 ): 
    '''Weight and Bias Updation'''
    W_2_new = W_2 - lr*s_2.mm(a_1.t())
    b_2_new = b_2 - lr*s_2
    W_1_new = W_1 - lr*s_1.mm(a_0.t())
    b_1_new = b_1 - lr*s_1
    return W_2_new, W_1_new, b_2_new, b_1_new
 
# Plotting Functions
 
def plot_error(lr,s,epoch,error):
    plt.figure(figsize = (10,5))
    plt.plot(epoch, error, c='blue', label=str(lr), alpha = 0.3)
    plt.xlabel('Epoch')
    plt.ylabel('Mean squared error')
    plt.legend(loc='best')
    plt.title('Mean Squared Error / {} neurons'.format(s))
    plt.show()
 
def plot_function(input,output):
    plt.figure(figsize = (10,5))
    input = input.numpy()
    plt.plot(input,output, c='green', label='function')
    plt.xlabel('data')
    plt.ylabel('Output')
    plt.legend(loc='best')
 
def plot_network(W_2, W_1, b_2, b_1, input, epochs):
    output_list = []
    for x in input:
        x = x.reshape(1,1)
        output, n_2, a_1, n_1 = forward(x,W_1,W_2,b_1,b_2)
        output = output[0]
        output_list.append(output)
    input = input.numpy()
    plt.plot(input,output_list,c='orange',label='Network')
    plt.title('Function vs Network after {} epochs'.format(epochs))
    plt.grid(True)
    plt.legend(loc='best')
    plt.show()
 
# Function to approximate
# 1. Cosine Function
 
data = torch.linspace(-3,3,100,dtype = dtype, device=device).reshape(100,1)
 
def approximate_function_1(a): 
    return math.cos(a)
 
var = []
for x in data.reshape(-1):
    var.append(approximate_function_1(x))
 
plot_function(data,var)
plt.savefig('images/cosine.jpeg')
 
# 2. Sine Function
 
def approximate_function_2(a): 
    return math.sin(a)
 
var_1 = []
for x in data.reshape(-1):
    var_1.append(approximate_function_2(x))
 
plot_function(data,var_1)
plt.savefig('images/sine.jpeg')
 
# Training
 
epochs=1000
s = 100
lr = 0.01
 
# 1. Cosine Function Approximation
 
error_list = []
epoch_list = []
 
W_1, b_1, W_2, b_2 = parameter_initialize(s)  # Parameter initialize
 
for epoch in tqdm(range(epochs)):
    epoch_list.append(epoch)
    p_1 = torch.linspace(-3,3,100,dtype = dtype, device=device).reshape(100,1)
    sum = 0
    for iter in range(p_1.shape[0]):
 
        a_0 = p_1[iter][0].reshape(1, 1)
 
        output, n_2, a_1, n_1 = forward(a_0, W_1, W_2, b_1, b_2)
        target = approximate_function_1(a_0)
        error = target - output
        square_error = (error.reshape(-1)[0]) * (error.reshape(-1)[0])
        sum += square_error
        s_1, s_2 = backward(error, a_1, W_2)
        W_2, W_1, b_2, b_1 = Update(W_1, W_2, b_1, b_2, lr, s_1, s_2, a_1, a_0)
    mean_square_error = sum/p_1.shape[0]
    #print("Epoch {} --> MSE: {}".format(epoch + 1,mean_square_error))
    error_list.append(mean_square_error)
 
y = []
 
for x in p_1.reshape(-1):
    y.append(approximate_function_1(x))
 
print('Final MSE : {}'.format(mean_square_error))
 
plot_error(lr, s, epoch_list, error_list)
 
plot_function(p_1,y)
plot_network(W_2, W_1, b_2, b_1, p_1, epochs)
 
# Sine Function Approximation
 
error_list = []
epoch_list = []
 
W_1, b_1, W_2, b_2 = parameter_initialize(s)  # Parameter initialize
 
for epoch in tqdm(range(epochs)):
    epoch_list.append(epoch)
    p_2 = torch.linspace(-3,3,100,dtype = dtype, device=device).reshape(100,1)
    sum = 0
    for iter in range(p_2.shape[0]):
 
        a_0 = p_2[iter][0].reshape(1, 1)
 
        output, n_2, a_1, n_1 = forward(a_0, W_1, W_2, b_1, b_2)
        target = approximate_function_2(a_0)
        error = target - output
        square_error = (error.reshape(-1)[0]) * (error.reshape(-1)[0])
        sum += square_error
 
        s_1, s_2 = backward(error, a_1, W_2)
        W_2, W_1, b_2, b_1 = Update(W_1, W_2, b_1, b_2, lr, s_1, s_2, a_1, a_0)
    mean_square_error = sum/p_2.shape[0]
    #print("Epoch {} --> MSE: {}".format(epoch + 1,mean_square_error))
    error_list.append(mean_square_error)
 
y = []
 
for x in p_2.reshape(-1):
    y.append(approximate_function_2(x))
 
print('Final MSE : {}'.format(mean_square_error))
 
plot_error(lr, s, epoch_list, error_list)
 
plot_function(p_2,y)
plot_network(W_2, W_1, b_2, b_1, p_2, epochs)
 
# Custom Function
 
def approximate_function_3(a): 
    return 10*pow(a,2)*math.sin(a)*math.cos(a)
 
var_3 = []
for x in data.reshape(-1):
    var_3.append(approximate_function_3(x))
 
plot_function(data, var_3)
 
error_list = []
epoch_list = []
 
W_1, b_1, W_2, b_2 = parameter_initialize(s)  # Parameter initialize
 
for epoch in tqdm(range(epochs)):
    epoch_list.append(epoch)
    p_3 = torch.linspace(-3,3,100,dtype = dtype, device=device).reshape(100,1)
    sum = 0
    for iter in range(p_3.shape[0]):
 
        a_0 = p_3[iter][0].reshape(1, 1)
 
        output, n_2, a_1, n_1 = forward(a_0, W_1, W_2, b_1, b_2)
        target = approximate_function_3(a_0)
        error = target - output
        square_error = (error.reshape(-1)[0]) * (error.reshape(-1)[0])
        sum += square_error
 
        s_1, s_2 = backward(error, a_1, W_2)
        W_2, W_1, b_2, b_1 = Update(W_1, W_2, b_1, b_2, lr, s_1, s_2, a_1, a_0)
    mean_square_error = sum/p_3.shape[0]
    #print("Epoch {} --> MSE: {}".format(epoch + 1,mean_square_error))
    error_list.append(mean_square_error)
 
y = []
 
for x in p_3.reshape(-1):
    y.append(approximate_function_3(x))
 
plot_error(lr, s, epoch_list, error_list)
 
plot_function(p_3,y)
plot_network(W_2, W_1, b_2, b_1, p_3, epochs)