Case_study_4_3

Train a vanilla two-layer neural network for a regression problem using SGD.

!wget --no-cache -O init.py -q https://raw.githubusercontent.com/jdariasl/OTBD/main/content/init.py
import init; init.init(force_download=False)

import numpy as np
import matplotlib.pyplot as plt
from local.lib.utils import forward, backward

#!sudo apt install cm-super dvipng texlive-latex-extra texlive-latex-recommended

# Main parameters
mu=.01                 # Step size
Ns=10000              # Number of samples
Nh=5                   # Number of neurons hidden layer
Ni=1                   # Number of inputs
No=1                   # Number of outputs
In=3
# Defining the input and the desired signals
x=(np.random.rand(Ns)-.5)*4
y=1+np.sin(In*np.pi/4*x);
# Defining the variables (weights and bias)
W1=np.zeros((Nh,Ni,Ns+1))            # Weights hidden layer
W2=np.zeros((No,Nh,Ns+1))            # Weights output layer
W1[:,:,0]=np.random.rand(Nh,Ni)      # Initialization
W2[:,:,0]=np.random.rand(No,Nh)      # Initialization
b1=np.zeros((Nh,Ns+1))               # Bias hidden layer
b1[:,0]=(np.random.rand(Nh)-.5)*4    # Iitialization
b2=np.zeros((No,Ns+1));              # Bias output layer
b2[:,0]=(np.random.rand(No)-.5)*4    # Initialization
tipo='linear'                        # Output nonlinearity
e=np.zeros(Ns);                      # Error signal

# Loop along the samples including the forward and backward steps
for k in range(Ns):
    z0=[x[k]]
    z1,z2,_,_=forward(W1[:,:,k],W2[:,:,k],b1[:,k],b2[:,k],z0,tipo)
    e[k]=y[k]-z2
    delta2, delta1=backward(W2[:,:,k],z1,z2,e[k],tipo)
    W2[:,:,k+1]=W2[:,:,k]+2*mu*delta2*z1.T
    b2[0,k+1]=b2[0,k]+mu*2*delta2
    W1[:,:,k+1]=W1[:,:,k]+mu*2*delta1*z0
    b1[:,k+1]=b1[:,k]+mu*2*delta1.flatten()

# How to present results    
test=np.arange(-2,2,.02)
reg=np.zeros(test.shape);
for k in range(len(test)):
    _, temp, _, _ =forward(W1[:,:,Ns],W2[:,:,Ns],b1[:,Ns],b2[0,Ns],[test[k]],'linear');
    reg[k] = temp
plt.rcParams['text.usetex'] = True
plt.plot(test,1+np.sin(In*np.pi/4*test),color='r',label='target')
plt.plot(test,reg, color = 'b', linestyle = 'dashed',label = 'prediction')
plt.legend()
plt.grid()
plt.xlabel(r'$x$')
plt.ylabel(r'$y$')
plt.title(f'Error: q=1,Nh={Nh}')
plt.show()

plt.plot(e**2)
plt.xlabel('Iterations')
plt.ylabel('MSE')
plt.title('Instantaneous loss function')
plt.show()