#!/usr/bin/env python
# coding: utf-8

# In[ ]:


import matplotlib.pyplot as plt
import numpy as np


#creat data set for traning
#xi(i=1,2,3) belong to -1 to 1
#if x1>=0.05,x2<=-0.15,y=[1 0 0]'
#if x2>-0.05,x3>0.1,y=[0 1 0]'
#else y=[0 0 1]'
#the size of data set 'm' is 1000
#define a1 is the value of input layer


m=1000
a1=2*np.random.random((3,m))-1
y=np.zeros((3,m))

for i in range(m):
    if a1[0,i]>=0.05 and a1[1,i]<=-0.15:
        y[0,i]=1
    elif a1[1,i]>=-0.05 and a1[2,i]>=0.1:
        y[1,i]=1
    else:
        y[2,i]=1



        
#I choose neural networks with one hidden layer with 5 
#every layer's parameter(neuron ‘a’ or parameter ’theta‘) is defined as only one single matrix
#define ai is the neuron of ith layer
#define bai is the ai combine with bias unit of ith layber
#define thetai is the theta of ith layer
#define bthetai is the thetai with the bias theta of ith layer
#define y1 is the the value of output layer
#define h(theta) is hypothesis fuction 
#and then calculate all parameter of first time


#define hypothesis function
h=lambda Z:1/(1+np.exp(-Z))

#calculate input layer
ba1=np.concatenate((np.ones((1,m)),a1),0)   
theta1=np.random.random((3,5))/100
btheta1=np.concatenate((np.ones((1,5)),theta),0)

#calculate hidden layer
a2=h(btheta1.T.dot(ba1))
ba2=np.concatenate((np.ones((1,m)),a2),0)
theta2=np.random.random((5,3))/100
btheta2=np.concatenate((np.ones((1,3)),theta2),0)

#calculate output layer
y1=h(btheta2.T.dot(ba2))





#define iteration is the iteration times
#define k is regulization coinfidence
#define J(theta) is cost fuction 
#define deltai is the delta of ith layer
#define Di is the derivative of j with respect to the theta of ith layer
#D is also a matrix with high deminsion
#define bDi is Di combine with J's derivative to bias theta
#define alpha is the learning rate


iteration=1000
k=10
J=np.zeros((1,iteration))
alpha=0.01

for j in range(iteration):
    
    #calculate cost function J
    J[0,j]=-1/m*np.sum(y*np.log(y1)+(1-y)*np.log(y1))+1/(2*m)*(np.sum(k*theta1)+np.sum(k*theta2))
    
    #calculate delta
    delta3=y1-y
    bdelta3=delta3
    delta2=theta2.dot(delta3)*a2*(1-a2)
    bdelta2=btheta2.dot(delta3)*ba2*(1-ba2)
    
    #calculate derivative
    bD2=1/m*ba2.dot(delta3.T)
    bD1=1/m*ba1.dot(delta2.T)
    bD1[0,:]=0
    bD2[0,:]=0
    change1=k*btheta1
    change1[1:4,:]=0
    change2=k*btheta2
    change2[1:6,:]=0
    bD1=bD1+change1
    bD2=bD2+change2
    
    #update the theta by using gradient decent algorithom with devivative value 
    theta1=theta1-alpha*bD1[1:4,:]
    theta2=theta2-alpha*bD2[1:6,:]
    btheta1=btheta1-alpha*bD1
    btheta2=btheta2-alpha*bD2

    

#xx is the sequence of 1,2,3....,iteration
#plot the J changing with times of iteration
#check the scatter to find whether the algorithom reaches convergence


xx=np.arange(iteration)+1
xx=xx[np.newaxis,:]
plt.scatter(yy,J)
plt.plot()




#use the training data to test the answer
#output the test answer and theta


test_a1=np.array([[0.2],[0.3],[0.5]])
test_ba1=np.concatenate((np.ones((1,1)),test_a1),0)
test_a2=h(btheta1.T.dot(test_ba1))
test_ba2=np.concatenate((np.ones((1,1)),test_a2),0)
test_y1=h(btheta2.T.dot(test_ba2))
print(test_y1)
print(btheta1)
print(btheta2)


# In[ ]:





# In[ ]:




