-
Notifications
You must be signed in to change notification settings - Fork 0
/
nn_numpy.py
103 lines (78 loc) · 3.33 KB
/
nn_numpy.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
import numpy as np
from Layer_Dense import Layer_Dense
from Activation_ReLu import Activation_ReLu
from Activation_Softmax import Activation_Softmax
from loss import *
from Optimizer import *
from data import *
import matplotlib.pyplot as plt
# Create dataset
X, y = vertical.create_data(samples=10, classes=3)
plt.scatter(X[:, 0], X[:, 1], c=y, cmap='brg')
plt.show()
# Create Dense layer with 2 input features and 64 output values
dense1 = Layer_Dense(2, 10)
# Create ReLU activation (to be used with Dense layer):
activation1 = Activation_ReLu()
dense2 = Layer_Dense(10, 10)
activation2 = Activation_ReLu()
# Create second Dense layer with 64 input features (as we take output
# of previous layer here) and 3 output values (output values)
dense3 = Layer_Dense(10, 3)
# Create Softmax classifier's combined loss and activation
loss_activation = Activation_Softmax_Loss_CategoricalCrossentropy()
# Create optimizer, choose one
optimizer = Optimizer_Adam(learning_rate=0.05, decay=5e-7)
#optimizer = Optimizer_RMSprop(learning_rate=0.02, decay=1e-5, rho=0.999)
#optimizer = Optimizer_SGD(decay=8e-8, momentum=0.9)
#optimizer = Optimizer_Adagrad(decay=1e-4)
# Train in loop
for epoch in range(100):
# Perform a forward pass of our training data through this layer
dense1.forward(X)
# Perform a forward pass through activation function
# takes the output of first dense layer here
activation1.forward(dense1.output)
dense2.forward(activation1.output)
activation2.forward(dense2.output)
# Perform a forward pass through second Dense layer
# takes outputs of activation function of first layer as inputs
dense3.forward(activation2.output)
# Perform a forward pass through the activation/loss function
# takes the output of second dense layer here and returns loss
loss = loss_activation.forward(dense3.output, y)
# Calculate accuracy from output of activation2 and targets
# calculate values along first axis
# np.argmax return array of index refering to position of maximun value along axis 1
predictions = np.argmax(loss_activation.output, axis=1)
if len(y.shape) == 2:
y = np.argmax(y, axis=1)
# np.argmax return indexs of max value each row (axis 1)
# np.argmax return array of index refering to position of maximun value along axis 1
accuracy = np.mean(predictions==y)
# if not -> to inverse boolan output
# true -> false
# false -> true
# mod 100 = 0 -> false, if not -> true
if not epoch % 10:
print(f'epoch: {epoch}, ' +
f'acc: {accuracy:.3f}, ' +
f'loss: {loss:.3f}, ' +
f'lr: {optimizer.current_learning_rate}')
# Backward pass
# using derivative chain rule from error -> Dloss_function output ->
# -> Dcategorical_function output an so on
loss_activation.backward(loss_activation.output, y)
dense3.backward(loss_activation.dinputs)
activation2.backward(dense3.dinputs)
dense2.backward(activation2.dinputs)
activation1.backward(dense2.dinputs)
dense1.backward(activation1.dinputs)
# Update weights and biases
# initialize preupdate params
optimizer.pre_update_params()
#the order can be changed
optimizer.update_params(dense2)
optimizer.update_params(dense3)
optimizer.update_params(dense1)
optimizer.post_update_params()