diff --git a/Datasets.py b/Datasets.py
new file mode 100644
index 0000000..a4b55cb
--- /dev/null
+++ b/Datasets.py
@@ -0,0 +1,31 @@
+import numpy as np
+from enum import Enum
+
+
+class Dataset(Enum):
+ CIRCLE = "circle",
+ SPIRAL = "spiral"
+
+
+def get_circle_dataset(points: int, min_range: float, max_range: float, radius: float):
+ # generating labelled training data
+ range_ = max_range-min_range
+ N = points
+ X = (np.random.rand(N, 2) * range_) + min_range
+ y = np.sqrt(np.sum(np.multiply(X, X), axis=1)) > radius
+ return np.stack((X[:, 0], X[:, 1], y), axis=1)
+
+
+def get_spiral_dataset(points: int, classes: int):
+ N = points # number of points per class
+ D = 2 # dimensionality
+ K = classes # number of classes
+ X = np.zeros((N * K, D)) # data matrix (each row = single example)
+ y = np.zeros(N * K, dtype='uint8') # class labels
+ for j in range(K):
+ ix = range(N * j, N * (j + 1))
+ r = np.linspace(0.0, 1, N) # radius
+ t = np.linspace(j * 4, (j + 1) * 4, N) + np.random.randn(N) * 0.2 # theta
+ X[ix] = np.c_[r * np.sin(t), r * np.cos(t)]
+ y[ix] = j
+ return np.stack((X[:, 0], X[:, 1], y), axis=1)
diff --git a/NetworkUtils.py b/NetworkUtils.py
new file mode 100644
index 0000000..afd3581
--- /dev/null
+++ b/NetworkUtils.py
@@ -0,0 +1,167 @@
+import os
+import numpy as np
+
+from Datasets import Dataset
+from RunningStats import RunningStats
+
+
+def shuffle_dataset(dataset: np.ndarray) -> (np.ndarray, np.ndarray, np.ndarray, np.ndarray):
+ np.random.shuffle(dataset)
+ size = int(dataset.shape[0]*0.1)
+ train_set = dataset[size:, :-1]
+ train_labels = dataset[size:, -1].astype("int32")
+ test_set = dataset[:size, :-1]
+ test_labels = dataset[:size, -1].astype("int32")
+
+ return train_set, train_labels, test_set, test_labels
+
+
+def sigmoid(x: np.ndarray) -> np.ndarray:
+ return 1.0 / (1.0 + np.exp(-x))
+
+
+def softmax(f: np.ndarray) -> np.ndarray:
+ exp_f = np.exp(f)
+ return exp_f / np.sum(exp_f, axis=1, keepdims=True)
+
+
+def xw_plus_b(x: np.ndarray, w: np.ndarray, b: np.ndarray) -> np.ndarray:
+ return np.matmul(x, w) + b
+
+
+def select_neurons(network, p1_enabled) -> (np.ndarray, np.ndarray, np.ndarray, np.ndarray):
+ if p1_enabled:
+ w1 = np.stack([network["W1"][:, i] for i in range(network["W1"].shape[1]) if i in p1_enabled], axis=1)
+ b1 = np.stack([network["b1"][:, i] for i in range(network["b1"].shape[1]) if i in p1_enabled], axis=1)
+ w2 = np.stack([network["W2"][i, :] for i in range(network["W2"].shape[0]) if i in p1_enabled], axis=0)
+ b2 = network["b2"]
+ else:
+ w1 = network["W1"]
+ b1 = network["b1"]
+ w2 = network["W2"]
+ b2 = network["b2"]
+
+ return w1, b1, w2, b2
+
+
+def forward(inputs, network, dataset: Dataset, p1_enabled=None, precision=None, squeeze=False):
+ w1, b1, w2, b2 = select_neurons(network, p1_enabled)
+ out1 = xw_plus_b(inputs, w1, b1)
+ out2 = np.maximum(out1, 0)
+ if squeeze and dataset is Dataset.CIRCLE:
+ out3 = np.squeeze(xw_plus_b(out2, w2, b2), axis=1)
+ else:
+ out3 = xw_plus_b(out2, w2, b2)
+ if dataset is Dataset.CIRCLE:
+ out4 = sigmoid(out3)
+ elif dataset is Dataset.SPIRAL:
+ out4 = softmax(out3)
+ else:
+ raise Exception("Invalid dataset type")
+ if not precision:
+ return out1, out2, out3, out4
+ else:
+ return np.reshape(out1, newshape=[precision, precision, -1]), \
+ np.reshape(out2, newshape=[precision, precision, -1]), \
+ np.reshape(out3, newshape=[precision, precision, -1]), \
+ np.reshape(out4, newshape=[precision, precision, -1])
+
+
+def train_network_sigmoid(dataset: np.ndarray, learning_rate: float, units: int, window_size: int, log_freq=100):
+ stats = RunningStats(window_size)
+
+ network = {
+ "W1": np.random.randn(2, units).astype("float32"),
+ "b1": np.zeros((1, units)).astype("float32"),
+ "W2": np.random.randn(units, 1).astype("float32"),
+ "b2": np.zeros((1, 1)).astype("float32")
+ }
+
+ episode = 0
+ while not stats.finished_window() or stats.window_improved():
+ episode += 1
+
+ train_set, train_labels, test_set, test_labels = shuffle_dataset(dataset)
+
+ _, train_out2, _, train_predictions = forward(train_set, network, Dataset.CIRCLE, squeeze=True)
+ d = -(np.subtract(train_labels, train_predictions))
+ network["W1"] -= learning_rate * np.dot(np.multiply(d, np.multiply(train_out2.T > 0, network["W2"])), train_set).T
+ network["b1"] -= learning_rate * np.sum(np.multiply(d, np.multiply(train_out2.T > 0, network["W2"])), axis=1)
+ network["W2"] -= learning_rate * np.reshape(np.dot(d, np.maximum(train_out2, 0)), [units, 1])
+ network["b2"] -= learning_rate * np.reshape(np.sum(d), [1, 1])
+
+ _, _, _, test_predictions = forward(test_set, network, Dataset.CIRCLE)
+ test_accuracy = (len(np.where((test_predictions[:, 0] > 0.5) == (test_labels == 1))[0]) / len(test_labels)) * 100
+ stats.insert(test_accuracy)
+
+ if log_freq and episode % log_freq:
+ print(f"Episode: {episode}, Test Accuracy: {test_accuracy:6.2f}, Running Avg: {stats.get_average():6.3f}")
+
+ return network
+
+
+def train_network_softmax(dataset: np.ndarray, learning_rate: float, units: int, window_size: int, log=True):
+ stats = RunningStats(window_size)
+
+ reg = 1e-3
+ network = {
+ "W1": np.random.randn(2, units).astype("float32"),
+ "b1": np.zeros((1, units)).astype("float32"),
+ "W2": np.random.randn(units, 2).astype("float32"),
+ "b2": np.zeros((1, 2)).astype("float32")
+ }
+
+ num_examples = int(dataset.shape[0]*0.9)
+ episode = 0
+ while not stats.finished_window() or stats.window_improved():
+ episode += 1
+
+ train_set, train_labels, test_set, test_labels = shuffle_dataset(dataset)
+
+ _, train_out2, _, train_predictions = forward(train_set, network, Dataset.SPIRAL, squeeze=True)
+
+ dscores = train_predictions
+ dscores[range(num_examples), train_labels] -= 1
+ dscores /= num_examples
+
+ dhidden = np.dot(dscores, network["W2"].T)
+ dhidden[train_out2 <= 0] = 0
+ network["W1"] -= learning_rate * reg * np.dot(train_set.T, dhidden)
+ network["b1"] -= learning_rate * np.sum(dhidden, axis=0, keepdims=True)
+ network["W2"] -= learning_rate * reg * np.dot(train_out2.T, dscores)
+ network["b2"] -= learning_rate * np.sum(dscores, axis=0, keepdims=True)
+
+ _, _, _, test_predictions = forward(test_set, network, Dataset.SPIRAL)
+ test_accuracy = (len(np.where((test_predictions[:, 0] > 0.5) == (test_labels == 0))[0]) / len(test_labels)) * 100
+ stats.insert(test_accuracy)
+
+ if log:
+ print(f"Episode: {episode}, Test Accuracy: {test_accuracy:6.2f}, Running Avg: {stats.get_average():6.3f}")
+
+ return network
+
+
+def save_network(network: dict, dataset: Dataset, path: str):
+ units = network["W1"].shape[1]
+ if not os.path.isdir(f"{path}/dataset_{dataset.name}"):
+ os.mkdir(f"{path}/dataset_{dataset.name}")
+ os.mkdir(f"{path}/dataset_{dataset.name}/units_{units}")
+ for key in network.keys():
+ with open(f"{path}/dataset_{dataset.name}/units_{units}/{key}", "wb+") as file:
+ file.write(network[key].tobytes())
+
+
+def check_saved_network(units: int, dataset: Dataset, path: str):
+ return os.path.isdir(f"{path}/dataset_{dataset.name}/units_{units}")
+
+
+def load_network(units: int, dataset: Dataset, path: str, dtype="float32"):
+ classes = 1 if dataset is Dataset.CIRCLE else 2
+ network = {"W1": [2, units], "b1": [1, units], "W2": [units, classes], "b2": [1, classes]}
+ for key in network.keys():
+ with open(f"{path}/dataset_{dataset.name}/units_{units}/{key}", "rb") as file:
+ data = file.read(len(np.zeros(shape=network[key], dtype=dtype).tobytes()))
+ network[key] = np.reshape(np.frombuffer(data, dtype=dtype), network[key])
+ network[key].setflags(write=True)
+
+ return network
diff --git a/NetworkVisualisation.py b/NetworkVisualisation.py
new file mode 100644
index 0000000..d70ea5c
--- /dev/null
+++ b/NetworkVisualisation.py
@@ -0,0 +1,299 @@
+import matplotlib.pyplot as plt
+from PlotUtils import plot_to_grid
+from Plot import Plot
+from Plot3D import Plot3D
+from matplotlib.widgets import Slider, CheckButtons
+from Datasets import get_spiral_dataset, get_circle_dataset
+from NetworkUtils import *
+
+
+def scale(vector, axis=0, scale=1):
+ result = vector - np.min(vector, axis=axis)
+ result /= np.max(result, axis=0)
+ result *= scale
+ return result
+
+
+def setup_data_space(min_range, max_range, precision):
+ dims = 2
+ dim_data = [np.linspace(min_range, max_range, precision) for _ in range(dims)]
+ data_grid = np.meshgrid(*dim_data)
+ return np.stack([data_dim.flatten() for data_dim in data_grid], axis=1), dim_data
+
+
+def scale_out2(out2, w2, all_p1_enabled, perceptron2, precision):
+ reshaped_out2 = np.reshape(out2, [precision * precision, -1])
+ scaled_outs = np.stack([reshaped_out2[:, i] * w2[perceptron, perceptron2] for i, perceptron in enumerate(all_p1_enabled)], axis=1)
+ return np.reshape(scaled_outs, [precision, precision, -1])
+
+
+class NetworkVisualisation(object):
+
+ def __init__(self, units, data_points, min_range, max_range, quality, dataset, saves_path=None, seed=1):
+ np.random.seed(seed)
+
+ self.precision = quality
+ self.min_range = min_range
+ self.max_range = max_range
+ self.dataset_type = dataset
+ if dataset is Dataset.CIRCLE:
+ self.dataset = get_circle_dataset(points=data_points, min_range=min_range, max_range=max_range, radius=0.8)
+ elif dataset is Dataset.SPIRAL:
+ self.dataset = get_spiral_dataset(data_points, classes=2)
+ else:
+ raise Exception("Invalid dataset type")
+ self.data_points = self.dataset[:, :-1]
+ self.data_labels = self.dataset[:, -1]
+ self.data_space, self.dim_data = setup_data_space(min_range, max_range, quality)
+
+ # Network Creation
+ if saves_path and check_saved_network(units, dataset, saves_path):
+ self.network = load_network(units, dataset, saves_path)
+ else:
+ if dataset is Dataset.CIRCLE:
+ self.network = train_network_sigmoid(self.dataset, units=units, learning_rate=5e-3, window_size=1000)
+ elif dataset is Dataset.SPIRAL:
+ self.network = train_network_softmax(self.dataset, units=units, learning_rate=1, window_size=1000)
+ else:
+ raise Exception("Invalid dataset type")
+ save_network(self.network, dataset, saves_path)
+ self.default_network = dict(zip(self.network.keys(), [layer.copy() for layer in self.network.values()]))
+
+ # GUI Visualisation
+ self.perceptron1 = 0
+ self.is_relu = True
+ self.perceptron2 = 0
+ self.connection = 0
+ self.is_pre_add = False
+ self.is_sig = True
+ self.all_p1_enabled = set(range(self.network["W1"].shape[1]))
+ self.ignore_update = False
+
+ fig = plt.figure(figsize=(13, 6.5))
+ self.plot_network(fig)
+ self.plot_controls(fig)
+
+ def plot_network(self, fig):
+ _, out2, out3, out4 = forward(self.data_space, self.network, self.dataset_type, precision=self.precision)
+ outer_points = self.data_points[self.data_labels == 1]
+ inner_points = self.data_points[self.data_labels == 0]
+
+ self.layer1_plot = Plot(fig, (4, 4), (0, 0), (1, 3), out2[:, :, 0], self.min_range, self.max_range)
+ self.layer1_plot.ax.scatter(outer_points[:, 0], outer_points[:, 1], s=3, c="g", alpha=0.5)
+ self.layer1_plot.ax.scatter(inner_points[:, 0], inner_points[:, 1], s=3, c="r", alpha=0.5)
+
+ self.layer1_3d_plot = Plot3D(fig, (4, 4), (1, 0), (1, 3), self.precision, out2)
+
+ self.layer2_plot = Plot(fig, (4, 4), (2, 0), (1, 3), out4[:, :, 0], self.min_range, self.max_range)
+ self.layer2_plot.ax.scatter(outer_points[:, 0], outer_points[:, 1], s=3, c="g", alpha=0.5)
+ self.layer2_plot.ax.scatter(inner_points[:, 0], inner_points[:, 1], s=3, c="r", alpha=0.5)
+
+ self.layer2_3d_plot = Plot3D(fig, (4, 4), (3, 0), (1, 3), self.precision, out4)
+
+ def plot_controls(self, fig):
+ step_size = 0.01
+ padding = 5
+
+ # Plot 1 controls
+ w1x_min = self.network["W1"][0].min()
+ w1x_max = self.network["W1"][0].max()
+ w1x_diff = (w1x_max - w1x_min)/2 + padding
+
+ w1y_min = self.network["W1"][1].min()
+ w1y_max = self.network["W1"][1].max()
+ w1y_diff = (w1y_max - w1y_min) / 2 + padding
+
+ w1b_min = self.network["b1"].min()
+ w1b_max = self.network["b1"].max()
+ w1b_diff = (w1b_max - w1b_min) / 2 + padding
+
+ p1x_ax = plot_to_grid(fig, (2, 16), (0, 12), (1, 1))
+ self.p1x_slid = Slider(p1x_ax, 'P1 x', valmin=w1x_min-w1x_diff, valmax=w1x_max+w1x_diff, valinit=self.network["W1"][0, 0], valstep=step_size)
+ self.p1x_slid.on_changed(self.p1x_changed)
+
+ p1y_ax = plot_to_grid(fig, (2, 16), (0, 13), (1, 1))
+ self.p1y_slid = Slider(p1y_ax, 'P1 y', valmin=w1y_min-w1y_diff, valmax=w1y_max+w1y_diff, valinit=self.network["W1"][1, 0], valstep=step_size)
+ self.p1y_slid.on_changed(self.p1y_changed)
+
+ p1b_ax = plot_to_grid(fig, (24, 16), (0, 14), (7, 1))
+ self.p1b_slid = Slider(p1b_ax, 'P1 b', valmin=w1b_min-w1b_diff, valmax=w1b_max+w1b_diff, valinit=self.network["b1"][0, 0], valstep=step_size)
+ self.p1b_slid.on_changed(self.p1b_changed)
+
+ p1_ax = plot_to_grid(fig, (24, 16), (0, 15), (7, 1))
+ self.p1_slid = Slider(p1_ax, 'P1', valmin=0, valmax=self.network["W1"].shape[1]-1, valinit=self.perceptron1, valstep=1)
+ self.p1_slid.on_changed(self.p1_changed)
+
+ p1_opt_ax = plot_to_grid(fig, (24, 16), (8, 14), (3, 2))
+ self.p1_opt_buttons = CheckButtons(p1_opt_ax, ["ReLU?", "Enabled?"], [self.is_relu, True])
+ self.p1_opt_buttons.on_clicked(self.p1_options_update)
+
+
+ # Plot 2 Controls
+ w2_min = self.network["W2"].min()
+ w2_max = self.network["W2"].max()
+ w2_diff = (w2_max - w2_min) / 2 + padding
+
+ w2b_abs = np.abs(self.network["b2"][0, 0]) + padding
+ w2b_min = self.network["b2"][0, 0]-w2b_abs
+ w2b_max = self.network["b2"][0, 0]+w2b_abs
+
+ p2_weight_val_ax = plot_to_grid(fig, (2, 16), (1, 12), (1, 1))
+ self.p2_dim_val_slid = Slider(p2_weight_val_ax, 'p2 w', valmin=w2_min-w2_diff, valmax=w2_max+w2_diff, valinit=self.network["W2"][0, 0], valstep=step_size)
+ self.p2_dim_val_slid.on_changed(self.p2_weight_changed)
+
+ p2_connection_dim_ax = plot_to_grid(fig, (2, 16), (1, 13), (1, 1))
+ self.p2_connection_dim_slid = Slider(p2_connection_dim_ax, 'p2 c', valmin=0, valmax=self.network["W2"].shape[0]-1, valinit=0, valstep=1)
+ self.p2_connection_dim_slid.on_changed(self.p2_connection_dim_changed)
+
+ p2b_ax = plot_to_grid(fig, (24, 16), (13, 14), (7, 1))
+ self.p2b_slid = Slider(p2b_ax, 'p2 b', valmin=w2b_min, valmax=w2b_max, valinit=self.network["b2"][0, 0], valstep=step_size)
+ self.p2b_slid.on_changed(self.p2b_changed)
+
+ p2_opt_ax = plot_to_grid(fig, (24, 16), (21, 14), (4, 2))
+ self.p2_opt_buttons = CheckButtons(p2_opt_ax, ["Pre-add?", "Transform?"], [self.is_pre_add, self.is_sig])
+ self.p2_opt_buttons.on_clicked(self.p2_options_update)
+
+ def p1_changed(self, val):
+ self.perceptron1 = int(val)
+ self.ignore_update = True
+ self.update_widgets()
+ self.ignore_update = False
+
+ self.update_just_plot1()
+
+ def p1x_changed(self, val):
+ self.network["W1"][0, self.perceptron1] = val
+ self.update_visuals()
+
+ def p1y_changed(self, val):
+ self.network["W1"][1, self.perceptron1] = val
+ self.update_visuals()
+
+ def p1b_changed(self, val):
+ self.network["b1"][0, self.perceptron1] = val
+ self.update_visuals()
+
+ def p1_options_update(self, label):
+ if label == "ReLU?":
+ self.is_relu = not self.is_relu
+ self.update_just_plot1()
+ elif label == "Enabled?":
+ is_enabled = self.p1_opt_buttons.get_status()[1]
+ if is_enabled and self.perceptron1 not in self.all_p1_enabled:
+ self.all_p1_enabled.add(self.perceptron1)
+ elif not is_enabled and self.perceptron1 in self.all_p1_enabled:
+ layer1_out = sorted(list(self.all_p1_enabled)).index(self.perceptron1)
+ self.layer1_3d_plot.remove_plot(layer1_out)
+ self.all_p1_enabled.remove(self.perceptron1)
+
+ self.update_visuals()
+
+ def p2_weight_changed(self, val):
+ self.network["W2"][self.connection, 0] = val
+
+ self.update_just_plot2()
+
+ def p2_connection_dim_changed(self, val):
+ self.connection = int(val)
+ self.ignore_update = True
+ self.p2_dim_val_slid.set_val(self.network["W2"][self.connection, 0])
+ self.p2_dim_val_slid.vline.set_xdata(self.default_network["W2"][self.connection, 0])
+ self.ignore_update = False
+
+ def p2b_changed(self, val):
+ self.network["b2"][0, 0] = val
+ self.update_just_plot2()
+
+ def p2_options_update(self, label):
+ if label == "Transform?":
+ self.is_sig = not self.is_sig
+ elif label == "Pre-add?":
+ self.is_pre_add = not self.is_pre_add
+
+ self.update_just_plot2()
+
+ def show(self):
+ plt.show()
+
+ def update_plot1(self, out1, out2):
+ if self.perceptron1 in self.all_p1_enabled:
+ self.layer1_plot.set_visible(True)
+ layer1_out = sorted(list(self.all_p1_enabled)).index(self.perceptron1)
+ if not self.is_relu:
+ layer1_data = out1[:, :, layer1_out]
+ else:
+ layer1_data = out2[:, :, layer1_out]
+ self.layer1_plot.update(layer1_data)
+ else:
+ self.layer1_plot.set_visible(False)
+
+ def update_3d_plot1(self, out1, out2):
+ if self.perceptron1 in self.all_p1_enabled:
+ if not self.is_relu:
+ self.layer1_3d_plot.update_all(out1)
+ else:
+ self.layer1_3d_plot.update_all(out2)
+
+ def update_plot2(self, out2, out3, out4):
+ if self.is_pre_add:
+ layer2_data = scale_out2(out2, self.network["W2"], self.all_p1_enabled, self.perceptron2, self.precision)
+ layer2_data = np.sum(layer2_data, axis=2)
+ elif not self.is_sig:
+ layer2_data = out3[:, :, 0]
+ else:
+ layer2_data = out4[:, :, 0]
+ self.layer2_plot.update(layer2_data)
+
+ def update_3d_plot2(self, out2, out3, out4):
+ if self.is_pre_add:
+ layer2_data = scale_out2(out2, self.network["W2"], self.all_p1_enabled, self.perceptron2, self.precision)
+ elif not self.is_sig:
+ layer2_data = out3
+ else:
+ layer2_data = out4
+ self.layer2_3d_plot.update_all(layer2_data)
+
+ def update_visuals(self):
+ if not self.ignore_update:
+ out1, out2, out3, out4 = forward(self.data_space, self.network, self.dataset_type, self.all_p1_enabled, self.precision)
+ self.update_plot1_visuals(out1, out2)
+ self.update_plot2_visuals(out2, out3, out4)
+ plt.draw()
+
+ def update_just_plot1(self):
+ if not self.ignore_update:
+ out1, out2, out3, out4 = forward(self.data_space, self.network, self.dataset_type, self.all_p1_enabled, self.precision)
+ self.update_plot1_visuals(out1, out2)
+ plt.draw()
+
+ def update_plot1_visuals(self, out1, out2):
+ self.update_plot1(out1, out2)
+ self.update_3d_plot1(out1, out2)
+
+ def update_just_plot2(self):
+ if not self.ignore_update:
+ out1, out2, out3, out4 = forward(self.data_space, self.network, self.dataset_type, self.all_p1_enabled, self.precision)
+ self.update_plot2_visuals(out2, out3, out4)
+ plt.draw()
+
+ def update_plot2_visuals(self, out2, out3, out4):
+ self.update_plot2(out2, out3, out4)
+ self.update_3d_plot2(out2, out3, out4)
+
+ def update_widgets(self):
+ self.p1b_slid.set_val(self.network["b1"][0, self.perceptron1])
+ self.p1x_slid.set_val(self.network["W1"][0, self.perceptron1])
+ self.p1y_slid.set_val(self.network["W1"][1, self.perceptron1])
+
+ self.p1b_slid.vline.set_xdata(self.default_network["b1"][0, self.perceptron1])
+ self.p1x_slid.vline.set_xdata(self.default_network["W1"][0, self.perceptron1])
+ self.p1y_slid.vline.set_xdata(self.default_network["W1"][1, self.perceptron1])
+
+ if (self.perceptron1 in self.all_p1_enabled and not self.p1_opt_buttons.get_status()[1]) or \
+ (self.perceptron1 not in self.all_p1_enabled and self.p1_opt_buttons.get_status()[1]):
+ self.p1_opt_buttons.set_active(1)
+
+
+if __name__ == '__main__':
+ NetworkVisualisation(units=5, data_points=1000, min_range=-1, max_range=1, quality=100, saves_path="resources/Saves", dataset=Dataset.CIRCLE).show()
+ # NetworkVisualisation(units=24, data_points=1000, min_range=-1, max_range=1, quality=100, saves_path="resources/Saves", dataset=Dataset.SPIRAL).show()
diff --git a/Plot.py b/Plot.py
new file mode 100644
index 0000000..3d9391f
--- /dev/null
+++ b/Plot.py
@@ -0,0 +1,15 @@
+import numpy as np
+from PlotUtils import subplot
+
+
+class Plot(object):
+
+ def __init__(self, fig, plot, loc, shape, data, min_range, max_range):
+ self.plot, self.ax = subplot(fig, plot, loc, shape, data, min_range, max_range)
+
+ def update(self, data):
+ self.plot.set_data(data)
+ self.plot.set_clim([np.min(data), np.max(data)])
+
+ def set_visible(self, is_visible):
+ self.plot.set_visible(is_visible)
diff --git a/Plot3D.py b/Plot3D.py
new file mode 100644
index 0000000..2acd216
--- /dev/null
+++ b/Plot3D.py
@@ -0,0 +1,35 @@
+import numpy as np
+from PlotUtils import subplot_3d
+
+
+class Plot3D(object):
+
+ def __init__(self, fig, plot, loc, shape, precision, data):
+ self.precision = precision
+ self.plots, self.axes = subplot_3d(fig, plot, loc, shape, precision, data)
+
+ def update_plane(self, all_data, plane, index):
+ if self.plots[index]:
+ for cont in self.plots[index].collections:
+ cont.remove()
+ vmax = np.max(all_data)
+ vmin = np.min(all_data)
+ self.plots[index] = self.axes.contourf(range(self.precision), range(self.precision), plane, self.precision, vmin=vmin, vmax=vmax, cmap='plasma')
+
+ def update_all(self, all_data):
+ self.remove_all_plots()
+ vmax = np.max(all_data)
+ vmin = np.min(all_data)
+ self.plots = [None]*all_data.shape[2]
+ for i in range(all_data.shape[2]):
+ self.plots[i] = self.axes.contourf(range(self.precision), range(self.precision), all_data[:, :, i], self.precision, vmin=vmin, vmax=vmax, cmap='plasma')
+
+ def remove_all_plots(self):
+ for i in range(len(self.plots)):
+ self.remove_plot(i)
+
+ def remove_plot(self, index):
+ if self.plots[index]:
+ for cont in self.plots[index].collections:
+ cont.remove()
+ self.plots[index] = None
diff --git a/PlotUtils.py b/PlotUtils.py
new file mode 100644
index 0000000..a4c12e1
--- /dev/null
+++ b/PlotUtils.py
@@ -0,0 +1,31 @@
+import numpy as np
+import matplotlib.gridspec as gridspec
+from mpl_toolkits.mplot3d import Axes3D # IS USED DO NOT REMOVE!!!
+
+
+def plot_to_grid(fig, plot, location, shape, is_3d=False):
+ gs = gridspec.GridSpec(*reversed(plot))
+ if is_3d:
+ return fig.add_subplot(gs[location[1]:location[1]+shape[1], location[0]:location[0]+shape[0]], projection='3d')
+ else:
+ return fig.add_subplot(gs[location[1]:location[1]+shape[1], location[0]:location[0]+shape[0]])
+
+
+def subplot_3d(fig, plot, location, shape, precision, data):
+ ax = plot_to_grid(fig, plot, location, shape, is_3d=True)
+ # ax.view_init(60, 0) - sets angle
+ vmax = np.max(data)
+ vmin = np.min(data)
+ ims = []
+ for i in range(data.shape[2]):
+ cnt = ax.contourf(range(precision), range(precision), data[:, :, i], precision, vmin=vmin, vmax=vmax, cmap='plasma', origin='lower')
+ ims.append(cnt)
+ ax.axis('off')
+ return ims, ax
+
+
+def subplot(fig, plot, location, shape, data, min_range, max_range):
+ ax = plot_to_grid(fig, plot, location, shape)
+ im = ax.imshow(data, interpolation='nearest', cmap='plasma', extent=[min_range, max_range, min_range, max_range], origin='lower')
+ ax.axis('off')
+ return im, ax
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..8d3d904
--- /dev/null
+++ b/README.md
@@ -0,0 +1,83 @@
+# 2-layer Network Visualisation
+
+A python application demonstrating how a 2-layer fully connected network with rectified
+linear units in the hidden layer interacts with the feature-space of a dataset.
+
+### Required Libraries
+- Numpy
+- Matplotlib
+
+### Datasets
+The application currently supports 2 features datasets with 2 classes max.
+Implemented datasets:
+- Circle: Uses a network trained with a sigmoid output
+
+
+- Spiral: Uses a network trained with a softmax output
+
+
+### Running the application
+To run the application you need:
+ - to clone this repo
+ - add `from NetworkVisualisation import NetworkVisualisation` to your file
+ - Create a `NetworkVisualisation` object
+ - Call the `show()` method.
+
+**Circle Dataset Sigmoid Network Example**
+```python
+vis = NetworkVisualisation(units=5, data_points=1000, min_range=-1, max_range=1, quality=100, saves_path="resources/Saves", dataset=Dataset.CIRCLE)
+vis.show()
+```
+
+**Spiral Dataset Softmax Network Example**
+```python
+vis = NetworkVisualisation(units=24, data_points=1000, min_range=-1, max_range=1, quality=100, saves_path="resources/Saves", dataset=Dataset.SPIRAL)
+vis.show()
+```
+
+**`NetworkVisualisation` Parameters:**
+- units: The amount of neurons in the hidden layer of the network
+- data_points: The amount of data points sampled from the dataset
+- min_range: The smallest dimensional value shown on the plots. (how much of the feature-space you see)
+- max_range: The largest dimensional value shown on the plots. (how much of the feature-space you see)
+- quality: The resolution of the plots. Larger values will slow down the application
+- dataset: The dataset enum. Dataset.CIRCLE or Dataset.SPIRAL
+- saves_path: Optional. This is so you don't have to retrain your network every time before the visualisation.
+- seed: Optional, The random seed used to sample network parameters.
+
+### Using the GUI
+
+
+The application offers an interface for adjusting network parameters through a series of sliders and check boxes.
+You can also change the viewing angle of the 3d plots by clicking and dragging.
+
+**GUI Layout**
+The layout is setup in two halves. The left half represents configuration of neurons in the hidden layer.
+And the right half represents the configuration of the output neuron.
+
+There are 4 plots. The plots are generated by sampling points from the feature-space, and generating outputs from a neuron/network. A heat map is used to show high(yellow) vs low(blue) output values.
+
+
+**Plots**
+- Plot 1 shows the feature-space outputs from one of the neurons in the hidden layer. The output dimension is flattened.
+- Plot 2 shows all the hyperplanes generated from the outputs of the neurons in the hidden layer.
+- Plot 3 shows the feature-space outputs from the entire network. Again with the output dimension flattened.
+- Plot 4 shows the same as plot 3 but in 3D space.
+
+**Slider Descriptions:**
+The sliders on the left half of the GUI correspond to the currently selected neuron, which is displayed in far left plot.
+- P1 x: The input connection weight for the x dimension feature to the selected neuron from the hidden layer.
+- P1 y: The input connection weight for the y dimension feature to the selected neuron from the hidden layer.
+- P1 b: The bias of the selected neuron from the hidden layer.
+- P1: The currently selected neuron from the hidden layer.
+- P2 w: The current connection weight.
+- P2 c: The current connection selected from the hidden layer neuron to the output neuron.
+- P2 b: The output neuron bias.
+
+**Checkbox Descriptions:**
+- ReLU?: Whether he first two plots will show the hidden layer outputs with/without the ReLU function applied.
+- Enabled?: Whether the current hidden layer neuron should be used.
+- Pre-add?: If enabled then the right side plots will show the hidden layer hyperplanes after they have been multiplied by the output neuron connection weights, but before they are all added together. If enabled then "Transform? will have no effect."
+- Transform?: Whether the right side plots should show the output neurons output before or after the transform (Checked is after).
+
+
diff --git a/RunningStats.py b/RunningStats.py
new file mode 100644
index 0000000..1376969
--- /dev/null
+++ b/RunningStats.py
@@ -0,0 +1,32 @@
+import numpy as np
+
+
+class RunningStats(object):
+
+ def __init__(self, window_size):
+ self.index = 1
+ self.window_size = window_size
+ self.values = np.zeros(window_size)
+ self.max_avg = 0
+ self.last_window_avg = 0
+
+ def get_average(self):
+ avg = np.average(self.values)
+ if avg > self.max_avg:
+ self.max_avg = avg
+ return avg
+
+ def insert(self, value):
+ self.values[self.index] = value
+ self.index += 1
+ if self.index >= self.window_size:
+ self.index = 0
+
+ def finished_window(self):
+ return self.index == 0
+
+ def window_improved(self):
+ window_avg = self.get_average()
+ result = window_avg > self.last_window_avg
+ self.last_window_avg = window_avg
+ return result
diff --git a/resources/Saves/dataset_CIRCLE/units_5/W1 b/resources/Saves/dataset_CIRCLE/units_5/W1
new file mode 100644
index 0000000..2980bad
--- /dev/null
+++ b/resources/Saves/dataset_CIRCLE/units_5/W1
@@ -0,0 +1 @@
+dJAyYJPrA=N? @xx>_>OA
\ No newline at end of file
diff --git a/resources/Saves/dataset_CIRCLE/units_5/W2 b/resources/Saves/dataset_CIRCLE/units_5/W2
new file mode 100644
index 0000000..d041bab
--- /dev/null
+++ b/resources/Saves/dataset_CIRCLE/units_5/W2
@@ -0,0 +1 @@
+d@@@5@
\ No newline at end of file
diff --git a/resources/Saves/dataset_CIRCLE/units_5/b1 b/resources/Saves/dataset_CIRCLE/units_5/b1
new file mode 100644
index 0000000..099eb51
--- /dev/null
+++ b/resources/Saves/dataset_CIRCLE/units_5/b1
@@ -0,0 +1 @@
+C|=
\ No newline at end of file
diff --git a/resources/Saves/dataset_CIRCLE/units_5/b2 b/resources/Saves/dataset_CIRCLE/units_5/b2
new file mode 100644
index 0000000..1eb45ad
--- /dev/null
+++ b/resources/Saves/dataset_CIRCLE/units_5/b2
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/resources/Saves/dataset_SPIRAL/units_24/W1 b/resources/Saves/dataset_SPIRAL/units_24/W1
new file mode 100644
index 0000000..a1c8164
Binary files /dev/null and b/resources/Saves/dataset_SPIRAL/units_24/W1 differ
diff --git a/resources/Saves/dataset_SPIRAL/units_24/W2 b/resources/Saves/dataset_SPIRAL/units_24/W2
new file mode 100644
index 0000000..62d5da0
--- /dev/null
+++ b/resources/Saves/dataset_SPIRAL/units_24/W2
@@ -0,0 +1 @@
+o>>34vj=Qa>8F?=%B4?8`?tt?8]? m젙gbA>3>э?9?M?a%>:$þq?o/>Q>?ѿ ?F|?b ˿+=!·E?1C꽁lK=>ռ
ꅿv3F>
\ No newline at end of file
diff --git a/resources/Saves/dataset_SPIRAL/units_24/b1 b/resources/Saves/dataset_SPIRAL/units_24/b1
new file mode 100644
index 0000000..4de0162
Binary files /dev/null and b/resources/Saves/dataset_SPIRAL/units_24/b1 differ
diff --git a/resources/Saves/dataset_SPIRAL/units_24/b2 b/resources/Saves/dataset_SPIRAL/units_24/b2
new file mode 100644
index 0000000..1221a47
--- /dev/null
+++ b/resources/Saves/dataset_SPIRAL/units_24/b2
@@ -0,0 +1 @@
+8?8
\ No newline at end of file
diff --git a/resources/pics/circle_sigmoid.png b/resources/pics/circle_sigmoid.png
new file mode 100644
index 0000000..fd1623e
Binary files /dev/null and b/resources/pics/circle_sigmoid.png differ
diff --git a/resources/pics/gui.png b/resources/pics/gui.png
new file mode 100644
index 0000000..bc609a5
Binary files /dev/null and b/resources/pics/gui.png differ
diff --git a/resources/pics/spiral_softmax.png b/resources/pics/spiral_softmax.png
new file mode 100644
index 0000000..f790bbf
Binary files /dev/null and b/resources/pics/spiral_softmax.png differ