diff --git a/.eslintrc.js b/.eslintrc.js index dce60ff..9e6cec2 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -2,5 +2,6 @@ module.exports = { extends: '@bamdadsabbagh/eslint-config', rules: { '@typescript-eslint/no-explicit-any': 'off', + 'brace-style': ['error', 'stroustrup'], }, }; diff --git a/index.html b/index.html index 0109f7c..49b706c 100644 --- a/index.html +++ b/index.html @@ -313,6 +313,7 @@

Features

+
Click anywhere to edit.
@@ -327,127 +328,7 @@

Features

- -
-
Node: -
-
- Bias - -
-
- -
-
- Weight - #1 -
-
- -
-
- Weight - #2 -
-
- -
-
- Weight - #3 -
-
- -
-
- Weight - #4 -
-
- -
-
- Weight - #5 -
-
- -
-
- Weight - #6 -
-
- -
-
- Weight - #7 -
-
- -
-
- Weight - #8 -
-
- -
-
+
@@ -912,6 +793,123 @@
Novation Launch Control XL (Controller)
+ +
+
+ Sources + Weights + Biases + Learning rate + Activation + Regularization + Regul. rate +
+
+ #1 + + +
+
+
+
+
+
+ #2 + + +
+
+
+
+
+
+ #3 + + +
+
+
+
+
+
+ #4 + + +
+
+
+
+
+
+ #5 + + +
+
+
+
+
+
+ #6 + + +
+
+
+
+
+
+ #7 + + +
+
+
+
+
+
+ #8 + + +
+
+
+
+
+
+ diff --git a/src/app/app.ts b/src/app/app.ts index 83bd211..8e900ec 100644 --- a/src/app/app.ts +++ b/src/app/app.ts @@ -1,5 +1,5 @@ -import { midi } from './midi/midi'; import { ui } from './ui/ui'; +import { midi } from './midi/midi'; // eslint-disable-next-line @typescript-eslint/no-var-requires const packageJson = require ('../../package.json'); @@ -23,5 +23,5 @@ app.init = async function (): Promise { this.isInitialized = true; // eslint-disable-next-line no-console - console.log (app); + console.log ({ version: app.version }); }; diff --git a/src/app/devices/controller.device.ts b/src/app/devices/controller.device.ts index fe1fd91..604745f 100644 --- a/src/app/devices/controller.device.ts +++ b/src/app/devices/controller.device.ts @@ -16,6 +16,17 @@ import { playgroundUi } from '../ui/playground.ui'; */ export const controllerDevice = Object.create (devicePrototype); +controllerDevice.shifted = { + 0: false, + 1: false, + 2: false, + 3: false, + 4: false, + 5: false, + 6: false, + 7: false, +}; + /** * Initialize the controller. * @@ -50,9 +61,11 @@ controllerDevice.drawLights = function () { let color; if (this.isDefaultMode) { color = this.settings.colors.amber; - } else if (this.isSingleMode) { + } + else if (this.isSingleMode) { color = this.settings.colors.black; - } else { + } + else { color = this.settings.colors.red; } @@ -71,9 +84,11 @@ controllerDevice.updateMode = function () { if (this.isDefaultMode) { this.setDefaultMode (); - } else if (this.isSingleMode) { + } + else if (this.isSingleMode) { this.setSingleMode (); - } else { + } + else { this.setMultipleMode (); } }; @@ -90,8 +105,9 @@ controllerDevice.setDefaultMode = function () { * Set the single mode. */ controllerDevice.setSingleMode = function () { - const selectedNode = playgroundFacade.selectedNodes[0]; - this.attachControlsToNeuron (selectedNode); + const node = playgroundFacade.selectedNodes[0]; + this.attachButtonsToNeuron (); + this.attachControlsToNeuron (node); }; /** @@ -99,7 +115,10 @@ controllerDevice.setSingleMode = function () { */ controllerDevice.setMultipleMode = function () { const { selectedNodes } = playgroundFacade; - selectedNodes.forEach ((n) => this.attachControlsToNeuron (n)); + selectedNodes.forEach ((node) => { + this.attachButtonsToNeuron (); + this.attachControlsToNeuron (node); + }); }; /** @@ -181,6 +200,32 @@ controllerDevice.attachControlsDefault = function () { }); }; +controllerDevice.attachButtonsToNeuron = function (): void { + this.addNoteListener ('on', (e) => { + const inputNote = parseInt (e.note.number); + const index = this.settings.rows.secondButtons.indexOf (inputNote); + if (index !== -1) { + this.shifted[index] = true; + this.playNote ({ + note: inputNote, + color: this.settings.colors.amber, + }); + } + }); + + this.addNoteListener ('off', (e) => { + const inputNote = parseInt (e.note.number); + const index = this.settings.rows.secondButtons.indexOf (inputNote); + if (index !== -1) { + this.shifted[index] = false; + this.playNote ({ + note: inputNote, + color: this.settings.colors.black, + }); + } + }); +}; + /** * Attach events to the ranges. * @@ -202,55 +247,152 @@ controllerDevice.attachControlsToNeuron = function (selectedNode: number): void // listen to changes this.addControlListener ((e) => { const inputNote = e.controller.number; - const firstFader = this.settings.rows.faders[0]; - const lastFader = this.settings.rows.faders[7]; - if (inputNote >= firstFader && inputNote <= lastFader) { - const index = inputNote - firstFader; + // first row: learning rate + if (this.settings.rows.firstPots.indexOf (inputNote) !== -1) { + const index = inputNote - this.settings.rows.firstPots[0]; + const learningRateOptionIndex = parseInt ( + rangeMap ( + e.value, + 0, + 127, + 0, + neuronCardUi.options.learningRate.length - 1, + ).toString (), + ); + + const learningRate = neuronCardUi.options.learningRate[learningRateOptionIndex]; + if (learningRate !== neuron.inputLinks[index].source.learningRate) { + networkState.updateSourceLearningRate (index, learningRate); + neuronCardUi.setLearningRate (index, learningRate); + playgroundFacade.updateUI (); + } + } + // second row: activation + else if (this.settings.rows.secondPots.indexOf (inputNote) !== -1) { + const index = inputNote - this.settings.rows.secondPots[0]; + const activationOptionIndex = parseInt ( + rangeMap ( + e.value, + 0, + 127, + 0, + neuronCardUi.options.activation.length - 1, + ).toString (), + ); + + const activation = neuronCardUi.options.activation[activationOptionIndex]; + + if (activation !== neuron.inputLinks[index].source.activation.name) { + networkState.updateSourceActivation (index, activation); + neuronCardUi.setActivation (index, activation); + playgroundFacade.updateUI (); + } + } + // third row: regularization + regularization rates (shifted) + else if (this.settings.rows.thirdPots.indexOf (inputNote) !== -1) { + const index = this.settings.rows.thirdPots.indexOf (inputNote); + // regularization + if (this.shifted[index] === false) { + const regularizationOptionIndex = parseInt ( + rangeMap ( + e.value, + 0, + 127, + 0, + neuronCardUi.options.regularization.length - 1, + ).toString (), + ); + + const regularization = neuronCardUi.options.regularization[regularizationOptionIndex]; + + if (regularization !== neuron.inputLinks[index].source.regularization.name) { + networkState.updateSourceRegularization (index, regularization); + neuronCardUi.setRegularization (index, regularization); + playgroundFacade.updateUI (); + } + } + // regularization rate + else { + const regularizationRateOptionIndex = parseInt ( + rangeMap ( + e.value, + 0, + 127, + 0, + neuronCardUi.options.regularizationRate.length - 1, + ).toString (), + ); + + const regularizationRate = neuronCardUi.options.regularizationRate[regularizationRateOptionIndex]; + + if (regularizationRate !== neuron.inputLinks[index].source.regularizationRate) { + networkState.updateSourceRegularizationRate (index, regularizationRate); + neuronCardUi.setRegularizationRate (index, regularizationRate); + playgroundFacade.updateUI (); + } + } + } + // faders: weights + biases (shifted) + else if (this.settings.rows.faders.indexOf (inputNote) !== -1) { + const index = this.settings.rows.faders.indexOf (inputNote); const source = links?.[index]?.source; if (typeof source === 'undefined') { return; } - // compute the new value - // intentionally use 2 decimals to avoid high frequency changes - const value = parseFloat ( - rangeMap (e.value, 0, 127, -1, 1) - .toFixed (2), - ); - - if (value.toFixed (1) === links[index].weight.toFixed (1)) { - // snap - links[index].hasSnapped = true; - - // automatic unsnap - if (links[index].snapTimer) { - clearTimeout (links[index].snapTimer); + // weights + if (this.shifted[index] === false) { + // compute the new value + // intentionally use 2 decimals to avoid high frequency changes + const value = parseFloat ( + rangeMap (e.value, 0, 127, -1, 1) + .toFixed (2), + ); + + if (value.toFixed (1) === links[index].weight.toFixed (1)) { + // snap + links[index].hasSnapped = true; + + // automatic unsnap + if (links[index].snapTimer) { + clearTimeout (links[index].snapTimer); + } + + links[index].snapTimer = setTimeout (() => { + links[index].hasSnapped = false; + this.playNote ({ + note: this.settings.outputByInput[inputNote], + color: this.settings.colors.red, + }); + }, 800); } - links[index].snapTimer = setTimeout (() => { - links[index].hasSnapped = false; + if (links[index].hasSnapped && source.isEnabled) { + networkState.setWeight (index, value); + neuronCardUi.setWeight (index, value); + playgroundFacade.updateWeightsUI (); + this.playNote ({ + note: this.settings.outputByInput[inputNote], + color: this.settings.colors.green, + }); + } + else { this.playNote ({ note: this.settings.outputByInput[inputNote], color: this.settings.colors.red, }); - }, 800); + } } - - if (links[index].hasSnapped && source.isEnabled) { - networkState.setWeight (index, value); - neuronCardUi.updateWeight (index, value); - playgroundFacade.updateWeightsUI (); - this.playNote ({ - note: this.settings.outputByInput[inputNote], - color: this.settings.colors.green, - }); - } else { - this.playNote ({ - note: this.settings.outputByInput[inputNote], - color: this.settings.colors.red, - }); + // biases + else { + const value = rangeMap (e.value, 0, 127, -1, 1); + if (value.toFixed (2) !== neuron.inputLinks[index].source.bias.toFixed (2)) { + neuron.inputLinks[index].source.bias = value; + neuronCardUi.setBias (index, value); + playgroundFacade.updateBiasesUI (); + } } } }); diff --git a/src/app/devices/selector.device.ts b/src/app/devices/selector.device.ts index 5ee056c..fb72a71 100644 --- a/src/app/devices/selector.device.ts +++ b/src/app/devices/selector.device.ts @@ -147,7 +147,8 @@ selectorDevice.attachNeurons = function (): void { if (isEnabled) { if (playgroundFacade.selectedNodes.indexOf (nodeIndex) === -1) { networkUi.toggleNodeSelection (nodeIndex, true); - } else { + } + else { networkUi.toggleNodeSelection (nodeIndex, false); } } @@ -202,9 +203,11 @@ selectorDevice.setNeuronLight = function (options: SetNeuronOptions): void { let color; if (isSelected) { color = this.settings.colorByState.neuronSelected; - } else if (isDisabled) { + } + else if (isDisabled) { color = this.settings.colorByState.neuronOff; - } else { + } + else { color = this.settings.colorByState.neuronOn; } @@ -293,14 +296,15 @@ selectorDevice.attachNavigation = function () { selectorDevice.attachLayers = function () { const layerPads = this.settings.functionKeys.firstRow.slice (1, -1); + const layersCount = networkState.neurons.length; // first draw - layerPads.forEach ((pad) => { + for (let i = 0; i < layersCount; ++i) { this.playOrBlinkNote ({ - note: pad, + note: layerPads[i], color: this.settings.colorByState.layer, }); - }); + } // listen for changes this.addControlListener ((e) => { diff --git a/src/app/facades/playground.facade.ts b/src/app/facades/playground.facade.ts index c515f7a..df4caab 100644 --- a/src/app/facades/playground.facade.ts +++ b/src/app/facades/playground.facade.ts @@ -7,6 +7,7 @@ import { updateUI, player, updateWeightsUI, + updateBiasesUI, } from '../../playground/playground'; export const playgroundFacade = Object.create (null); @@ -47,6 +48,12 @@ playgroundFacade.updateWeightsUI = function () { } }; +playgroundFacade.updateBiasesUI = function () { + if (this.isPlaying !== true) { + updateBiasesUI (this.network); + } +}; + Object.defineProperty (playgroundFacade, 'isPlaying', { get () { return player.getIsPlaying (); diff --git a/src/app/state/network.state.ts b/src/app/state/network.state.ts index af9ed93..7bb2783 100644 --- a/src/app/state/network.state.ts +++ b/src/app/state/network.state.ts @@ -1,5 +1,6 @@ import { playgroundFacade } from '../facades/playground.facade'; import { Link } from './network.state.types'; +import { activations, regularizations } from '../../playground/state'; /** * State object for the network. @@ -54,7 +55,8 @@ networkState.getNeuronAndLayerIndexes = function (nodeIndex: number): GetNeuronA let neuronIndex; if (nodeIndex % neuronsPerLayer === 0) { neuronIndex = neuronsPerLayer; - } else { + } + else { neuronIndex = nodeIndex % neuronsPerLayer; } @@ -109,8 +111,6 @@ networkState.toggleOutput = function (outputIndex: number): void { networkState.toggleNeuron = function (nodeIndex: number): void { const { neuron, isEnabled } = this.getNeuron (nodeIndex); - // todo how to impact node.bias ? - neuron.isEnabled = !isEnabled; // input weights @@ -122,7 +122,8 @@ networkState.toggleNeuron = function (nodeIndex: number): void { if (neuron.isEnabled) { link.isDead = false; link.weight = link.savedWeight || Math.random () - 0.5; - } else { + } + else { link.isDead = true; link.savedWeight = link.weight; link.weight = 0; @@ -138,7 +139,8 @@ networkState.toggleNeuron = function (nodeIndex: number): void { if (neuron.isEnabled) { link.isDead = false; link.weight = link.savedWeight || Math.random () - 0.5; - } else { + } + else { link.isDead = true; link.savedWeight = link.weight; link.weight = 0; @@ -169,21 +171,52 @@ networkState.toggleInput = function (slug: string): any { return input; }; -networkState.setWeight = function (weightIndex, value) { - let targetNeuronOrNeurons; +networkState.getSelectedNeurons = function () { const { selectedNodes } = playgroundFacade; + + let targets; // neuron or neurons + if (selectedNodes.length === 0) { return; - } else if (selectedNodes.length === 1) { - targetNeuronOrNeurons = [this.getNeuron (selectedNodes[0]).neuron]; - } else { - targetNeuronOrNeurons = selectedNodes.map ((nodeIndex) => this.getNeuron (nodeIndex).neuron); } + else if (selectedNodes.length === 1) { + targets = [this.getNeuron (selectedNodes[0]).neuron]; + } + else { + targets = selectedNodes.map ((i) => this.getNeuron (i).neuron); + } + return targets; +}; - targetNeuronOrNeurons.forEach ((neuron) => { - const weight = neuron.inputLinks?.[weightIndex]?.weight; +networkState.setWeight = function (index, value) { + this.getSelectedNeurons ().forEach ((neuron) => { + const weight = neuron.inputLinks?.[index]?.weight; if (typeof weight !== 'undefined') { - neuron.inputLinks[weightIndex].weight = value; + neuron.inputLinks[index].weight = value; } }); }; + +networkState.updateSourceLearningRate = function (index, value) { + this.getSelectedNeurons ().forEach ((neuron) => { + neuron.inputLinks[index].source.learningRate = value; + }); +}; + +networkState.updateSourceActivation = function (index, value) { + this.getSelectedNeurons ().forEach ((neuron) => { + neuron.inputLinks[index].source.activation = activations[value]; + }); +}; + +networkState.updateSourceRegularization = function (index, value) { + this.getSelectedNeurons ().forEach ((neuron) => { + neuron.inputLinks[index].source.regularization = regularizations[value]; + }); +}; + +networkState.updateSourceRegularizationRate = function (index, value) { + this.getSelectedNeurons ().forEach ((neuron) => { + neuron.inputLinks[index].source.regularizationRate = value; + }); +}; diff --git a/src/app/ui/network.ui.ts b/src/app/ui/network.ui.ts index 5cfd023..8125ebb 100644 --- a/src/app/ui/network.ui.ts +++ b/src/app/ui/network.ui.ts @@ -22,12 +22,13 @@ networkUi.toggleNeuron = function (index: number) { networkState.toggleNeuron (index); + neuronCardUi.updateCard (); + playgroundFacade.updateWeightsUI (); + selectorDevice.setNeuronLight ({ index, isDisabled: !nextEnabled, }); - - playgroundFacade.updateUI (); }; networkUi.toggleInput = function (slug: string, render = false) { @@ -40,7 +41,7 @@ networkUi.toggleInput = function (slug: string, render = false) { canvas.classed ('disabled', !input.isEnabled); } - playgroundFacade.updateUI (); + playgroundFacade.updateWeightsUI (); // device if (selectorDevice.isInitialized === true) { @@ -56,7 +57,8 @@ networkUi.toggleNodeSelection = function (nodeIndex: number, isSelected: boolean // playground local state if (isSelected) { playgroundFacade.selectNode (nodeIndex); - } else { + } + else { playgroundFacade.unselectNode (nodeIndex); } @@ -64,7 +66,8 @@ networkUi.toggleNodeSelection = function (nodeIndex: number, isSelected: boolean const canvas = d3.select (`#canvas-${nodeIndex}`); canvas.classed ('selected', isSelected); - neuronCardUi.updateCard (nodeIndex); + neuronCardUi.updateCard (); + selectorDevice.setNeuronLight ({ index: nodeIndex, isSelected }); controllerDevice.onSelectionEvent (); }; diff --git a/src/app/ui/neuron-card.ui.ts b/src/app/ui/neuron-card.ui.ts index fa5ed09..e81d008 100644 --- a/src/app/ui/neuron-card.ui.ts +++ b/src/app/ui/neuron-card.ui.ts @@ -4,80 +4,146 @@ import { networkState } from '../state/network.state'; export const neuronCardUi = Object.create (null); -neuronCardUi.weightSelector = '.neuron-card__weight'; -neuronCardUi.weights = null; -neuronCardUi.cardSelector = '#neuron-card'; -neuronCardUi.card = null; +neuronCardUi.nodeSelectors = { + node: '#neuron-card', + row: 'div.row:not(.header)', + weight: 'input.weight', + bias: 'input.bias', + learningRate: 'div.learning-rate', + activation: 'div.activation', + regularization: 'div.regularization', + regularizationRate: 'div.regularization-rate', +}; + +neuronCardUi.placeholders = { + undefined: 'ø', + multi: 'multi.', + disabled: 'disabled', +}; + +neuronCardUi.options = { + learningRate: [0.00001, 0.0001, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10], + activation: ['relu', 'tanh', 'sigmoid', 'linear'], + regularization: ['none', 'L1', 'L2'], + regularizationRate: [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10], +}; neuronCardUi.init = function () { - this.fetchWeights (); this.fetchCard (); + this.createLearningRates (); + this.createActivations (); + this.createRegularizations (); + this.createRegularizationRates (); this.attachEvents (); }; neuronCardUi.fetchCard = function () { - this.card = d3.select ('#neuron-card'); + this.node = d3.select (this.nodeSelectors.node); + this.rows = this.node.selectAll (this.nodeSelectors.row)[0]; + this.weights = this.node.selectAll (this.nodeSelectors.weight)[0]; + this.biases = this.node.selectAll (this.nodeSelectors.bias)[0]; + this.learningRates = this.node.selectAll (this.nodeSelectors.learningRate)[0]; + this.activations = this.node.selectAll (this.nodeSelectors.activation)[0]; + this.regularizations = this.node.selectAll (this.nodeSelectors.regularization)[0]; + this.regularizationRates = this.node.selectAll (this.nodeSelectors.regularizationRate)[0]; }; -neuronCardUi.fetchWeights = function () { - this.weights = document.querySelectorAll (this.weightSelector); +neuronCardUi.createOptions = function (parent, options) { + const select = document.createElement ('select'); + + options.forEach ((option) => { + const optionElement = document.createElement ('option'); + optionElement.value = option; + optionElement.innerText = option; + select.appendChild (optionElement); + }); + + parent.appendChild (select); }; -neuronCardUi.updateCard = function (nodeIndex: number) { - const { selectedNodes } = playgroundFacade; +neuronCardUi.createLearningRates = function () { + this.learningRates.forEach ((learningRate) => { + this.createOptions (learningRate, this.options.learningRate); + }); +}; +neuronCardUi.createActivations = function () { + this.activations.forEach ((activation) => { + this.createOptions (activation, this.options.activation); + }); +}; + +neuronCardUi.createRegularizations = function () { + this.regularizations.forEach ((regularization) => { + this.createOptions (regularization, this.options.regularization); + }); +}; + +neuronCardUi.createRegularizationRates = function () { + this.regularizationRates.forEach ((regularizationRate) => { + this.createOptions (regularizationRate, this.options.regularizationRate); + }); +}; + +neuronCardUi.updateCard = function () { + const { selectedNodes } = playgroundFacade; if (selectedNodes.length === 0) { - this.card.style ('display', 'none'); + this.node.style ('display', 'none'); return; } - this.card.style ('display', 'flex'); - - const nodeTitle = this.card.select ('.node'); - const inputs = this.card.selectAll ('input')[0]; - - nodeTitle.text ( - selectedNodes.length === 1 - ? `Node: ${selectedNodes[0]}` - : `Nodes: ${ - selectedNodes - .sort ((a, b) => a - b) - .join (', ') - }`, - ); - - const { neuron } = networkState.getNeuron (nodeIndex); - const inputPlaceholder = 'ø or multi.'; - - const biasInput = inputs[0] as HTMLInputElement; - biasInput.placeholder = inputPlaceholder; - biasInput.value = selectedNodes.length === 1 - ? neuron.bias.toPrecision (2) - : null; - - const { inputLinks } = neuron; - inputs.slice (1).forEach ((input: HTMLInputElement, k) => { - if (typeof inputLinks[k] === 'undefined') { - input.value = null; - input.placeholder = inputPlaceholder; - input.disabled = true; - return; - } + this.node.style ('display', 'block'); - input.disabled = false; + this.rows.forEach ((row, index) => { + // single selection + if (selectedNodes.length === 1) { + const link = networkState.getNeuron (selectedNodes[0]).neuron.inputLinks[index]; - if (selectedNodes.length > 1) { - input.value = null; - input.placeholder = inputPlaceholder; - return; - } + if (typeof link === 'undefined') { + this.setWeight (index); + this.setBias (index); + this.setLearningRate (index); + this.setActivation (index); + this.setRegularization (index); + this.setRegularizationRate (index); + return; + } + + const weight = link.weight; + const bias = link.source.bias; - input.value = inputLinks[k].weight.toPrecision (2); + if (link.isDead === true) { + this.setWeight (index); + this.setBias (index); + this.setLearningRate (index); + this.setActivation (index); + this.setRegularization (index); + this.setRegularizationRate (index); + } + else { + this.setWeight (index, weight); + this.setBias (index, bias); + this.setLearningRate (index, link.source.learningRate); + this.setActivation (index, link.source.activation.name); + this.setRegularization (index, link.source.regularization.name); + this.setRegularizationRate (index, link.source.regularizationRate); + } + } + // multi selection + else { + this.setWeight (index, null); + this.setBias (index, null); + this.setLearningRate (index, null); + this.setActivation (index, null); + this.setRegularization (index, null); + this.setRegularizationRate (index, null); + } }); -}; -neuronCardUi.updateWeight = function (index, weight) { - this.weights[index].value = weight.toPrecision (2); + // dumb refresh if only one node is selected + if (selectedNodes.length === 1) { + requestAnimationFrame (() => this.updateCard ()); + } }; neuronCardUi.attachEvents = function () { @@ -87,7 +153,108 @@ neuronCardUi.attachEvents = function () { const value = parseFloat ((e.target as HTMLInputElement).value); networkState.setWeight (index, value); playgroundFacade.updateUI (); + weight.blur (); + }; + }); + } + + if (this.learningRates) { + this.learningRates.forEach ((learningRate, index) => { + learningRate.children[0].onchange = (e: InputEvent) => { + const value = parseFloat ((e.target as HTMLInputElement).value); + networkState.updateSourceLearningRate (index, value); + }; + }); + } + + if (this.activations) { + this.activations.forEach ((activation, index) => { + activation.children[0].onchange = (e: InputEvent) => { + const value = (e.target as HTMLInputElement).value; + networkState.updateSourceActivation (index, value); }; }); } + + if (this.regularizations) { + this.regularizations.forEach ((regularization, index) => { + regularization.children[0].onchange = (e: InputEvent) => { + const value = (e.target as HTMLInputElement).value; + networkState.updateSourceRegularization (index, value); + }; + }); + } + + if (this.regularizationRates) { + this.regularizationRates.forEach ((regularizationRate, index) => { + regularizationRate.children[0].onchange = (e: InputEvent) => { + const value = parseFloat ((e.target as HTMLInputElement).value); + networkState.updateSourceRegularizationRate (index, value); + }; + }); + } +}; + +neuronCardUi.setInput = function (pool, index, payload) { + const isFocused = pool[index] === document.activeElement; + if (isFocused) { + return; + } + + if (typeof payload === 'undefined') { + pool[index].disabled = true; + pool[index].value = null; + } + else if (payload === null) { + pool[index].disabled = false; + pool[index].value = null; + } + else { + pool[index].disabled = false; + pool[index].value = payload.toFixed (3); + } +}; + +neuronCardUi.setWeight = function (index, weight?) { + this.setInput (this.weights, index, weight); +}; + +neuronCardUi.setBias = function (index, bias?) { + this.setInput (this.biases, index, bias); +}; + +neuronCardUi.setDropdown = function (pool, index, payload) { + const didNotChange = pool[index].children[0].value === payload; + if (didNotChange) { + return; + } + + if (typeof payload === 'undefined') { + pool[index].children[0].disabled = true; + pool[index].children[0].value = null; + } + else if (payload === null) { + pool[index].children[0].disabled = false; + pool[index].children[0].value = null; + } + else { + pool[index].children[0].disabled = false; + pool[index].children[0].value = payload; + } +}; + +neuronCardUi.setLearningRate = function (index, learningRate?) { + this.setDropdown (this.learningRates, index, learningRate); +}; + +neuronCardUi.setActivation = function (index, activation?) { + this.setDropdown (this.activations, index, activation); +}; + +neuronCardUi.setRegularization = function (index, regularization?) { + this.setDropdown (this.regularizations, index, regularization); +}; + +neuronCardUi.setRegularizationRate = function (index, regularizationRate?) { + this.setDropdown (this.regularizationRates, index, regularizationRate); }; diff --git a/src/app/ui/ui.ts b/src/app/ui/ui.ts index 0052cf0..3aa2795 100644 --- a/src/app/ui/ui.ts +++ b/src/app/ui/ui.ts @@ -19,8 +19,6 @@ ui.init = async function () { neuronCardUi.init (); devicesUi.init (); helpUi.init (); - - notificationsUi.notify ('test'); }; /** diff --git a/src/coolearning/coolearning.ts b/src/coolearning/coolearning.ts index b6baeb6..001d462 100644 --- a/src/coolearning/coolearning.ts +++ b/src/coolearning/coolearning.ts @@ -9,7 +9,8 @@ export function Coolearning (): void { window.addEventListener ('load', () => { try { app.init (); - } catch (error) { + } + catch (error) { // eslint-disable-next-line no-console console.error (error); notificationsUi.notify ( diff --git a/src/playground/nn.ts b/src/playground/nn.ts index e094bf4..5cdac79 100644 --- a/src/playground/nn.ts +++ b/src/playground/nn.ts @@ -13,6 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +import { State } from './state'; + /** * A node in a neural network. Each node has a state * (total input, output, and their respectively derivatives) which changes @@ -37,25 +39,36 @@ export class Node { * bias term. */ accInputDer = 0; + /** * Number of accumulated err. derivatives with respect to the total input * since the last update. */ numAccumulatedDers = 0; + + isEnabled: boolean; + /** Activation function that takes total input and returns node's output */ + learningRate; activation: ActivationFunction; - isEnabled: boolean; + regularization: RegularizationFunction; + regularizationRate; /** * Creates a new node with the provided id and activation function. */ - constructor (id: string, activation: ActivationFunction, initZero?: boolean) { + // constructor (id: string, activation: ActivationFunction, initZero?: boolean) { + constructor (id: string, state: State, initZero?: boolean) { this.id = id; this.isEnabled = true; - this.activation = activation; + // this.activation = activation; if (initZero) { this.bias = 0; } + this.learningRate = state.learningRate; + this.activation = state.activation; + this.regularization = state.regularization; + this.regularizationRate = state.regularizationRate; } /** Recomputes the node's output and returns it. */ @@ -83,12 +96,14 @@ export interface ErrorFunction { export interface ActivationFunction { output: (input: number) => number; der: (input: number) => number; + name: string; } /** Function that computes a penalty cost for a given weight in the network. */ export interface RegularizationFunction { output: (weight: number) => number; der: (weight: number) => number; + name: string; } /** Built-in error functions */ @@ -104,9 +119,11 @@ export class Errors { (Math as any).tanh = (Math as any).tanh || function (x) { if (x === Infinity) { return 1; - } else if (x === -Infinity) { + } + else if (x === -Infinity) { return -1; - } else { + } + else { let e2x = Math.exp (2 * x); return (e2x - 1) / (e2x + 1); } @@ -120,10 +137,12 @@ export class Activations { let output = Activations.TANH.output (x); return 1 - output * output; }, + name: 'tanh', }; public static RELU: ActivationFunction = { output: x => Math.max (0, x), der: x => x <= 0 ? 0 : 1, + name: 'relu', }; public static SIGMOID: ActivationFunction = { output: x => 1 / (1 + Math.exp (-x)), @@ -131,22 +150,29 @@ export class Activations { let output = Activations.SIGMOID.output (x); return output * (1 - output); }, + name: 'sigmoid', }; public static LINEAR: ActivationFunction = { output: x => x, der: _x => 1, + name: 'linear', }; } /** Build-in regularization functions */ export class RegularizationFunction { + public static NONE: any = { + name: 'none', + }; public static L1: RegularizationFunction = { output: w => Math.abs (w), der: w => w < 0 ? -1 : (w > 0 ? 1 : 0), + name: 'L1', }; public static L2: RegularizationFunction = { output: w => 0.5 * w * w, der: w => w, + name: 'L2', }; } @@ -192,21 +218,26 @@ export class Link { * @param networkShape The shape of the network. E.g. [1, 2, 3, 1] means * the network will have one input node, 2 nodes in first hidden layer, * 3 nodes in second hidden layer and 1 output node. + * @param state The application state. * @param activation The activation function of every hidden node. * @param outputActivation The activation function for the output nodes. * @param inputIds List of ids for the input nodes. * @param initZero */ export function buildNetwork ( - networkShape: number[], activation: ActivationFunction, + networkShape: number[], + state: State, + activation: ActivationFunction, outputActivation: ActivationFunction, - inputIds: string[], initZero?: boolean): Node[][] { + inputIds: string[], + initZero?: boolean, +): Node[][] { let numLayers = networkShape.length; let id = 1; /** List of layers, with each layer being a list of nodes. */ let network: Node[][] = []; for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) { - let isOutputLayer = layerIdx === numLayers - 1; + // let isOutputLayer = layerIdx === numLayers - 1; let isInputLayer = layerIdx === 0; let currentLayer: Node[] = []; network.push (currentLayer); @@ -215,12 +246,25 @@ export function buildNetwork ( let nodeId = id.toString (); if (isInputLayer) { nodeId = inputIds[i]; - } else { + } + else { id++; } - let node = new Node (nodeId, - isOutputLayer ? outputActivation : activation, initZero); + + // let node = new Node ( + // nodeId, + // isOutputLayer ? outputActivation : activation, + // initZero, + // ); + // + let node = new Node ( + nodeId, + state, + initZero, + ); + currentLayer.push (node); + if (layerIdx >= 1) { // Add links from nodes in the previous layer to this node. for (let j = 0; j < network[layerIdx - 1].length; j++) { @@ -274,8 +318,11 @@ export function forwardProp (network: Node[][], inputs: number[]): number { * derivatives with respect to each node, and each weight * in the network. */ -export function backProp (network: Node[][], target: number, - errorFunc: ErrorFunction): void { +export function backProp ( + network: Node[][], + target: number, + errorFunc: ErrorFunction, +): void { // The output node is a special case. We use the user-defined error // function for the derivative. let outputNode = network[network.length - 1][0]; @@ -323,75 +370,75 @@ export function backProp (network: Node[][], target: number, } } -type UpdateWeightsProps = { - network: Node[][], - learningRate: number, - regularization: RegularizationFunction, - regularizationRate: number, -} - /** * Updates the weights of the network using the previously accumulated error * derivatives. */ -export function updateWeights ( - { - network, - learningRate, - regularization, - regularizationRate, - }: UpdateWeightsProps, -) { +export function updateWeights (network: Node[][]): void { for (let layerIdx = 1; layerIdx < network.length; layerIdx++) { let currentLayer = network[layerIdx]; for (let i = 0; i < currentLayer.length; i++) { let node = currentLayer[i]; - // Update the node's bias. - if (node.numAccumulatedDers > 0) { - node.bias -= learningRate * node.accInputDer / node.numAccumulatedDers; - node.accInputDer = 0; - node.numAccumulatedDers = 0; + updateNode (node); + } + } +} + +export function updateNode (node: Node): void { + let learningRate = node.learningRate; + let regularization = node.regularization; + let regularizationRate = node.regularizationRate; + + // Update the node's bias. + if (node.numAccumulatedDers > 0) { + node.bias -= learningRate * node.accInputDer / node.numAccumulatedDers; + node.accInputDer = 0; + node.numAccumulatedDers = 0; + } + + // Update the weights coming into this node. + for (let j = 0; j < node.inputLinks.length; j++) { + let link = node.inputLinks[j]; + if (link.isDead) { + continue; + } + + let regulDer = regularization !== RegularizationFunction.NONE + ? regularization.der (link.weight) + : 0; + + if (link.numAccumulatedDers > 0) { + // Update the weight based on dE/dw. + link.weight = link.weight - (learningRate / link.numAccumulatedDers) * link.accErrorDer; + + // Further update the weight based on regularization. + let newLinkWeight = link.weight - (learningRate * regularizationRate) * regulDer; + + // todo investigate + if ( + regularization === RegularizationFunction.L1 + && link.weight * newLinkWeight < 0 + ) { + // The weight crossed 0 due to the regularization term. Set it to 0. + link.weight = 0; + link.isDead = true; } - // Update the weights coming into this node. - for (let j = 0; j < node.inputLinks.length; j++) { - let link = node.inputLinks[j]; - if (link.isDead) { - continue; - } - let regulDer = regularization - ? regularization.der (link.weight) - : 0; - if (link.numAccumulatedDers > 0) { - // Update the weight based on dE/dw. - link.weight = link.weight - - (learningRate / link.numAccumulatedDers) * link.accErrorDer; - // Further update the weight based on regularization. - let newLinkWeight = link.weight - - (learningRate * regularizationRate) * regulDer; - - // todo investigate - if ( - regularization === RegularizationFunction.L1 - && link.weight * newLinkWeight < 0 - ) { - // The weight crossed 0 due to the regularization term. Set it to 0. - link.weight = 0; - link.isDead = true; - } else { - link.weight = newLinkWeight; - } - - link.accErrorDer = 0; - link.numAccumulatedDers = 0; - } + else { + link.weight = newLinkWeight; } + + link.accErrorDer = 0; + link.numAccumulatedDers = 0; } } } /** Iterates over every node in the network/ */ -export function forEachNode (network: Node[][], ignoreInputs: boolean, - accessor: (node: Node) => any) { +export function forEachNode ( + network: Node[][], + ignoreInputs: boolean, + accessor: (node: Node) => any, +) { for (let layerIdx = ignoreInputs ? 1 : 0; layerIdx < network.length; layerIdx++) { diff --git a/src/playground/playground.ts b/src/playground/playground.ts index 8bb366d..31a950c 100644 --- a/src/playground/playground.ts +++ b/src/playground/playground.ts @@ -111,7 +111,8 @@ class Player { if (this.isPlaying) { this.isPlaying = false; this.pause (); - } else { + } + else { this.isPlaying = true; if (iter === 0) { simulationStarted (); @@ -256,25 +257,25 @@ function makeGUI () { d3.select (`canvas[data-regDataset=${regDatasetKey}]`) .classed ('selected', true); - d3.select ('#add-layers').on ('click', () => { - if (state.numHiddenLayers >= 6) { - return; - } - state.networkShape[state.numHiddenLayers] = 2; - state.numHiddenLayers++; - parametersChanged = true; - reset (); - }); - - d3.select ('#remove-layers').on ('click', () => { - if (state.numHiddenLayers <= 0) { - return; - } - state.numHiddenLayers--; - state.networkShape.splice (state.numHiddenLayers); - parametersChanged = true; - reset (); - }); + // d3.select ('#add-layers').on ('click', () => { + // if (state.numHiddenLayers >= 6) { + // return; + // } + // state.networkShape[state.numHiddenLayers] = 2; + // state.numHiddenLayers++; + // parametersChanged = true; + // reset (); + // }); + + // d3.select ('#remove-layers').on ('click', () => { + // if (state.numHiddenLayers <= 0) { + // return; + // } + // state.numHiddenLayers--; + // state.networkShape.splice (state.numHiddenLayers); + // parametersChanged = true; + // reset (); + // }); let showTestData = d3.select ('#show-test-data').on ('change', function () { state.showTestData = this.checked; @@ -315,10 +316,12 @@ function makeGUI () { if (state.noise > currentMax) { if (state.noise <= 80) { noise.property ('max', state.noise); - } else { + } + else { state.noise = 50; } - } else if (state.noise < 0) { + } + else if (state.noise < 0) { state.noise = 0; } noise.property ('value', state.noise); @@ -334,6 +337,9 @@ function makeGUI () { d3.select ('label[for=\'batchSize\'] .value').text (state.batchSize); let activationDropdown = d3.select ('#activations').on ('change', function () { + nn.forEachNode (network, true, (node) => { + node.activation = activations[this.value]; + }); state.activation = activations[this.value]; parametersChanged = true; state.serialize (); @@ -343,6 +349,9 @@ function makeGUI () { getKeyFromValue (activations, state.activation)); let learningRate = d3.select ('#learningRate').on ('change', function () { + nn.forEachNode (network, true, (node) => { + node.learningRate = this.value; + }); state.learningRate = +this.value; parametersChanged = true; state.serialize (); @@ -351,6 +360,9 @@ function makeGUI () { learningRate.property ('value', state.learningRate); let regularDropdown = d3.select ('#regularizations').on ('change', function () { + nn.forEachNode (network, true, (node) => { + node.regularization = regularizations[this.value]; + }); state.regularization = regularizations[this.value]; parametersChanged = true; state.serialize (); @@ -360,6 +372,9 @@ function makeGUI () { getKeyFromValue (regularizations, state.regularization)); let regularRate = d3.select ('#regularRate').on ('change', function () { + nn.forEachNode (network, true, (node) => { + node.regularizationRate = this.value; + }); state.regularizationRate = +this.value; parametersChanged = true; state.serialize (); @@ -408,7 +423,7 @@ function makeGUI () { } } -function updateBiasesUI (network: nn.Node[][]) { +export function updateBiasesUI (network: nn.Node[][]) { nn.forEachNode (network, true, node => { d3.select (`rect#bias-${node.id}`).style ('fill', colorScale (node.bias)); }); @@ -485,7 +500,8 @@ function drawNode (cx: number, cy: number, nodeId: string, isInput: boolean, if (label.substring (lastIndex)) { text.append ('tspan').text (label.substring (lastIndex)); } - } else { + } + else { text.append ('tspan').text (label); } nodeGroup.classed (activeOrNotClass, true); @@ -535,19 +551,25 @@ function drawNode (cx: number, cy: number, nodeId: string, isInput: boolean, }) .on ('mouseup', () => { - if (mouseTimer === null) return; + if (mouseTimer === null) { + return; + } clearTimeout (mouseTimer); mouseTimer = null; - if (div.classed ('disabled')) return; + if (div.classed ('disabled')) { + return; + } if (Number.isNaN (parseInt (nodeId))) { return; - } else { + } + else { if (!div.classed ('selected')) { networkUi.toggleNodeSelection (parseInt (nodeId), true); - } else { + } + else { networkUi.toggleNodeSelection (parseInt (nodeId), false); } } @@ -696,42 +718,43 @@ function addPlusMinusControl (x: number, layerIdx: number) { .classed ('plus-minus-neurons', true) .style ('left', `${x - 10}px`); - let i = layerIdx - 1; - let firstRow = div.append ('div').attr ('class', `ui-numNodes${layerIdx}`); - firstRow.append ('button') - .attr ('class', 'mdl-button mdl-js-button mdl-button--icon') - .on ('click', () => { - let numNeurons = state.networkShape[i]; - if (numNeurons >= 8) { - return; - } - state.networkShape[i]++; - parametersChanged = true; - reset (); - }) - .append ('i') - .attr ('class', 'material-icons') - .text ('add'); - - firstRow.append ('button') - .attr ('class', 'mdl-button mdl-js-button mdl-button--icon') - .on ('click', () => { - let numNeurons = state.networkShape[i]; - if (numNeurons <= 1) { - return; - } - state.networkShape[i]--; - parametersChanged = true; - reset (); - }) - .append ('i') - .attr ('class', 'material-icons') - .text ('remove'); - - let suffix = state.networkShape[i] > 1 ? 's' : ''; - div.append ('div').text ( - state.networkShape[i] + ' neuron' + suffix, - ); + // let i = layerIdx - 1; + // let firstRow = div.append ('div').attr ('class', `ui-numNodes${layerIdx}`); + // firstRow.append ('button') + // .attr ('class', 'mdl-button mdl-js-button mdl-button--icon') + // .on ('click', () => { + // let numNeurons = state.networkShape[i]; + // if (numNeurons >= 8) { + // return; + // } + // state.networkShape[i]++; + // parametersChanged = true; + // reset (); + // }) + // .append ('i') + // .attr ('class', 'material-icons') + // .text ('add'); + + // firstRow.append ('button') + // .attr ('class', 'mdl-button mdl-js-button mdl-button--icon') + // .on ('click', () => { + // let numNeurons = state.networkShape[i]; + // if (numNeurons <= 1) { + // return; + // } + // state.networkShape[i]--; + // parametersChanged = true; + // reset (); + // }) + // .append ('i') + // .attr ('class', 'material-icons') + // .text ('remove'); + + // let suffix = state.networkShape[i] > 1 ? 's' : ''; + // div.append ('div').text ( + // state.networkShape[i] + ' neuron' + suffix, + // ); + div.append ('div').text (`Layer ${layerIdx}`); } // noinspection JSUnusedLocalSymbols @@ -751,7 +774,8 @@ function updateHoverCard (type: HoverType, nodeOrLink?: nn.Node | nn.Link, if (this.value != null && this.value !== '') { if (type === HoverType.WEIGHT) { (nodeOrLink as nn.Link).weight = +this.value; - } else { + } + else { (nodeOrLink as nn.Node).bias = +this.value; } updateUI (); @@ -959,12 +983,7 @@ function oneStep (): void { nn.forwardProp (network, input); nn.backProp (network, point.label, nn.Errors.SQUARE); if ((i + 1) % state.batchSize === 0) { - nn.updateWeights ({ - network, - learningRate: state.learningRate, - regularization: state.regularization, - regularizationRate: state.regularizationRate, - }); + nn.updateWeights (network); } }); // Compute the loss. @@ -1004,9 +1023,12 @@ function reset (onStartup = false) { iter = 0; let numInputs = constructInput (0, 0).length; let shape = [numInputs].concat (state.networkShape).concat ([1]); - let outputActivation = (state.problem === Problem.REGRESSION) ? - nn.Activations.LINEAR : nn.Activations.TANH; - network = nn.buildNetwork (shape, state.activation, outputActivation, constructInputIds (), state.initZero); + + let outputActivation = state.problem === Problem.REGRESSION + ? nn.Activations.LINEAR + : nn.Activations.TANH; + + network = nn.buildNetwork (shape, state, state.activation, outputActivation, constructInputIds (), state.initZero); lossTrain = getLoss (network, trainData); lossTest = getLoss (network, testData); drawNetwork (network); diff --git a/src/playground/state.ts b/src/playground/state.ts index 6150e1d..dcbefa0 100644 --- a/src/playground/state.ts +++ b/src/playground/state.ts @@ -29,7 +29,7 @@ export let activations: { [key: string]: nn.ActivationFunction } = { /** A map between names and regularization functions. */ export let regularizations: { [key: string]: nn.RegularizationFunction } = { - 'none': null, + 'none': nn.RegularizationFunction.NONE, 'L1': nn.RegularizationFunction.L1, 'L2': nn.RegularizationFunction.L2, }; @@ -142,7 +142,7 @@ export class State { tutorial: string = null; percTrainData = 50; activation = nn.Activations.TANH; - regularization: nn.RegularizationFunction = null; + regularization: nn.RegularizationFunction = nn.RegularizationFunction.NONE; problem = Problem.CLASSIFICATION; initZero = false; hideText = true; diff --git a/styles.css b/styles.css index 178e6ce..971bb45 100644 --- a/styles.css +++ b/styles.css @@ -368,31 +368,6 @@ header h1 .optional { width: 60px; } -/* Neuron Card */ -#neuron-card { - display: none; - position: absolute; - padding: 5px; - border: 1px solid #aaa; - z-index: 1000; - background: #fff; - cursor: default; - border-radius: 5px; - left: -203px; - width: 170px; - top: 72px; - flex-wrap: wrap; - justify-content: space-between; -} - -#neuron-card div { - padding: 2px; -} - -#neuron-card input { - width: 80px; -} - /* Main Part*/ #main-part { @@ -1102,3 +1077,42 @@ Help Dialog width: 600px; height: 600px; } + +/** +Neuron card + */ +#neuron-card { + display: none; + flex-direction: column; + + position: absolute; + left: 50%; + top: 5px; + transform: translate(-50%, 0); + + z-index: 1000; + + background: #fff; + cursor: default; + border-radius: 5px; + border: 1px solid #aaa; + + padding: 5px; + + font-size: 0.8em; + text-align: center; +} + +#neuron-card .row { + display: grid; + grid-gap: 5px; + grid-template-columns: 40px repeat(6, 70px); +} + +#neuron-card .header { + font-style: italic; +} + +#neuron-card input { + width: 60px; +}