From 47fddac6f50bc855f6ab54488fedd76c3b667b7a Mon Sep 17 00:00:00 2001 From: cknd Date: Thu, 4 Feb 2016 11:34:58 +0100 Subject: [PATCH 001/306] new world & worldadapter to read multidimensional timeseries data --- micropsi_core/world/timeseries/timeseries.py | 60 ++++++++++++++++++++ micropsi_core/world/world.py | 5 ++ 2 files changed, 65 insertions(+) create mode 100644 micropsi_core/world/timeseries/timeseries.py diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py new file mode 100644 index 00000000..8f41dfcb --- /dev/null +++ b/micropsi_core/world/timeseries/timeseries.py @@ -0,0 +1,60 @@ +""" +Worlds and bodies for agents whose habitats are ordered sequences of vectors. +""" +import os.path +from configuration import config as cfg +from micropsi_core.world.world import World +from micropsi_core.world.worldadapter import WorldAdapter +import numpy as np + + +class TimeSeries(World): + """ A world that cycles through a fixed time series loaded from a file. + This world looks for a file named timeseries.npz in the data_directory + that has been set in configuration. This is a stopgap, we want to add the + option to choose a file whenever such worlds are instantiated in the GUI. + + The file should be a numpy archive with the following fields: + + 'startdate', 'enddate': datetime objects + 'data': numpy array of shape (nr of ids) x (nr minutes between startdate and enddate) + 'ids': a list of IDs - the legend for data's first axis. + """ + supported_worldadapters = ['TimeSeriesRunner'] + + def __init__(self, filename, world_type="Island", name="", owner="", engine=None, uid=None, version=1): + World.__init__(self, filename, world_type=world_type, name=name, owner=owner, uid=uid, version=version) + path = os.path.join(cfg['micropsi2']['data_directory'], 'timeseries.npz') + print("loading timeseries from", path, "for world", uid) + with np.load(path) as f: + self.timeseries = f['data'] + self.ids = f['ids'] + self.startdate = f['startdate'] + self.enddate = f['enddate'] + self.len_ts = self.timeseries.shape[1] + + # todo: option to use only a subset of the data (e.g. for training/test) + + @property + def t(self): + """current index into the original time series""" + return self.current_step % self.len_ts + + @property + def state(self): + return self.timeseries[:, self.t] + + +class TimeSeriesRunner(WorldAdapter): + supported_datasources = [] + supported_datatargets = [] + + def __init__(self, world, uid=None, **data): + super().__init__(world, uid, **data) + for idx, ID in enumerate(self.world.ids): + self.supported_datasources.append(str(ID)) + + def update_data_sources_and_targets(self): + state = self.world.state + for idx, ID in enumerate(self.world.ids): + self.datasources[str(ID)] = state[idx] diff --git a/micropsi_core/world/world.py b/micropsi_core/world/world.py index 6517dbdd..fd8a9c55 100644 --- a/micropsi_core/world/world.py +++ b/micropsi_core/world/world.py @@ -327,3 +327,8 @@ def set_agent_properties(self, uid, position=None, orientation=None, name=None, pass else: sys.stdout.write("Could not import minecraft world.\nError: %s \n\n" % e.msg) + +try: + from micropsi_core.world.timeseries import timeseries +except ImportError as e: + sys.stdout.write("Could not import timeseries world.\nError: %s \n\n" % e.msg) From 2110ee68ab9b6b7e2a276c440403d1427cec7037 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 4 Feb 2016 18:37:58 +0100 Subject: [PATCH 002/306] version bump for 0.8 --- CHANGELOG.md | 4 ++++ configuration.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95fb49f9..fd990e7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ +0.8-alpha6 (unreleased) +========== + + 0.7-alpha5 (2016-02-04) ========== diff --git a/configuration.py b/configuration.py index 25958ed6..aa01aa8a 100644 --- a/configuration.py +++ b/configuration.py @@ -24,7 +24,7 @@ warnings.warn('Can not read config from inifile %s' % filename) raise RuntimeError('Can not read config from inifile %s' % filename) -config['micropsi2']['version'] = "0.7-alpha5" +config['micropsi2']['version'] = "0.8-alpha6-dev" config['micropsi2']['apptitle'] = "MicroPsi" homedir = config['micropsi2']['data_directory'].startswith('~') From c47d4b31a5357c89f5d6ac83f7039679c392f8b1 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 4 Feb 2016 20:24:40 +0100 Subject: [PATCH 003/306] show worldadapter docstring in creation dialog --- .../static/css/micropsi-styles.css | 4 ++++ micropsi_server/static/js/dialogs.js | 2 +- .../view/worldadapter_selector.tpl | 23 ++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/micropsi_server/static/css/micropsi-styles.css b/micropsi_server/static/css/micropsi-styles.css index 2164adfa..02a908f1 100644 --- a/micropsi_server/static/css/micropsi-styles.css +++ b/micropsi_server/static/css/micropsi-styles.css @@ -626,4 +626,8 @@ p.clear { .docstring { white-space: pre-line; +} + +.hint.small { + font-size: 0.9em; } \ No newline at end of file diff --git a/micropsi_server/static/js/dialogs.js b/micropsi_server/static/js/dialogs.js index da3335d5..a58f66c1 100644 --- a/micropsi_server/static/js/dialogs.js +++ b/micropsi_server/static/js/dialogs.js @@ -639,7 +639,7 @@ $(function() { updateWorldAdapterSelector = function() { var option = $("#nn_world option:selected"); if (option) { - $("#nn_worldadapter").load("/create_worldadapter_selector/"+option.val()); + $("#nn_worldadapter").parent().load("/create_worldadapter_selector/"+option.val()); } }; diff --git a/micropsi_server/view/worldadapter_selector.tpl b/micropsi_server/view/worldadapter_selector.tpl index b02f9ff2..734f58d0 100644 --- a/micropsi_server/view/worldadapter_selector.tpl +++ b/micropsi_server/view/worldadapter_selector.tpl @@ -1,5 +1,5 @@ - % if not world_uid in worlds: % else: @@ -10,6 +10,27 @@ %end %end + +%end + +
Select a worldadapter to see a description
+ +%if world_uid in worlds: + %end From 0cb245534ede12fe4bc344f9c12929ed76d0c567 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 5 Feb 2016 15:27:29 +0100 Subject: [PATCH 004/306] datasources and targets can only be obtained from an instance --- micropsi_core/_runtime_api_world.py | 18 ++++++++++-------- .../tests/test_runtime_world_basics.py | 13 ++++++++----- .../minecraft/minecraft_histogram_vision.py | 1 - .../world/minecraft/minecraft_vision.py | 3 --- micropsi_server/micropsi_app.py | 4 ++-- micropsi_server/static/js/nodenet.js | 6 ++---- 6 files changed, 22 insertions(+), 23 deletions(-) diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index edf01e77..006218a8 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -49,16 +49,18 @@ def get_world_properties(world_uid): return data -def get_worldadapters(world_uid): - """Returns the world adapters available in the given world""" - +def get_worldadapters(world_uid, nodenet_uid=None): + """ Returns the world adapters available in the given world. Provide an optional nodenet_uid of an agent + in the given world to obtain datasources and datatargets for the agent's worldadapter""" data = {} if world_uid in micropsi_core.runtime.worlds: - for name, worldadapter in micropsi_core.runtime.worlds[world_uid].supported_worldadapters.items(): - data[name] = { - 'datasources': worldadapter.supported_datasources, - 'datatargets': worldadapter.supported_datatargets - } + world = micropsi_core.runtime.worlds[world_uid] + for name, worldadapter in world.supported_worldadapters.items(): + data[name] = {'description': worldadapter.__doc__} + if nodenet_uid and nodenet_uid in world.agents: + agent = world.agents[nodenet_uid] + data[agent.__class__.__name__]['datasources'] = sorted(agent.datasources.keys()) + data[agent.__class__.__name__]['datatargets'] = sorted(agent.datatargets.keys()) return data diff --git a/micropsi_core/tests/test_runtime_world_basics.py b/micropsi_core/tests/test_runtime_world_basics.py index cb52ae57..76ece5be 100644 --- a/micropsi_core/tests/test_runtime_world_basics.py +++ b/micropsi_core/tests/test_runtime_world_basics.py @@ -42,12 +42,15 @@ def test_get_world_properties(test_world): assert test_world == wp["uid"] -def test_get_worldadapters(test_world): +def test_get_worldadapters(test_world, test_nodenet): wa = micropsi.get_worldadapters(test_world) - assert 'engine_l' in wa['Braitenberg']['datatargets'] - assert 'engine_r' in wa['Braitenberg']['datatargets'] - assert 'brightness_l' in wa['Braitenberg']['datasources'] - assert 'brightness_r' in wa['Braitenberg']['datasources'] + assert 'Braitenberg' in wa + assert 'description' in wa['Braitenberg'] + assert 'datasources' not in wa['Braitenberg'] + runtime.set_nodenet_properties(test_nodenet, worldadapter='Braitenberg', world_uid=test_world) + wa = micropsi.get_worldadapters(test_world, test_nodenet) + assert wa['Braitenberg']['datatargets'] == ['engine_l', 'engine_r'] + assert wa['Braitenberg']['datasources'] == ['brightness_l', 'brightness_r'] def test_add_worldobject(test_world): diff --git a/micropsi_core/world/minecraft/minecraft_histogram_vision.py b/micropsi_core/world/minecraft/minecraft_histogram_vision.py index e3ed3a6b..6572aa9b 100644 --- a/micropsi_core/world/minecraft/minecraft_histogram_vision.py +++ b/micropsi_core/world/minecraft/minecraft_histogram_vision.py @@ -62,7 +62,6 @@ def __init__(self, world, uid=None, **data): for j in range(self.num_fov): name = "fov__%02d_%02d" % (i, j) self.datasources[name] = 0. - MinecraftHistogramVision.supported_datasources.append(name) self.simulated_vision = False if 'simulate_vision' in cfg['minecraft']: diff --git a/micropsi_core/world/minecraft/minecraft_vision.py b/micropsi_core/world/minecraft/minecraft_vision.py index 51a98f71..3dce1fd4 100644 --- a/micropsi_core/world/minecraft/minecraft_vision.py +++ b/micropsi_core/world/minecraft/minecraft_vision.py @@ -47,14 +47,12 @@ def __init__(self, world, uid=None, **data): for j in range(self.len_y): name = "fov__%02d_%02d" % (i, j) self.datasources[name] = 0. - self.supported_datasources.append(name) # add datasources for fovea position sensors aka fov_pos__*_* for x in range(self.tiling_x): for y in range(self.tiling_y): name = "fov_pos__%02d_%02d" % (y, x) self.datasources[name] = 0. - self.supported_datasources.append(name) # add fovea actors to datatargets, datatarget_feedback, datatarget_history, and actions for x in range(self.tiling_x): @@ -63,7 +61,6 @@ def __init__(self, world, uid=None, **data): self.datatargets[name] = 0. self.datatarget_feedback[name] = 0. self.datatarget_history[name] = 0. - self.supported_datatargets.append(name) self.actions.append(name) self.simulated_vision = False diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index 5fc6e815..ad41fe89 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -864,9 +864,9 @@ def get_world_properties(world_uid): @rpc("get_worldadapters") -def get_worldadapters(world_uid): +def get_worldadapters(world_uid, nodenet_uid=None): try: - return True, runtime.get_worldadapters(world_uid) + return True, runtime.get_worldadapters(world_uid, nodenet_uid=nodenet_uid) except KeyError: return False, 'World %s not found' % world_uid diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index 89e42d3f..083eaf02 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -182,7 +182,7 @@ function get_available_worlds(){ function get_available_worldadapters(world_uid, callback){ worldadapters = {}; if(world_uid){ - api.call("get_worldadapters", {world_uid: world_uid}, + api.call("get_worldadapters", {world_uid: world_uid, nodenet_uid: currentNodenet}, success=function(data){ worldadapters = data; currentWorld = world_uid; @@ -192,9 +192,7 @@ function get_available_worldadapters(world_uid, callback){ keys.sort(); for (var idx in keys){ name = keys[idx]; - worldadapters[name].datasources = worldadapters[name].datasources.sort(); - worldadapters[name].datatargets = worldadapters[name].datatargets.sort(); - str += ''; + str += ''; } $('#nodenet_worldadapter').html(str); if(callback){ From df2718c67eabeaf2854cdccaad175a9580f02b9f Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 5 Feb 2016 16:00:34 +0100 Subject: [PATCH 005/306] get rid of supported_datasources/targets altogether --- micropsi_core/tests/test_node_logic.py | 3 -- micropsi_core/world/island/island.py | 15 +++--- .../structured_objects/structured_objects.py | 40 +++++++-------- micropsi_core/world/minecraft/minecraft.py | 40 ++++++++------- .../minecraft/minecraft_graph_locomotion.py | 44 ++++++---------- .../minecraft/minecraft_histogram_vision.py | 51 +++++++++---------- micropsi_core/world/worldadapter.py | 13 ++--- 7 files changed, 93 insertions(+), 113 deletions(-) diff --git a/micropsi_core/tests/test_node_logic.py b/micropsi_core/tests/test_node_logic.py index c996420a..6887dd59 100644 --- a/micropsi_core/tests/test_node_logic.py +++ b/micropsi_core/tests/test_node_logic.py @@ -22,9 +22,6 @@ def __init__(self, filename, world_type="DummyWorld", name="", owner="", uid=Non class DummyWorldAdapter(WorldAdapter): - supported_datasources = ['test_source'] - supported_datatargets = ['test_target'] - def __init__(self, world, uid=None, **data): WorldAdapter.__init__(self, world, uid=uid, **data) self.datasources = {'test_source': 0.7} diff --git a/micropsi_core/world/island/island.py b/micropsi_core/world/island/island.py index d07061c1..2407e8c5 100644 --- a/micropsi_core/world/island/island.py +++ b/micropsi_core/world/island/island.py @@ -291,12 +291,13 @@ def action_drink(self): class Survivor(WorldAdapter): - supported_datasources = ['body-energy', 'body-water', 'body-integrity'] - supported_datatargets = ['action_eat', 'action_drink', 'loco_north', 'loco_south', 'loco_east', 'loco_west'] def __init__(self, world, uid=None, **data): super(Survivor, self).__init__(world, uid, **data) + self.datasources = dict((s, 0) for s in ['body-energy', 'body-water', 'body-integrity']) + self.datatargets = dict((t, 0) for t in ['action_eat', 'action_drink', 'loco_north', 'loco_south', 'loco_east', 'loco_west']) + self.currentobject = None self.energy = 1.0 @@ -311,7 +312,7 @@ def __init__(self, world, uid=None, **data): self.datasources['body-integrity'] = self.integrity def initialize_worldobject(self, data): - if not "position" in data: + if "position" not in data: self.position = self.world.groundmap['start_position'] def update_data_sources_and_targets(self): @@ -417,15 +418,15 @@ class Braitenberg(WorldAdapter): # maximum speed speed_limit = 1. - supported_datasources = ['brightness_l', 'brightness_r'] - supported_datatargets = ['engine_l', 'engine_r'] - def __init__(self, world, uid=None, **data): super(Braitenberg, self).__init__(world, uid, **data) + + self.datasources = {'brightness_l': 0, 'brightness_r': 0} + self.datatargets = {'engine_l': 0, 'engine_r': 0} self.datatarget_feedback = {'engine_l': 0, 'engine_r': 0} def initialize_worldobject(self, data): - if not "position" in data: + if "position" not in data: self.position = self.world.groundmap['start_position'] def update_data_sources_and_targets(self): diff --git a/micropsi_core/world/island/structured_objects/structured_objects.py b/micropsi_core/world/island/structured_objects/structured_objects.py index 0b6761a2..bba92c33 100644 --- a/micropsi_core/world/island/structured_objects/structured_objects.py +++ b/micropsi_core/world/island/structured_objects/structured_objects.py @@ -10,30 +10,30 @@ class StructuredObjects(WorldAdapter): """A world adapter exposing objects composed of basic shapes and colors to the agent""" - shapetypes = [] - shapecolors = [] + def __init__(self, world, uid=None, **data): + super(StructuredObjects, self).__init__(world, uid, **data) - supported_datasources = ['fov-x', 'fov-y', 'major-newscene'] - supported_datatargets = ['fov_x', 'fov_y', 'fov_reset'] + self.datasources = {'fov-x': 0, 'fov-y': 0, 'major-newscene': 0} + self.datatargets = {'fov_x': 0, 'fov_y': 0, 'fov_reset': 0} - for key, objecttype in OBJECTS.items(): - for shapeline in objecttype['shape_grid']: - for shape in shapeline: - if shape is not None and shape.type not in shapetypes: - shapetypes.append(shape.type) - if shape is not None and shape.color not in shapecolors: - shapecolors.append(shape.color) + self.shapetypes = [] + self.shapecolors = [] - for shapetype in shapetypes: - supported_datasources.append('fovea-'+shapetype) - supported_datasources.append('presence-'+shapetype) + for key, objecttype in OBJECTS.items(): + for shapeline in objecttype['shape_grid']: + for shape in shapeline: + if shape is not None and shape.type not in self.shapetypes: + self.shapetypes.append(shape.type) + if shape is not None and shape.color not in self.shapecolors: + self.shapecolors.append(shape.color) - for shapecolor in shapecolors: - supported_datasources.append("fovea-"+shapecolor) - supported_datasources.append("presence-"+shapecolor) + for shapetype in self.shapetypes: + self.datasources['fovea-' + shapetype] = 0 + self.datasources['presence-' + shapetype] = 0 - def __init__(self, world, uid=None, **data): - super(StructuredObjects, self).__init__(world, uid, **data) + for shapecolor in self.shapecolors: + self.datasources["fovea-" + shapecolor] = 0 + self.datasources["presence-" + shapecolor] = 0 self.currentobject = None self.scene = None @@ -42,7 +42,7 @@ def __init__(self, world, uid=None, **data): self.scene.load_object("PalmTree", OBJECTS["PalmTree"]["shape_grid"]) def initialize_worldobject(self, data): - if not "position" in data: + if "position" not in data: self.position = self.world.groundmap['start_position'] def get_datasource(self, key): diff --git a/micropsi_core/world/minecraft/minecraft.py b/micropsi_core/world/minecraft/minecraft.py index 0f84bcd2..d20da8dc 100644 --- a/micropsi_core/world/minecraft/minecraft.py +++ b/micropsi_core/world/minecraft/minecraft.py @@ -322,9 +322,6 @@ class MinecraftWorldAdapter(WorldAdapter): moves into one of the four cardinal directions ( until it dies ). """ - supported_datasources = ['x', 'y', 'z', 'yaw', 'pitch', 'groundtype'] - supported_datatargets = ['go_north', 'go_east', 'go_west', 'go_south', 'yaw', 'pitch'] - spawn_position = { 'x': -105, 'y': 63, @@ -334,6 +331,9 @@ class MinecraftWorldAdapter(WorldAdapter): def __init__(self, world, uid=None, **data): world.spockplugin.clientinfo.spawn_position = self.spawn_position WorldAdapter.__init__(self, world, uid=uid, **data) + self.datasources = dict((i, 0) for i in ['x', 'y', 'z', 'yaw', 'pitch', 'groundtype']) + self.datatargets = dict((i, 0) for i in ['go_north', 'go_east', 'go_west', 'go_south', 'yaw', 'pitch']) + def initialize_worldobject(self, data): @@ -405,22 +405,24 @@ def get_groundtype(self): class MinecraftBraitenberg(WorldAdapter): - supported_datasources = [ - 'diamond_offset_x', - 'diamond_offset_z', - 'grd_stone', - 'grd_dirt', - 'grd_wood', - 'grd_coal', - 'obstcl_x+', - 'obstcl_x-', - 'obstcl_z+', - 'obstcl_z-' - ] - supported_datatargets = [ - 'move_x', - 'move_z' - ] + def __init__(self, world, uid=None, **data): + super().__init__(world, uid, **data) + self.datasources = { + 'diamond_offset_x': 0, + 'diamond_offset_z': 0, + 'grd_stone': 0, + 'grd_dirt': 0, + 'grd_wood': 0, + 'grd_coal': 0, + 'obstcl_x+': 0, + 'obstcl_x-': 0, + 'obstcl_z+': 0, + 'obstcl_z-': 0 + } + self.datatargets = { + 'move_x': 0, + 'move_z': 0 + } def update_data_sources_and_targets(self): """called on every world simulation step to advance the life of the agent""" diff --git a/micropsi_core/world/minecraft/minecraft_graph_locomotion.py b/micropsi_core/world/minecraft/minecraft_graph_locomotion.py index a6a38c9f..74ecfeaf 100644 --- a/micropsi_core/world/minecraft/minecraft_graph_locomotion.py +++ b/micropsi_core/world/minecraft/minecraft_graph_locomotion.py @@ -9,26 +9,6 @@ class MinecraftGraphLocomotion(WorldAdapter): - supported_datasources = [ - 'health', - 'food', - 'temperature', - 'food_supply', - 'fatigue', - 'awake', - 'current_location_index' - ] - - supported_datatargets = [ - 'take_exit_one', - 'take_exit_two', - 'take_exit_three', - 'pitch', - 'yaw', - 'eat', - 'sleep' - ] - loco_node_template = { 'uid': "", 'name': "", @@ -168,16 +148,22 @@ class MinecraftGraphLocomotion(WorldAdapter): def __init__(self, world, uid=None, **data): super().__init__(world, uid, **data) - self.datatarget_feedback = {} + self.datasources = { + 'health': 1, + 'food': 1, + 'temperature': 0.5, + 'food_supply': 0, + 'fatigue': 0, + 'awake': 1, + 'current_location_index': 0 + } + + targets = ['take_exit_one', 'take_exit_two', 'take_exit_three', 'pitch', 'yaw', 'eat', 'sleep'] self.datatarget_history = {} - for key in self.datatargets: - self.datatarget_feedback[key] = 0 - self.datatarget_history[key] = 0 - - self.datasources['health'] = 1 - self.datasources['food'] = 1 - self.datasources['temperature'] = 0.5 - self.datasources['awake'] = 1 + for t in targets: + self.datatargets[t] = 0 + self.datatarget_feedback[t] = 0 + self.datatarget_history[t] = 0 # a collection of conditions to check on every update(..), eg., for action feedback self.waiting_list = [] diff --git a/micropsi_core/world/minecraft/minecraft_histogram_vision.py b/micropsi_core/world/minecraft/minecraft_histogram_vision.py index 6572aa9b..93ba2f0d 100644 --- a/micropsi_core/world/minecraft/minecraft_histogram_vision.py +++ b/micropsi_core/world/minecraft/minecraft_histogram_vision.py @@ -6,32 +6,6 @@ class MinecraftHistogramVision(MinecraftGraphLocomotion, MinecraftProjectionMixin): - supported_datasources = MinecraftGraphLocomotion.supported_datasources + [ - 'fov_x', # fovea sensors receive their input from the fovea actors - 'fov_y', - 'fov_hist__-01', # these names must be the most commonly observed block types - 'fov_hist__000', - 'fov_hist__001', - 'fov_hist__002', - 'fov_hist__003', - 'fov_hist__004', - 'fov_hist__009', - 'fov_hist__012', - 'fov_hist__017', - 'fov_hist__018', - 'fov_hist__020', - 'fov_hist__026', - 'fov_hist__031', - 'fov_hist__064', - 'fov_hist__106', - ] - - supported_datatargets = MinecraftGraphLocomotion.supported_datatargets + [ - 'orientation', - 'fov_x', - 'fov_y' - ] - # specs for vision /fovea # focal length larger 0 means zoom in, smaller 0 means zoom out # ( small values of focal length distort the image if things are close ) @@ -56,6 +30,31 @@ class MinecraftHistogramVision(MinecraftGraphLocomotion, MinecraftProjectionMixi def __init__(self, world, uid=None, **data): super().__init__(world, uid, **data) + self.datasources.update({ + 'fov_x': 0, # fovea sensors receive their input from the fovea actors + 'fov_y': 0, + 'fov_hist__-01': 0, # these names must be the most commonly observed block types + 'fov_hist__000': 0, + 'fov_hist__001': 0, + 'fov_hist__002': 0, + 'fov_hist__003': 0, + 'fov_hist__004': 0, + 'fov_hist__009': 0, + 'fov_hist__012': 0, + 'fov_hist__017': 0, + 'fov_hist__018': 0, + 'fov_hist__020': 0, + 'fov_hist__026': 0, + 'fov_hist__031': 0, + 'fov_hist__064': 0, + 'fov_hist__106': 0, + }) + + self.datatargets.update({ + 'orientation': 0, + 'fov_x': 0, + 'fov_y': 0 + }) # add datasources for fovea for i in range(self.num_fov): diff --git a/micropsi_core/world/worldadapter.py b/micropsi_core/world/worldadapter.py index 147a302b..29a48a7c 100644 --- a/micropsi_core/world/worldadapter.py +++ b/micropsi_core/world/worldadapter.py @@ -29,16 +29,10 @@ class WorldAdapter(WorldObject): The agent writes activation values into data targets, and receives it from data sources. The world adapter takes care of translating between the world and these values at each world cycle. """ - supported_datasources = [] - supported_datatargets = [] def __init__(self, world, uid=None, **data): self.datasources = {} - for key in self.supported_datasources: - self.datasources[key] = 0 self.datatargets = {} - for key in self.supported_datatargets: - self.datatargets[key] = 0 self.datatarget_feedback = {} self.datasource_lock = Lock() self.datasource_snapshots = {} @@ -92,7 +86,7 @@ def update(self): def reset_datatargets(self): """ resets (zeros) the datatargets """ - for datatarget in self.supported_datatargets: + for datatarget in self.datatargets: self.datatargets[datatarget] = 0 def update_data_sources_and_targets(self): @@ -106,8 +100,9 @@ def is_alive(self): class Default(WorldAdapter): - supported_datasources = ['static_on', 'random', 'static_off'] - supported_datatargets = ['echo'] + def __init__(self, world, uid=None, **data): + self.datasources = dict((s, 0) for s in ['static_on', 'random', 'static_off']) + self.datatargets = {'echo': 0} def update_data_sources_and_targets(self): import random From 5e13e8b1a6f28072001e57ad2c0a9923ce6ebada Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 5 Feb 2016 16:03:19 +0100 Subject: [PATCH 006/306] *finally* sort the worldadapters in the creation-dialog --- micropsi_server/view/worldadapter_selector.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_server/view/worldadapter_selector.tpl b/micropsi_server/view/worldadapter_selector.tpl index 734f58d0..76b4558e 100644 --- a/micropsi_server/view/worldadapter_selector.tpl +++ b/micropsi_server/view/worldadapter_selector.tpl @@ -3,7 +3,7 @@ % if not world_uid in worlds: % else: - % for type in worlds[world_uid].supported_worldadapters: + % for type in sorted(worlds[world_uid].supported_worldadapters.keys()): % if defined("nodenet_uid") and nodenet_uid in nodenets and nodenets[nodenet_uid].worldadapter == type: % else: From c862306c8507002cb1c8409171a296bb767f9c21 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Fri, 5 Feb 2016 18:18:28 +0100 Subject: [PATCH 007/306] Keep input stable for 3 nodenet steps per timeseries entry so FF reaches output nodes --- micropsi_core/world/timeseries/timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 8f41dfcb..17e8a6cf 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -38,7 +38,7 @@ def __init__(self, filename, world_type="Island", name="", owner="", engine=None @property def t(self): """current index into the original time series""" - return self.current_step % self.len_ts + return (self.current_step / 3) % self.len_ts @property def state(self): From 1dbe2a8bf59a5ee0896a0617b4a6cd16d54570fd Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 5 Feb 2016 18:40:30 +0100 Subject: [PATCH 008/306] make worlds configurable worlds can now define (in a static getter method) a list of configuration options. the frontend will ask the user for values for these options, and pass them to the world's init in the config-parameter config parameters once set will be persisted and loaded again with the world data --- micropsi_core/_runtime_api_world.py | 17 ++------- micropsi_core/runtime.py | 2 + micropsi_core/tests/test_node_logic.py | 2 +- .../tests/test_runtime_world_basics.py | 10 +++++ micropsi_core/world/island/island.py | 2 +- micropsi_core/world/minecraft/minecraft.py | 2 +- micropsi_core/world/world.py | 19 +++++++++- micropsi_server/micropsi_app.py | 13 +++++-- micropsi_server/view/world_form.tpl | 37 +++++++++++++++++++ 9 files changed, 83 insertions(+), 21 deletions(-) diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index 006218a8..7ed4f414 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -88,7 +88,7 @@ def set_worldagent_properties(world_uid, uid, position=None, orientation=None, n return micropsi_core.runtime.worlds[world_uid].set_agent_properties(uid, position, orientation, name, parameters) -def new_world(world_name, world_type, owner="", uid=None): +def new_world(world_name, world_type, owner="", uid=None, config={}): """Creates a new world and registers it. Arguments: @@ -106,7 +106,7 @@ def new_world(world_name, world_type, owner="", uid=None): filename = os.path.join(micropsi_core.runtime.RESOURCE_PATH, micropsi_core.runtime.WORLD_DIRECTORY, uid + ".json") micropsi_core.runtime.world_data[uid] = Bunch(uid=uid, name=world_name, world_type=world_type, filename=filename, version=1, - owner=owner) + owner=owner, config=config) with open(filename, 'w+') as fp: fp.write(json.dumps(micropsi_core.runtime.world_data[uid], sort_keys=True, indent=4)) try: @@ -201,16 +201,7 @@ def get_world_class_from_name(world_type): def get_available_world_types(): - """Returns the names of the available world types""" + """Returns a mapping of the available world type names to their classes""" import importlib from micropsi_core.world.world import World - return [cls.__name__ for cls in tools.itersubclasses(vars()['World'])] - for cls in tools.itersubclasses(World): - if 'minecraft' in cls.__name__.toLower(): - try: - import spock - except: - # requirement not satisfied, ignore - continue - available_worlds.append(cls.__name__) - return available_worlds + return dict((cls.__name__, cls) for cls in tools.itersubclasses(vars()['World'])) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 6d8dbb29..2bfaeb74 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1231,6 +1231,8 @@ def parse_definition(json, filename=None): result['world_type'] = json['world_type'] if "settings" in json: result['settings'] = json['settings'] + if "config" in json: + result['config'] = json['config'] return Bunch(**result) diff --git a/micropsi_core/tests/test_node_logic.py b/micropsi_core/tests/test_node_logic.py index 6887dd59..06706b6e 100644 --- a/micropsi_core/tests/test_node_logic.py +++ b/micropsi_core/tests/test_node_logic.py @@ -14,7 +14,7 @@ class DummyWorld(World): supported_worldadapters = ['DummyWorldAdapter'] - def __init__(self, filename, world_type="DummyWorld", name="", owner="", uid=None, version=1): + def __init__(self, filename, world_type="DummyWorld", name="", owner="", uid=None, version=1, config={}): World.__init__(self, filename, world_type=world_type, name=name, owner=owner, uid=uid, version=version) self.current_step = 0 self.data['assets'] = {} diff --git a/micropsi_core/tests/test_runtime_world_basics.py b/micropsi_core/tests/test_runtime_world_basics.py index 76ece5be..707122fa 100644 --- a/micropsi_core/tests/test_runtime_world_basics.py +++ b/micropsi_core/tests/test_runtime_world_basics.py @@ -200,6 +200,16 @@ def test_worldadapter_update_calls_reset_datatargets(test_world, test_nodenet): runtime.step_nodenet(test_nodenet) world.agents[test_nodenet].reset_datatargets.assert_called_once_with() + +def test_worlds_are_configurable(): + res, uid = runtime.new_world('testworld', 'Island', config={'foo': 'bar', '42': '23'}) + assert uid in runtime.worlds + assert runtime.worlds[uid].data['config']['foo'] == 'bar' + runtime.revert_world(uid) + assert runtime.worlds[uid].data['config']['foo'] == 'bar' + assert runtime.worlds[uid].data['config']['42'] == '23' + + """ def test_get_world_view(micropsi, test_world): assert 0 diff --git a/micropsi_core/world/island/island.py b/micropsi_core/world/island/island.py index 2407e8c5..6e04f6d6 100644 --- a/micropsi_core/world/island/island.py +++ b/micropsi_core/world/island/island.py @@ -43,7 +43,7 @@ class Island(World): } } - def __init__(self, filename, world_type="Island", name="", owner="", engine=None, uid=None, version=1): + def __init__(self, filename, world_type="Island", name="", owner="", engine=None, uid=None, version=1, config={}): World.__init__(self, filename, world_type=world_type, name=name, owner=owner, uid=uid, version=version) self.load_groundmap() # self.current_step = 0 diff --git a/micropsi_core/world/minecraft/minecraft.py b/micropsi_core/world/minecraft/minecraft.py index d20da8dc..4907291c 100644 --- a/micropsi_core/world/minecraft/minecraft.py +++ b/micropsi_core/world/minecraft/minecraft.py @@ -41,7 +41,7 @@ class Minecraft(World): 'thread': None } - def __init__(self, filename, world_type="Minecraft", name="", owner="", engine=None, uid=None, version=1): + def __init__(self, filename, world_type="Minecraft", name="", owner="", engine=None, uid=None, version=1, config={}): """ Initializes spock client including MicropsiPlugin, starts minecraft communication thread. """ diff --git a/micropsi_core/world/world.py b/micropsi_core/world/world.py index 6517dbdd..0a30899c 100644 --- a/micropsi_core/world/world.py +++ b/micropsi_core/world/world.py @@ -65,9 +65,23 @@ def is_active(self): def is_active(self, is_active): self.data['is_active'] = is_active + @staticmethod + def get_config_options(): + """ Returns a list of configuration-options for this world. + Expected format: + [{ + 'name': 'param1', + 'description': 'this is just an example', + 'options': ['value1', 'value2'], + 'default': 'value1' + }] + description, options and default are optional settings + """ + return [] + supported_worldadapters = ['Default'] - def __init__(self, filename, world_type="", name="", owner="", uid=None, engine=None, version=WORLD_VERSION): + def __init__(self, filename, world_type="", name="", owner="", uid=None, engine=None, version=WORLD_VERSION, config={}): """Create a new MicroPsi simulation environment. Arguments: @@ -84,7 +98,8 @@ def __init__(self, filename, world_type="", name="", owner="", uid=None, engine= "version": WORLD_VERSION, # used to check compatibility of the world data "objects": {}, "agents": {}, - "current_step": 0 + "current_step": 0, + "config": config } folder = self.__module__.split('.') diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index ad41fe89..82a6f19a 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -617,7 +617,9 @@ def edit_world_form(): token = request.get_cookie("token") id = request.params.get('id', None) title = 'Edit World' if id is not None else 'New World' - return template("world_form.tpl", title=title, worldtypes=runtime.get_available_world_types(), + worldtypes = runtime.get_available_world_types() + return template("world_form.tpl", title=title, + worldtypes=worldtypes, version=VERSION, user_id=usermanager.get_user_id_for_session_token(token), permissions=usermanager.get_permissions_for_session_token(token)) @@ -626,9 +628,14 @@ def edit_world_form(): @micropsi_app.route("/world/edit", method="POST") def edit_world(): params = dict((key, request.forms.getunicode(key)) for key in request.forms) + type = params['world_type'] + config = {} + for p in params: + if p.startswith(type + '_'): + config[p[len(type) + 1:]] = params[p] user_id, permissions, token = get_request_data() if "manage worlds" in permissions: - result, uid = runtime.new_world(params['world_name'], params['world_type'], user_id) + result, uid = runtime.new_world(params['world_name'], params['world_type'], user_id, config=config) if result: return dict(status="success", msg="World created", world_uid=uid) else: @@ -914,7 +921,7 @@ def new_world(world_name, world_type, owner=None): @rpc("get_available_world_types") def get_available_world_types(): - return True, runtime.get_available_world_types() + return True, sorted(runtime.get_available_world_types().keys()) @rpc("delete_world", permission_required="manage worlds") diff --git a/micropsi_server/view/world_form.tpl b/micropsi_server/view/world_form.tpl index d189b1d2..64c72154 100644 --- a/micropsi_server/view/world_form.tpl +++ b/micropsi_server/view/world_form.tpl @@ -52,6 +52,35 @@ + %for type in worldtypes: + % for param in worldtypes[type].get_config_options(): + + %end + %end + + %if defined("world"): %end @@ -67,3 +96,11 @@ + + \ No newline at end of file From f6df2676231c98721e6f242269921fe1a0391ff9 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 5 Feb 2016 21:49:44 +0100 Subject: [PATCH 009/306] remove newlines for js --- micropsi_server/view/worldadapter_selector.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_server/view/worldadapter_selector.tpl b/micropsi_server/view/worldadapter_selector.tpl index 76b4558e..850d249c 100644 --- a/micropsi_server/view/worldadapter_selector.tpl +++ b/micropsi_server/view/worldadapter_selector.tpl @@ -20,7 +20,7 @@ $(function(){ var adapters = {}; %for name in worlds[world_uid].supported_worldadapters: - adapters["{{name}}"] = "{{worlds[world_uid].supported_worldadapters[name].__doc__ or ''}}"; + adapters["{{name}}"] = "{{(worlds[world_uid].supported_worldadapters[name].__doc__ or '').replace('\n', ' ')}}"; %end var el = $('#nn_worldadapter'); var updateDescription = function(){ From 39e46e9c6e687fd56bb4fb147793e6982fe6f674 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 8 Feb 2016 16:46:12 +0100 Subject: [PATCH 010/306] fix deleting worlds --- micropsi_core/_runtime_api_world.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index 7ed4f414..757ea0d5 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -120,9 +120,9 @@ def new_world(world_name, world_type, owner="", uid=None, config={}): def delete_world(world_uid): """Removes the world with the given uid from the server (and unloads it from memory if it is running.)""" world = micropsi_core.runtime.worlds[world_uid] - for uid in world.agents: - world.unregister_agent(micropsi_core.runtime.nodenets[uid]) - nodenets[uid].world = None + for uid in list(world.agents.keys()): + world.unregister_nodenet(micropsi_core.runtime.nodenets[uid]) + micropsi_core.runtime.nodenets[uid].world = None del micropsi_core.runtime.worlds[world_uid] os.remove(micropsi_core.runtime.world_data[world_uid].filename) del micropsi_core.runtime.world_data[world_uid] From 5d6718b66022d118defe98bbf13b0ddeb7c111ca Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 8 Feb 2016 16:46:36 +0100 Subject: [PATCH 011/306] prevent 2 instances of minecraft being instantiated at once --- micropsi_core/_runtime_api_world.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index 757ea0d5..2a1cb568 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -103,6 +103,11 @@ def new_world(world_name, world_type, owner="", uid=None, config={}): if uid is None: uid = tools.generate_uid() + if world_type.startswith('Minecraft'): + for uid in micropsi_core.runtime.worlds: + if micropsi_core.runtime.worlds[uid].__class__.__name__.startswith('Minecraft'): + raise RuntimeError("Only one instance of a minecraft world is supported right now") + filename = os.path.join(micropsi_core.runtime.RESOURCE_PATH, micropsi_core.runtime.WORLD_DIRECTORY, uid + ".json") micropsi_core.runtime.world_data[uid] = Bunch(uid=uid, name=world_name, world_type=world_type, filename=filename, version=1, From 08750cc30e53aca99311943dad5b9cefe74012d4 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 8 Feb 2016 16:47:35 +0100 Subject: [PATCH 012/306] revert-icon now reverts both nodenet and world --- micropsi_core/_runtime_api_world.py | 3 +++ micropsi_core/runtime.py | 4 +++- micropsi_core/world/world.py | 4 ++++ micropsi_server/micropsi_app.py | 5 +++++ micropsi_server/static/js/dialogs.js | 6 +++--- micropsi_server/tests/test_json_api.py | 14 ++++++++++++++ micropsi_server/view/menu.tpl | 2 +- 7 files changed, 33 insertions(+), 5 deletions(-) diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index 2a1cb568..f7bd3f0e 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -159,6 +159,9 @@ def set_world_properties(world_uid, world_name=None, owner=None): def revert_world(world_uid): """Reverts the world to the last saved state.""" data = micropsi_core.runtime.world_data[world_uid] + if world_uid in micropsi_core.runtime.worlds: + micropsi_core.runtime.worlds[world_uid].__del__() + del micropsi_core.runtime.worlds[world_uid] micropsi_core.runtime.worlds[world_uid] = get_world_class_from_name(data.world_type)(**data) return True diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 2bfaeb74..69dcf95c 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -564,8 +564,10 @@ def step_nodenet(nodenet_uid): return nodenets[nodenet_uid].current_step -def revert_nodenet(nodenet_uid): +def revert_nodenet(nodenet_uid, also_revert_world=False): """Returns the nodenet to the last saved state.""" + if also_revert_world and nodenet_uid in nodenets and nodenets[nodenet_uid].world: + revert_world(nodenets[nodenet_uid].world) unload_nodenet(nodenet_uid) load_nodenet(nodenet_uid) return True diff --git a/micropsi_core/world/world.py b/micropsi_core/world/world.py index 0a30899c..5320880a 100644 --- a/micropsi_core/world/world.py +++ b/micropsi_core/world/world.py @@ -322,6 +322,10 @@ def set_agent_properties(self, uid, position=None, orientation=None, name=None, return True return False + def __del__(self): + """Empty destructor""" + pass + # imports of individual world types: try: diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index 82a6f19a..432bb625 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -827,6 +827,11 @@ def step_simulation(nodenet_uid): return True, runtime.step_nodenet(nodenet_uid) +@rpc("revert_simulation", permission_required="manage nodenets") +def revert_simulation(nodenet_uid): + return runtime.revert_nodenet(nodenet_uid, True) + + @rpc("revert_nodenet", permission_required="manage nodenets") def revert_nodenet(nodenet_uid): return runtime.revert_nodenet(nodenet_uid) diff --git a/micropsi_server/static/js/dialogs.js b/micropsi_server/static/js/dialogs.js index a58f66c1..c35ed79c 100644 --- a/micropsi_server/static/js/dialogs.js +++ b/micropsi_server/static/js/dialogs.js @@ -837,13 +837,13 @@ function stopNodenetrunner(event){ }); } -function resetNodenet(event){ +function revertAll(event){ event.preventDefault(); nodenetRunning = false; if(currentNodenet){ $('#loading').show(); api.call( - 'revert_nodenet', + 'revert_simulation', {nodenet_uid: currentNodenet}, function(){ window.location.reload(); @@ -857,7 +857,7 @@ function resetNodenet(event){ $(function() { $('#nodenet_start').on('click', startNodenetrunner); $('#nodenet_stop').on('click', stopNodenetrunner); - $('#nodenet_reset').on('click', resetNodenet); + $('#revert_all').on('click', revertAll); $('#nodenet_step_forward').on('click', stepNodenet); }); diff --git a/micropsi_server/tests/test_json_api.py b/micropsi_server/tests/test_json_api.py index e6e3396e..cf775209 100644 --- a/micropsi_server/tests/test_json_api.py +++ b/micropsi_server/tests/test_json_api.py @@ -277,6 +277,20 @@ def test_revert_nodenet(app, test_nodenet, test_world): assert data['worldadapter'] is None +def test_revert_both(app, test_nodenet, test_world): + app.set_auth() + app.post_json('/rpc/set_nodenet_properties', params=dict(nodenet_uid=test_nodenet, worldadapter="Braitenberg", world_uid=test_world)) + for i in range(5): + app.get_json('/rpc/step_simulation(nodenet_uid="%s")' % test_nodenet) + res = app.get_json('/rpc/get_current_state(nodenet_uid="%s")' % test_nodenet) + assert res.json_body['data']['current_nodenet_step'] > 0 + assert res.json_body['data']['current_world_step'] > 0 + app.get_json('/rpc/revert_simulation(nodenet_uid="%s")' % test_nodenet) + res = app.get_json('/rpc/get_current_state(nodenet_uid="%s")' % test_nodenet) + assert res.json_body['data']['current_nodenet_step'] == 0 + assert res.json_body['data']['current_world_step'] == 0 + + def test_save_nodenet(app, test_nodenet, test_world): app.set_auth() response = app.post_json('/rpc/set_nodenet_properties', params=dict(nodenet_uid=test_nodenet, nodenet_name="new_name", worldadapter="Braitenberg", world_uid=test_world)) diff --git a/micropsi_server/view/menu.tpl b/micropsi_server/view/menu.tpl index 8359dfdf..17d5563e 100644 --- a/micropsi_server/view/menu.tpl +++ b/micropsi_server/view/menu.tpl @@ -116,7 +116,7 @@ World:0
Net:0 - + From 710b9aa4b81984384844c769bab612186a06a4d5 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 8 Feb 2016 16:48:40 +0100 Subject: [PATCH 013/306] make minecraft world revertable --- micropsi_core/world/minecraft/minecraft.py | 28 ++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/micropsi_core/world/minecraft/minecraft.py b/micropsi_core/world/minecraft/minecraft.py index 4907291c..4573f9c8 100644 --- a/micropsi_core/world/minecraft/minecraft.py +++ b/micropsi_core/world/minecraft/minecraft.py @@ -1,12 +1,12 @@ from threading import Thread +from spock import plugins as spockplugins from spock.client import Client -from spock.plugins import DefaultPlugins from spock.plugins.core.event import EventPlugin from spock.plugins.helpers.clientinfo import ClientInfoPlugin from spock.plugins.helpers.move import MovementPlugin -from spock.plugins.helpers.world import WorldPlugin from spock.plugins.helpers.reconnect import ReConnectPlugin +from spock.plugins.helpers.world import WorldPlugin from micropsi_core.world.world import World from micropsi_core.world.worldadapter import WorldAdapter @@ -35,24 +35,21 @@ class Minecraft(World): 'y': 256, } - # thread and spock only exist once - instances = { - 'spock': None, - 'thread': None - } - def __init__(self, filename, world_type="Minecraft", name="", owner="", engine=None, uid=None, version=1, config={}): """ Initializes spock client including MicropsiPlugin, starts minecraft communication thread. """ from micropsi_core.runtime import add_signal_handler + self.instances = { + 'spock': None, + 'thread': None + } # do spock things first, then initialize micropsi world because the latter requires self.spockplugin - # register all necessary spock plugins # DefaultPlugins contain EventPlugin, NetPlugin, TimerPlugin, AuthPlugin, # ThreadPoolPlugin, StartPlugin and KeepalivePlugin - plugins = DefaultPlugins + plugins = spockplugins.DefaultPlugins plugins.append(ClientInfoPlugin) plugins.append(MovementPlugin) plugins.append(WorldPlugin) @@ -127,10 +124,17 @@ def get_config(self): def kill_minecraft_thread(self, *args): """ """ - self.spockplugin.event.kill() - self.instances['thread'].join() + if hasattr(self, 'spockplugin'): + self.spockplugin.event.kill() + self.instances['thread'].join() # self.spockplugin.threadpool.shutdown(False) + def __del__(self): + from importlib import reload + self.kill_minecraft_thread() + reload(spockplugins) + + class Minecraft2D(Minecraft): """ mandatory: list of world adapters that are supported""" From d7a2ca6a1ad8ba863509c1012ca68e4fefe42570 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 8 Feb 2016 16:49:06 +0100 Subject: [PATCH 014/306] fix error handling of form dialogs --- micropsi_server/static/js/dialogs.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/micropsi_server/static/js/dialogs.js b/micropsi_server/static/js/dialogs.js index c35ed79c..8e03bbf7 100644 --- a/micropsi_server/static/js/dialogs.js +++ b/micropsi_server/static/js/dialogs.js @@ -106,6 +106,13 @@ var dialogs = { } else { dialogs.setModalForm(data, callback); } + }, + error: function(data, param1){ + if(data.status == 500){ + $('body').html(data.responseText); + } else { + dialogs.notification(data.statusText, "error"); + } } }); }, From b6b5c73e22c947e0ec2da0aaf18e5fb04a882155 Mon Sep 17 00:00:00 2001 From: cknd Date: Mon, 8 Feb 2016 18:01:18 +0100 Subject: [PATCH 015/306] timeseries world now applies a z transform per ID, todo: make this optional via doiks new config options --- micropsi_core/world/timeseries/timeseries.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 17e8a6cf..8ff96114 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -31,6 +31,18 @@ def __init__(self, filename, world_type="Island", name="", owner="", engine=None self.ids = f['ids'] self.startdate = f['startdate'] self.enddate = f['enddate'] + + z_transform = True # todo use the new configurable world options + if z_transform: + data_z = np.empty_like(self.timeseries) + data_z[:] = np.nan + for i,row in enumerate(self.timeseries): + if not np.all(np.isnan(row)): + var = np.nanvar(row) + if var > 0: + data_z[i,:] = (row-np.nanmean(row))/np.sqrt(var) + self.timeseries = data_z + self.len_ts = self.timeseries.shape[1] # todo: option to use only a subset of the data (e.g. for training/test) From fa919f1464ae1a105beaa6f8f3c6743c66ef305b Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 8 Feb 2016 18:21:32 +0100 Subject: [PATCH 016/306] get rid of bottle's deprecation warnings rebase and include changed from keywords to functions. --- micropsi_server/view/about.tpl | 4 ++-- micropsi_server/view/boilerplate.tpl | 4 ++-- micropsi_server/view/error.tpl | 4 ++-- micropsi_server/view/minidoc.tpl | 4 ++-- micropsi_server/view/nodenet_mgt.tpl | 4 ++-- micropsi_server/view/user_mgt.tpl | 4 ++-- micropsi_server/view/viewer.tpl | 21 +++++++++------------ micropsi_server/view/world.tpl | 2 +- 8 files changed, 22 insertions(+), 25 deletions(-) diff --git a/micropsi_server/view/about.tpl b/micropsi_server/view/about.tpl index ab32707a..d6a094ec 100644 --- a/micropsi_server/view/about.tpl +++ b/micropsi_server/view/about.tpl @@ -1,4 +1,4 @@ -%include menu.tpl version = version +%include("menu.tpl", version=version)

@@ -30,4 +30,4 @@

-%rebase boilerplate title = "About MicroPsi 2" +%rebase("boilerplate.tpl", title="About MicroPsi 2") diff --git a/micropsi_server/view/boilerplate.tpl b/micropsi_server/view/boilerplate.tpl index 2cf3466b..f38cdf50 100644 --- a/micropsi_server/view/boilerplate.tpl +++ b/micropsi_server/view/boilerplate.tpl @@ -43,11 +43,11 @@
-%include +{{!base}}
-%include dialogs.tpl +%include("dialogs.tpl") \ No newline at end of file diff --git a/micropsi_server/view/error.tpl b/micropsi_server/view/error.tpl index 07c67787..54a01644 100644 --- a/micropsi_server/view/error.tpl +++ b/micropsi_server/view/error.tpl @@ -1,4 +1,4 @@ -%include menu.tpl +%include("menu.tpl")
@@ -41,4 +41,4 @@ Life must go on
-%rebase boilerplate title = "MicroPsi Error" +%rebase("boilerplate.tpl", title="MicroPsi Error") diff --git a/micropsi_server/view/minidoc.tpl b/micropsi_server/view/minidoc.tpl index 6f24c61e..18e44b28 100644 --- a/micropsi_server/view/minidoc.tpl +++ b/micropsi_server/view/minidoc.tpl @@ -1,4 +1,4 @@ -%include menu.tpl +%include("menu.tpl")
@@ -16,4 +16,4 @@
-%rebase boilerplate title = "{{title}}" +%rebase("boilerplate.tpl", title = "{{title}}") diff --git a/micropsi_server/view/nodenet_mgt.tpl b/micropsi_server/view/nodenet_mgt.tpl index 08ef2dc6..2c0a060b 100644 --- a/micropsi_server/view/nodenet_mgt.tpl +++ b/micropsi_server/view/nodenet_mgt.tpl @@ -1,4 +1,4 @@ -%include menu.tpl version = version, permissions = permissions, user_id = user_id +%include("menu.tpl", version=version, permissions=permissions, user_id=user_id)

@@ -45,4 +45,4 @@ Save all nodenets

-%rebase boilerplate title = "Manage Nodenets" +%rebase("boilerplate.tpl", title="Manage Nodenets") diff --git a/micropsi_server/view/user_mgt.tpl b/micropsi_server/view/user_mgt.tpl index 0a884a42..87561df8 100644 --- a/micropsi_server/view/user_mgt.tpl +++ b/micropsi_server/view/user_mgt.tpl @@ -1,4 +1,4 @@ -%include menu.tpl version = version, permissions = permissions, user_id = user_id +%include("menu.tpl", version=version, permissions=permissions, user_id=user_id)

@@ -61,4 +61,4 @@

-%rebase boilerplate title = "Manage Users" +%rebase("boilerplate.tpl", title="Manage Users") diff --git a/micropsi_server/view/viewer.tpl b/micropsi_server/view/viewer.tpl index 56da4d4c..14db5250 100644 --- a/micropsi_server/view/viewer.tpl +++ b/micropsi_server/view/viewer.tpl @@ -1,5 +1,5 @@ -%include menu.tpl version = version, user_id = user_id, permissions = permissions +%include("menu.tpl", version=version, permissions=permissions, user_id=user_id) @@ -11,25 +11,22 @@

%if mode == "nodenet": - %include nodenet + %include("nodenet.tpl") % end %if mode == "monitors": - %include monitors logging_levels=logging_levels + %include("monitors.tpl", logging_levels=logging_levels) %end %if mode == "world": - %include world mine=mine,others=others,current=current,world_assets=world_assets -% end -%if mode == "face": - %include face + %include("world.tpl", mine=mine, others=others, current=current, world_assets=world_assets) % end %if mode == "dashboard": - %include dashboard logging_levels=logging_levels + %include("dashboard.tpl", logging_levels=logging_levels) % end %if mode == "all": - %include nodenet - %include monitors logging_levels=logging_levels - %include world mine=mine,others=others,current=current,world_assets=world_assets + %include("nodenet.tpl") + %include("monitors.tpl", logging_levels=logging_levels) + %include("world.tpl", mine=mine, others=others, current=current, world_assets=world_assets) % end %if defined('first_user') and first_user: @@ -41,4 +38,4 @@ %end
-%rebase boilerplate title = "MicroPsi Simulator" +%rebase("boilerplate.tpl", title="MicroPsi Simulator") diff --git a/micropsi_server/view/world.tpl b/micropsi_server/view/world.tpl index 72bf71e5..dee7dd99 100644 --- a/micropsi_server/view/world.tpl +++ b/micropsi_server/view/world.tpl @@ -12,7 +12,7 @@
- %include nodenet_list type="world",mine=mine,others=others,current=current + %include("nodenet_list.tpl", type="world", mine=mine, others=others, current=current)
From f408ecbe6ae5843e63fff3093f0445a38f350d67 Mon Sep 17 00:00:00 2001 From: cknd Date: Mon, 8 Feb 2016 19:22:13 +0100 Subject: [PATCH 017/306] added option to randomize the sequence of presented inputs (todo make it really optional i.e. configurable) --- micropsi_core/world/timeseries/timeseries.py | 21 +++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 8ff96114..f6a6dc7a 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -32,29 +32,32 @@ def __init__(self, filename, world_type="Island", name="", owner="", engine=None self.startdate = f['startdate'] self.enddate = f['enddate'] - z_transform = True # todo use the new configurable world options + self.shuffle = True # todo use the new configurable world options. + z_transform = True # todo use the new configurable world options + if z_transform: data_z = np.empty_like(self.timeseries) data_z[:] = np.nan - for i,row in enumerate(self.timeseries): + for i, row in enumerate(self.timeseries): if not np.all(np.isnan(row)): var = np.nanvar(row) if var > 0: - data_z[i,:] = (row-np.nanmean(row))/np.sqrt(var) + data_z[i,:] = (row - np.nanmean(row)) / np.sqrt(var) self.timeseries = data_z self.len_ts = self.timeseries.shape[1] # todo: option to use only a subset of the data (e.g. for training/test) - @property - def t(self): - """current index into the original time series""" - return (self.current_step / 3) % self.len_ts - @property def state(self): - return self.timeseries[:, self.t] + t = (self.current_step - 1) % self.len_ts + if self.shuffle: + if t == 0: + idxs = np.arange(self.len_ts) + self.permutation = np.random.permutation(idxs) + t = self.permutation[t] + return self.timeseries[:, t] class TimeSeriesRunner(WorldAdapter): From 468edd78ec9ddba6e1f68bc7061d33bbd5a5db4c Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 8 Feb 2016 19:43:58 +0100 Subject: [PATCH 018/306] explicitly call descructor when deleting as well --- micropsi_core/_runtime_api_world.py | 1 + 1 file changed, 1 insertion(+) diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index f7bd3f0e..dc21d4e9 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -128,6 +128,7 @@ def delete_world(world_uid): for uid in list(world.agents.keys()): world.unregister_nodenet(micropsi_core.runtime.nodenets[uid]) micropsi_core.runtime.nodenets[uid].world = None + micropsi_core.runtime.worlds[world_uid].__del__() del micropsi_core.runtime.worlds[world_uid] os.remove(micropsi_core.runtime.world_data[world_uid].filename) del micropsi_core.runtime.world_data[world_uid] From 127ef00772b4a2b266e0776b8518ed16cb473c1d Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 8 Feb 2016 19:45:08 +0100 Subject: [PATCH 019/306] fix minecraft2d projection visualisation --- micropsi_core/world/minecraft/minecraft.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/micropsi_core/world/minecraft/minecraft.py b/micropsi_core/world/minecraft/minecraft.py index 4573f9c8..bed7a60c 100644 --- a/micropsi_core/world/minecraft/minecraft.py +++ b/micropsi_core/world/minecraft/minecraft.py @@ -135,7 +135,6 @@ def __del__(self): reload(spockplugins) - class Minecraft2D(Minecraft): """ mandatory: list of world adapters that are supported""" supported_worldadapters = [ @@ -157,6 +156,16 @@ def step(self): # a 2D perspective projection self.get_perspective_projection(self.spockplugin.clientinfo.position) + def get_world_view(self, step): + """ returns a list of world objects, and the current step of the simulation """ + return { + 'objects': self.get_world_objects(), + 'agents': self.data.get('agents', {}), + 'current_step': self.current_step, + 'projection': self.data['projection'], + 'assets': self.assets + } + def get_perspective_projection(self, agent_info): """ """ From bf73a4ece580d669dec7d93c12b258c74261f3fe Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 8 Feb 2016 19:53:08 +0100 Subject: [PATCH 020/306] show blockworld iframe for standard minecraft world --- micropsi_server/static/minecraft/minecraft.js | 62 ++----------------- .../static/minecraft/minecraft.tpl | 15 ----- 2 files changed, 5 insertions(+), 72 deletions(-) diff --git a/micropsi_server/static/minecraft/minecraft.js b/micropsi_server/static/minecraft/minecraft.js index ab6ae0d1..01375f5f 100644 --- a/micropsi_server/static/minecraft/minecraft.js +++ b/micropsi_server/static/minecraft/minecraft.js @@ -32,41 +32,12 @@ function get_world_data(){ } function set_world_data(data){ currentWorldSimulationStep = data.current_step; + + agent_html = ''; for (var key in data.agents) { - if (data.agents[key].minecraft_vision_pixel) { - - if (current_layer == 1) { - console.log("activating second layer ..."); - secondLayer.activate(); - } - else{ - console.log("activating first layer ..."); - firstLayer.activate(); - } - - var minecraft_pixel = data.agents[key].minecraft_vision_pixel; - for (var x = 0; x < WIDTH; x++) { - for (var y = 0; y < HEIGHT; y++) { - - var raster = new Raster('mc_block_img_' + minecraft_pixel[(y + x * HEIGHT) * 2]); - raster.position = new Point(world.width / WIDTH * x, world.height / HEIGHT * y); - var distance = minecraft_pixel[(y + x * HEIGHT) * 2 + 1]; - raster.scale((world.width / WIDTH) / 64 * (1 / Math.pow(distance, 1 / 5)), (world.height / HEIGHT) / 64 * (1 / Math.pow(distance, 1 / 5))); - } - } - if (current_layer == 1) { - console.log("removing frist layer children ..."); - firstLayer.removeChildren(); - current_layer = 0; - } - else{ - console.log("removing frist layer children ..."); - secondLayer.removeChildren(); - current_layer = 1; - } - break; - } + agent_html += ""+data.agents[key].name+ ' ('+data.agents[key].type+')'; } + $('#world_agents_list table').html(agent_html); updateViewSize(); if (worldRunning) { @@ -112,35 +83,12 @@ function setCurrentWorld(uid) { expires: 7, path: '/' }); - loadWorldInfo(); -} - -function loadWorldInfo() { - - var all_images = "" - - var editor_div = $("#world_forms"); - - $.getScript('/static/minecraft/minecraft_struct.js', function () { - for (var i = -1; i < 173; i++) { - - var block_name = block_names["" + i]; - all_images = all_images + ''; - - editor_div.html('
' + all_images + '
'); - } - }); - - editor_div.html('
' + all_images + '
'); - - firstLayer = project.activeLayer; - secondLayer = new Layer(); - firstLayer.activate(); api.call('get_world_properties', { world_uid: currentWorld }, success = function (data) { refreshWorldView(); + $('#world').parent().html(''); }, error = function (data) { $.cookie('selected_world', '', { expires: -1, diff --git a/micropsi_server/static/minecraft/minecraft.tpl b/micropsi_server/static/minecraft/minecraft.tpl index 81c82886..0c919743 100644 --- a/micropsi_server/static/minecraft/minecraft.tpl +++ b/micropsi_server/static/minecraft/minecraft.tpl @@ -8,26 +8,11 @@

World Status

-
-

Scene Viewer

-

- - -

-
-

Agents

-

World Objects

-
- - -
-
\ No newline at end of file From 7fecf7a4d3550a6f45a6e815276baaaaf7515e11 Mon Sep 17 00:00:00 2001 From: cknd Date: Tue, 9 Feb 2016 15:41:34 +0100 Subject: [PATCH 021/306] support input squashing & clipping --- micropsi_core/world/timeseries/timeseries.py | 38 +++++++++++++++----- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index f6a6dc7a..4c19c641 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -32,19 +32,41 @@ def __init__(self, filename, world_type="Island", name="", owner="", engine=None self.startdate = f['startdate'] self.enddate = f['enddate'] - self.shuffle = True # todo use the new configurable world options. - z_transform = True # todo use the new configurable world options + # todo use the new configurable world options. + self.shuffle = True # randomize order of presentation + z_transform = True # for each ID, center on mean & normalize by standard deviation + clip_and_scale = False # for each ID, center on mean & clip to 4 standard deviations and rescale to [0,1]. + sigmoid = True # for each ID, z-transform and apply a sigmoid activation function + assert(not (clip_and_scale and sigmoid)) - if z_transform: + def sigm(X): + """ sigmoid that avoids float overflows for very small inputs. + expects a numpy float array. + """ + cutoff = np.log(np.finfo(X.dtype).max) - 1 + X[np.nan_to_num(X) <= -cutoff] = -cutoff + return 1. / (1. + np.exp(-X)) + + + if z_transform or clip_and_scale or sigmoid: data_z = np.empty_like(self.timeseries) data_z[:] = np.nan + pstds = [] for i, row in enumerate(self.timeseries): if not np.all(np.isnan(row)): - var = np.nanvar(row) - if var > 0: - data_z[i,:] = (row - np.nanmean(row)) / np.sqrt(var) - self.timeseries = data_z - + std = np.sqrt(np.nanvar(row)) + if std > 0: + if not clip_and_scale: + row_z = (row - np.nanmean(row)) / std + if clip_and_scale: + row_z = row - np.nanmean(row) + pstd = std * 4 + row_z[np.nan_to_num(row_z) > pstd] = pstd + row_z[np.nan_to_num(row_z) < -pstd] = -pstd + row_z = ((row_z / pstd) + 1) * 0.5 + data_z[i,:] = row_z + self.timeseries = data_z if not sigmoid else sigm(data_z) + # import ipdb; ipdb.set_trace() self.len_ts = self.timeseries.shape[1] # todo: option to use only a subset of the data (e.g. for training/test) From 0f82826186755a31bd3bd3c982c6513cbc585323 Mon Sep 17 00:00:00 2001 From: cknd Date: Tue, 9 Feb 2016 16:33:22 +0100 Subject: [PATCH 022/306] option to spout dummy data --- micropsi_core/world/timeseries/timeseries.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 4c19c641..444b20fd 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -26,6 +26,7 @@ def __init__(self, filename, world_type="Island", name="", owner="", engine=None World.__init__(self, filename, world_type=world_type, name=name, owner=owner, uid=uid, version=version) path = os.path.join(cfg['micropsi2']['data_directory'], 'timeseries.npz') print("loading timeseries from", path, "for world", uid) + with np.load(path) as f: self.timeseries = f['data'] self.ids = f['ids'] @@ -33,10 +34,11 @@ def __init__(self, filename, world_type="Island", name="", owner="", engine=None self.enddate = f['enddate'] # todo use the new configurable world options. - self.shuffle = True # randomize order of presentation + dummydata = False # present the same random pattern in each step. z_transform = True # for each ID, center on mean & normalize by standard deviation clip_and_scale = False # for each ID, center on mean & clip to 4 standard deviations and rescale to [0,1]. sigmoid = True # for each ID, z-transform and apply a sigmoid activation function + self.shuffle = True # randomize order of presentation assert(not (clip_and_scale and sigmoid)) def sigm(X): @@ -48,7 +50,7 @@ def sigm(X): return 1. / (1. + np.exp(-X)) - if z_transform or clip_and_scale or sigmoid: + if (z_transform or clip_and_scale or sigmoid) and not dummydata: data_z = np.empty_like(self.timeseries) data_z[:] = np.nan pstds = [] @@ -66,7 +68,12 @@ def sigm(X): row_z = ((row_z / pstd) + 1) * 0.5 data_z[i,:] = row_z self.timeseries = data_z if not sigmoid else sigm(data_z) - # import ipdb; ipdb.set_trace() + + if dummydata: + print("! Using dummy data") + n_ids = self.timeseries.shape[0] + self.timeseries = np.tile(np.random.rand(n_ids,1),(1,10)) + self.len_ts = self.timeseries.shape[1] # todo: option to use only a subset of the data (e.g. for training/test) From 01c0af04ab66337901776d0c2ba50c0964c6f858 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 9 Feb 2016 18:15:15 +0100 Subject: [PATCH 023/306] search for nodetypes, nodefunctions, recipes recursively All nodefunctions.py, nodetypes.json and recipes.py files that reside within the RESOURCE_PATH (in any directory-depth) will be respected. --- micropsi_core/nodenet/node.py | 20 ++++--- micropsi_core/runtime.py | 99 +++++++++++++++++++++-------------- 2 files changed, 75 insertions(+), 44 deletions(-) diff --git a/micropsi_core/nodenet/node.py b/micropsi_core/nodenet/node.py index 40f4a841..21774398 100644 --- a/micropsi_core/nodenet/node.py +++ b/micropsi_core/nodenet/node.py @@ -593,14 +593,20 @@ def nodefunction_name(self): @nodefunction_name.setter def nodefunction_name(self, nodefunction_name): + import os + from importlib.machinery import SourceFileLoader + from micropsi_core.runtime import RESOURCE_PATH self._nodefunction_name = nodefunction_name try: - from micropsi_core.nodenet import nodefunctions - if hasattr(nodefunctions, nodefunction_name): - self.nodefunction = getattr(nodefunctions, nodefunction_name) + if self.path: + module = SourceFileLoader("nodefunctions", os.path.join(RESOURCE_PATH, self.path, "nodefunctions.py")).load_module() + self.nodefunction = getattr(module, nodefunction_name) else: - import nodefunctions as custom_nodefunctions - self.nodefunction = getattr(custom_nodefunctions, nodefunction_name) + from micropsi_core.nodenet import nodefunctions + if hasattr(nodefunctions, nodefunction_name): + self.nodefunction = getattr(nodefunctions, nodefunction_name) + else: + self.logger.warn("Can not find definition of nodefunction %s" % nodefunction_name) except (ImportError, AttributeError) as err: self.logger.warn("Import error while importing node function: nodefunctions.%s %s" % (nodefunction_name, err)) @@ -608,7 +614,7 @@ def nodefunction_name(self, nodefunction_name): def __init__(self, name, nodenet, slottypes=None, gatetypes=None, parameters=None, nodefunction_definition=None, nodefunction_name=None, parameter_values=None, gate_defaults=None, - symbol=None, shape=None, engine=None, parameter_defaults=None): + symbol=None, shape=None, engine=None, parameter_defaults=None, path=''): """Initializes or creates a nodetype. Arguments: @@ -627,6 +633,8 @@ def __init__(self, name, nodenet, slottypes=None, gatetypes=None, parameters=Non self.slottypes = slottypes or {} self.gatetypes = gatetypes or {} + self.path = path + self.logger = nodenet.logger self.gate_defaults = {} diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 6d8dbb29..9f8f8561 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1267,37 +1267,51 @@ def init_worlds(world_data): return worlds -def load_user_files(do_reload=False): - # see if we have additional nodetypes defined by the user. - import sys +def load_user_files(path=RESOURCE_PATH, reload_nodefunctions=False): + global native_modules, custom_recipes + for f in os.listdir(path): + if not f.startswith('.') and f != '__pycache__': + abspath = os.path.join(path, f) + if os.path.isdir(abspath): + load_user_files(abspath) + elif f == 'nodetypes.json': + parse_native_module_file(abspath) + elif f == 'recipes.py': + parse_recipe_file(abspath, reload_nodefunctions) + elif f == 'nodefunctions.py' and reload_nodefunctions: + reload_nodefunctions_file(abspath) + + +def parse_native_module_file(path): global native_modules - native_modules = {} - custom_nodetype_file = os.path.join(RESOURCE_PATH, 'nodetypes.json') - if os.path.isfile(custom_nodetype_file): - try: - with open(custom_nodetype_file) as fp: - native_modules = json.load(fp) - except ValueError: - logging.getLogger('system').warn("Nodetype data in %s not well-formed." % custom_nodetype_file) - - sys.path.append(RESOURCE_PATH) - parse_recipe_file() - return native_modules - - -def parse_recipe_file(): - custom_recipe_file = os.path.join(RESOURCE_PATH, 'recipes.py') - if not os.path.isfile(custom_recipe_file): - return - + try: + with open(path) as fp: + modules = json.load(fp) + diffpath = os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) + for key in modules: + modules[key]['path'] = diffpath + if key in native_modules: + logging.getLogger("system").warn("Native module names must be unique. %s is not." % key) + native_modules[key] = modules[key] + sys.path.append(path) + except ValueError: + logging.getLogger('system').warn("Nodetype data in %s not well-formed." % path) + + +def parse_recipe_file(path, reload=False): + global custom_recipes import importlib.machinery import inspect - global custom_recipes - loader = importlib.machinery.SourceFileLoader("recipes", custom_recipe_file) + diffpath = os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) + loader = importlib.machinery.SourceFileLoader("recipes", path) recipes = loader.load_module() # import recipes - custom_recipes = {} + all_modules = inspect.getmembers(recipes, inspect.ismodule) + for name, module in all_modules: + if module.__file__.startswith(RESOURCE_PATH): + loader = importlib.machinery.SourceFileLoader(name, module.__file__) + loader.load_module() all_functions = inspect.getmembers(recipes, inspect.isfunction) for name, func in all_functions: argspec = inspect.getargspec(func) @@ -1314,33 +1328,40 @@ def parse_recipe_file(): 'name': arg, 'default': default }) + if name in custom_recipes: + logging.getLogger("system").warn("Recipe function names must be unique. %s is not." % name) custom_recipes[name] = { 'name': name, 'parameters': params, 'function': func, - 'docstring': inspect.getdoc(func) + 'docstring': inspect.getdoc(func), + 'path': diffpath } +def reload_nodefunctions_file(path): + import importlib + import inspect + + loader = importlib.machinery.SourceFileLoader("nodefunctions", path) + nodefuncs = loader.load_module() + for name, module in inspect.getmembers(nodefuncs, inspect.ismodule): + if module.__file__.startswith(RESOURCE_PATH): + loader = importlib.machinery.SourceFileLoader(name, module.__file__) + loader.load_module() + + def reload_native_modules(): # stop nodenets, save state + global native_modules, custom_recipes + native_modules = {} + custom_recipes = {} runners = {} for uid in nodenets: if nodenets[uid].is_active: runners[uid] = True nodenets[uid].is_active = False - load_user_files(True) - import importlib - import inspect - custom_nodefunctions_file = os.path.join(RESOURCE_PATH, 'nodefunctions.py') - if os.path.isfile(custom_nodefunctions_file): - loader = importlib.machinery.SourceFileLoader("nodefunctions", custom_nodefunctions_file) - nodefuncs = loader.load_module() - for key, obj in inspect.getmembers(nodefuncs): - if inspect.ismodule(obj): - if obj.__file__.startswith(RESOURCE_PATH): - loader = importlib.machinery.SourceFileLoader(key, obj.__file__) - loader.load_module() + load_user_files(reload_nodefunctions=True) for nodenet_uid in nodenets: nodenets[nodenet_uid].reload_native_modules(filter_native_modules(nodenets[nodenet_uid].engine)) # restart previously active nodenets @@ -1348,6 +1369,8 @@ def reload_native_modules(): nodenets[uid].is_active = True return True +native_modules = {} +custom_recipes = {} load_definitions() init_worlds(world_data) From 8b219712fa5b484172e411ef770fa439cfc4de3c Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 12:10:59 +0100 Subject: [PATCH 024/306] Adapting timeseries world adapter to configurable worlds changes --- micropsi_core/world/timeseries/timeseries.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 444b20fd..3ecff76f 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -90,13 +90,11 @@ def state(self): class TimeSeriesRunner(WorldAdapter): - supported_datasources = [] - supported_datatargets = [] def __init__(self, world, uid=None, **data): super().__init__(world, uid, **data) for idx, ID in enumerate(self.world.ids): - self.supported_datasources.append(str(ID)) + self.datasources[str(ID)] = 0 def update_data_sources_and_targets(self): state = self.world.state From ca65a1e0a4e0fff7ddd3076f562a527163ab5117 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 12:11:25 +0100 Subject: [PATCH 025/306] Proof-of-concept changes for sensors --- .../nodenet/theano_engine/theano_node.py | 20 ++--- .../nodenet/theano_engine/theano_nodenet.py | 82 +++++++++---------- .../nodenet/theano_engine/theano_partition.py | 6 ++ .../theano_engine/theano_stepoperators.py | 9 +- micropsi_core/world/worldadapter.py | 4 + 5 files changed, 66 insertions(+), 55 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index f0bcccfd..9ab48a54 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -244,15 +244,12 @@ def set_parameter(self, parameter, value): else: value = None if self.type == "Sensor" and parameter == "datasource": - if self.uid in self._nodenet.inverted_sensor_map: - olddatasource = self._nodenet.inverted_sensor_map[self.uid] # first, clear old data source association - if self.uid in self._nodenet.sensormap.get(olddatasource, []): - self._nodenet.sensormap.get(olddatasource, []).remove(self.uid) - - connectedsensors = self._nodenet.sensormap.get(value, []) # then, set the new one - connectedsensors.append(self.uid) - self._nodenet.sensormap[value] = connectedsensors - self._nodenet.inverted_sensor_map[self.uid] = value + if value is not None and value != "": + sensor_element = self._partition.allocated_node_offsets[self._id] + GEN + old_datasource_index = np.where(self._partition.sensor_indices == sensor_element)[0] + self._partition.sensor_indices[old_datasource_index] = 0 + datasource_index = self._nodenet.worldadapter_instance.get_available_datasources().index(value) + self._partition.sensor_indices[datasource_index] = sensor_element elif self.type == "Actor" and parameter == "datatarget": if self.uid in self._nodenet.inverted_actuator_map: olddatatarget = self._nodenet.inverted_actuator_map[self.uid] # first, clear old data target association @@ -291,7 +288,10 @@ def clear_parameter(self, parameter): def clone_parameters(self): parameters = {} if self.type == "Sensor": - parameters['datasource'] = self._nodenet.inverted_sensor_map.get(self.uid, None) + #parameters['datasource'] = self._nodenet.inverted_sensor_map.get(self.uid, None) + sensor_element = self._partition.allocated_node_offsets[self._id] + GEN + datasource_index = np.where(self._partition.sensor_indices == sensor_element)[0] + parameters['datasource'] = self._nodenet.worldadapter_instance.get_available_datasources()[datasource_index] elif self.type == "Actor": parameters['datatarget'] = self._nodenet.inverted_actuator_map.get(self.uid, None) elif self.type == "Activator": diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 50a13b15..79d3fd41 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -179,10 +179,10 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No self.names = {} # map of data sources to string node IDs - self.sensormap = {} + #self.sensormap = {} # map of string node IDs to data sources - self.inverted_sensor_map = {} + #self.inverted_sensor_map = {} # map of data targets to string node IDs self.actuatormap = {} @@ -309,7 +309,7 @@ def save(self, filename): metadata['positions'] = self.positions metadata['names'] = self.names metadata['actuatormap'] = self.actuatormap - metadata['sensormap'] = self.sensormap + #metadata['sensormap'] = self.sensormap metadata['nodes'] = self.construct_native_modules_and_comments_dict() metadata['monitors'] = self.construct_monitors_dict() metadata['modulators'] = self.construct_modulators_dict() @@ -359,9 +359,9 @@ def load(self, filename): # was saved). self.reload_native_modules(self.native_module_definitions) - for sensor, id_list in self.sensormap.items(): - for id in id_list: - self.inverted_sensor_map[id] = sensor + #for sensor, id_list in self.sensormap.items(): + # for id in id_list: + # self.inverted_sensor_map[id] = sensor for actuator, id_list in self.actuatormap.items(): for id in id_list: self.inverted_actuator_map[id] = actuator @@ -398,8 +398,8 @@ def initialize_nodenet(self, initfrom): self.positions[key] = (self.positions[key] + [0] * 3)[:3] if 'actuatormap' in initfrom: self.actuatormap = initfrom['actuatormap'] - if 'sensormap' in initfrom: - self.sensormap = initfrom['sensormap'] + #if 'sensormap' in initfrom: + # self.sensormap = initfrom['sensormap'] if 'current_step' in initfrom: self._step = initfrom['current_step'] @@ -628,11 +628,9 @@ def create_node(self, nodetype, nodespace_uid, position, name=None, uid=None, pa if nodetype == "Sensor": if 'datasource' in parameters: datasource = parameters['datasource'] - if datasource is not None: - connectedsensors = self.sensormap.get(datasource, []) - connectedsensors.append(uid) - self.sensormap[datasource] = connectedsensors - self.inverted_sensor_map[uid] = datasource + if datasource is not None and datasource != "": + datasource_index = self.worldadapter_instance.get_available_datasources().index(datasource) + partition.sensor_indices[datasource_index] = partition.allocated_node_offsets[id] + GEN elif nodetype == "Actor": if 'datatarget' in parameters: datatarget = parameters['datatarget'] @@ -723,13 +721,13 @@ def delete_node(self, uid): partition.delete_node(node_id) # remove sensor association if there should be one - if uid in self.inverted_sensor_map: - sensor = self.inverted_sensor_map[uid] - del self.inverted_sensor_map[uid] - if sensor in self.sensormap: - self.sensormap[sensor].remove(uid) - if len(self.sensormap[sensor]) == 0: - del self.sensormap[sensor] + #if uid in self.inverted_sensor_map: + # sensor = self.inverted_sensor_map[uid] + # del self.inverted_sensor_map[uid] + # if sensor in self.sensormap: + # self.sensormap[sensor].remove(uid) + # if len(self.sensormap[sensor]) == 0: + # del self.sensormap[sensor] # remove actuator association if there should be one if uid in self.inverted_actuator_map: @@ -953,14 +951,14 @@ def clear_supplements(self, uid): def get_sensors(self, nodespace=None, datasource=None): sensors = {} sensorlist = [] - if datasource is None: - for ds_sensors in self.sensormap.values(): - sensorlist.extend(ds_sensors) - elif datasource in self.sensormap: - sensorlist = self.sensormap[datasource] - for uid in sensorlist: - if nodespace is None or self.get_partition(uid).allocated_node_parents[node_from_id(uid)] == nodespace_from_id(nodespace): - sensors[uid] = self.get_node(uid) + #if datasource is None: + # for ds_sensors in self.sensormap.values(): + # sensorlist.extend(ds_sensors) + #elif datasource in self.sensormap: + # sensorlist = self.sensormap[datasource] + #for uid in sensorlist: + # if nodespace is None or self.get_partition(uid).allocated_node_parents[node_from_id(uid)] == nodespace_from_id(nodespace): + # sensors[uid] = self.get_node(uid) return sensors def get_actors(self, nodespace=None, datatarget=None): @@ -1302,7 +1300,7 @@ def get_standard_nodetype_definitions(self): """ return copy.deepcopy(STANDARD_NODETYPES) - def set_sensors_and_actuator_feedback_to_values(self, datasource_to_value_map, datatarget_to_value_map): + def set_sensors_and_actuator_feedback_to_values(self, sensor_values, actuator_feedback_values): """ Sets the sensors for the given data sources to the given values """ @@ -1310,21 +1308,23 @@ def set_sensors_and_actuator_feedback_to_values(self, datasource_to_value_map, d for partition in self.partitions.values(): a_array = partition.a.get_value(borrow=True) - for datasource in datasource_to_value_map: - value = datasource_to_value_map.get(datasource) - sensor_uids = self.sensormap.get(datasource, []) + a_array[partition.sensor_indices] = sensor_values - for sensor_uid in sensor_uids: - if self.get_partition(sensor_uid).pid == partition.pid: - a_array[partition.allocated_node_offsets[node_from_id(sensor_uid)] + GEN] = value + #for datasource in datasource_to_value_map: + # value = datasource_to_value_map.get(datasource) + # sensor_uids = self.sensormap.get(datasource, []) - for datatarget in datatarget_to_value_map: - value = datatarget_to_value_map.get(datatarget) - actuator_uids = self.actuatormap.get(datatarget, []) + # for sensor_uid in sensor_uids: + # if self.get_partition(sensor_uid).pid == partition.pid: + # a_array[partition.allocated_node_offsets[node_from_id(sensor_uid)] + GEN] = value - for actuator_uid in actuator_uids: - if self.get_partition(actuator_uid).pid == partition.pid: - a_array[partition.allocated_node_offsets[node_from_id(actuator_uid)] + GEN] = value + #for datatarget in datatarget_to_value_map: + # value = datatarget_to_value_map.get(datatarget) + # actuator_uids = self.actuatormap.get(datatarget, []) + + # for actuator_uid in actuator_uids: + # if self.get_partition(actuator_uid).pid == partition.pid: + # a_array[partition.allocated_node_offsets[node_from_id(actuator_uid)] + GEN] = value partition.a.set_value(a_array, borrow=True) diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index 911a0088..d18aef82 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -237,6 +237,12 @@ def __init__(self, nodenet, pid, sparse=True, initial_number_of_nodes=2000, aver self.allocated_elements_to_activators = np.zeros(self.NoE, dtype=np.int32) + # this should really be len(self.nodenet.worldadapter_instance.get_available_datasources()) + # instead of self.NoE (which is the most pessimistic assumption: all elements are sensors) + # worldadapter instances are very unreliable buggers however, and you never know when they're there + # TODO: fix this to len(available_datasources) + self.sensor_indices = np.zeros(self.NoE, dtype=np.int32) + self.inlinks = {} self.deleted_items = {} diff --git a/micropsi_core/nodenet/theano_engine/theano_stepoperators.py b/micropsi_core/nodenet/theano_engine/theano_stepoperators.py index 36078c50..6919a40a 100644 --- a/micropsi_core/nodenet/theano_engine/theano_stepoperators.py +++ b/micropsi_core/nodenet/theano_engine/theano_stepoperators.py @@ -44,15 +44,16 @@ def read_sensors_and_actuator_feedback(self): if self.worldadapter is None: return - datasource_to_value_map = {} - for datasource in self.worldadapter.get_available_datasources(): - datasource_to_value_map[datasource] = self.worldadapter.get_datasource(datasource) + sensor_values = self.worldadapter.get_datasources() + #datasource_to_value_map = {} + #for datasource in self.worldadapter.get_available_datasources(): + # datasource_to_value_map[datasource] = self.worldadapter.get_datasource(datasource) datatarget_to_value_map = {} for datatarget in self.worldadapter.get_available_datatargets(): datatarget_to_value_map[datatarget] = self.worldadapter.get_datatarget_feedback(datatarget) - self.nodenet.set_sensors_and_actuator_feedback_to_values(datasource_to_value_map, datatarget_to_value_map) + self.nodenet.set_sensors_and_actuator_feedback_to_values(sensor_values, []) def write_actuators(self): if self.worldadapter is None: diff --git a/micropsi_core/world/worldadapter.py b/micropsi_core/world/worldadapter.py index 29a48a7c..f9c917ea 100644 --- a/micropsi_core/world/worldadapter.py +++ b/micropsi_core/world/worldadapter.py @@ -66,6 +66,10 @@ def get_datasource(self, key): """allows the agent to read a value from a datasource""" return self.datasource_snapshots.get(key) + def get_datasources(self): + """allows the agent to read all datasource values""" + return [float(x) for x in self.datasource_snapshots.values()] + def add_to_datatarget(self, key, value): """allows the agent to write a value to a datatarget""" if key in self.datatargets: From 576c367adab691429959c17bd23e07eed52197dd Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 13:05:24 +0100 Subject: [PATCH 026/306] Printing a warning when reassigning data sources --- .../nodenet/theano_engine/theano_node.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index 9ab48a54..c2c8feaf 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -247,8 +247,18 @@ def set_parameter(self, parameter, value): if value is not None and value != "": sensor_element = self._partition.allocated_node_offsets[self._id] + GEN old_datasource_index = np.where(self._partition.sensor_indices == sensor_element)[0] + self._partition.sensor_indices[old_datasource_index] = 0 datasource_index = self._nodenet.worldadapter_instance.get_available_datasources().index(value) + + if self._partition.sensor_indices[datasource_index] != sensor_element and \ + self._partition.sensor_indices[datasource_index] > 0: + + other_sensor_element = self._partition.sensor_indices[datasource_index] + other_sensor_id = node_to_id(self._partition.allocated_elements_to_nodes[other_sensor_element], self._partition.pid) + + self.logger.warn("Datasource %s had already been assigned to sensor %s, which will now be unassigned." % (value, other_sensor_id)) + self._partition.sensor_indices[datasource_index] = sensor_element elif self.type == "Actor" and parameter == "datatarget": if self.uid in self._nodenet.inverted_actuator_map: @@ -288,10 +298,12 @@ def clear_parameter(self, parameter): def clone_parameters(self): parameters = {} if self.type == "Sensor": - #parameters['datasource'] = self._nodenet.inverted_sensor_map.get(self.uid, None) sensor_element = self._partition.allocated_node_offsets[self._id] + GEN datasource_index = np.where(self._partition.sensor_indices == sensor_element)[0] - parameters['datasource'] = self._nodenet.worldadapter_instance.get_available_datasources()[datasource_index] + if len(datasource_index) == 0: + parameters['datasource'] = None + else: + parameters['datasource'] = self._nodenet.worldadapter_instance.get_available_datasources()[datasource_index] elif self.type == "Actor": parameters['datatarget'] = self._nodenet.inverted_actuator_map.get(self.uid, None) elif self.type == "Activator": From 9c6eccc05f8cceb5ff502895419a3cff18c967c6 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 17:47:13 +0100 Subject: [PATCH 027/306] Init with proper size --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 10 ++++++++++ .../nodenet/theano_engine/theano_partition.py | 6 +----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 79d3fd41..281a18dd 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -166,6 +166,16 @@ class TheanoNodenet(Nodenet): def engine(self): return "theano_engine" + @property + def worldadapter_instance(self): + return self._worldadapter_instance + + @worldadapter_instance.setter + def worldadapter_instance(self, _worldadapter_instance): + self._worldadapter_instance = _worldadapter_instance + for partition in self.partitions.values(): + partition.sensor_indices = np.zeros(len(_worldadapter_instance.get_available_datasources()), np.int32) + @property def current_step(self): return self._step diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index d18aef82..32270ea4 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -237,11 +237,7 @@ def __init__(self, nodenet, pid, sparse=True, initial_number_of_nodes=2000, aver self.allocated_elements_to_activators = np.zeros(self.NoE, dtype=np.int32) - # this should really be len(self.nodenet.worldadapter_instance.get_available_datasources()) - # instead of self.NoE (which is the most pessimistic assumption: all elements are sensors) - # worldadapter instances are very unreliable buggers however, and you never know when they're there - # TODO: fix this to len(available_datasources) - self.sensor_indices = np.zeros(self.NoE, dtype=np.int32) + self.sensor_indices = np.zeros(0, dtype=np.int32) self.inlinks = {} From 161f990cc74b8f1696337577b12184720fd862fb Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 17:47:21 +0100 Subject: [PATCH 028/306] Unassign on delete --- micropsi_core/nodenet/theano_engine/theano_partition.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index 32270ea4..b827b440 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -1431,6 +1431,10 @@ def delete_node(self, node_id): self.g_function_selector.set_value(g_function_selector_array, borrow=True) self.allocated_elements_to_nodes[np.where(self.allocated_elements_to_nodes == node_id)[0]] = 0 + if type == SENSOR: + sensor_index = np.where(self.sensor_indices == node_id)[0] + self.sensor_indices[sensor_index] = 0 + if type == PIPE: n_function_selector_array = self.n_function_selector.get_value(borrow=True) n_function_selector_array[offset + GEN] = NFPG_PIPE_NON From af660220301c0dff456bc8704fc277de9b399ef8 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 18:18:40 +0100 Subject: [PATCH 029/306] Persistency for sensor assignments --- .../nodenet/theano_engine/theano_node.py | 1 + .../nodenet/theano_engine/theano_nodenet.py | 54 ++++++++++--------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index c2c8feaf..60f656d7 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -259,6 +259,7 @@ def set_parameter(self, parameter, value): self.logger.warn("Datasource %s had already been assigned to sensor %s, which will now be unassigned." % (value, other_sensor_id)) + self._nodenet.sensormap[value] = self.uid self._partition.sensor_indices[datasource_index] = sensor_element elif self.type == "Actor" and parameter == "datatarget": if self.uid in self._nodenet.inverted_actuator_map: diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 281a18dd..2cf41039 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -176,6 +176,21 @@ def worldadapter_instance(self, _worldadapter_instance): for partition in self.partitions.values(): partition.sensor_indices = np.zeros(len(_worldadapter_instance.get_available_datasources()), np.int32) + for datasource, node_id in self.sensormap.items(): + if not isinstance(node_id, str): + node_id = node_id[0] + if self.get_partition(node_id) == partition: + sensor_element = partition.allocated_node_offsets[node_from_id(node_id)] + GEN + datasource_index = _worldadapter_instance.get_available_datasources().index(datasource) + + if partition.sensor_indices[datasource_index] != sensor_element and partition.sensor_indices[datasource_index] > 0: + other_sensor_element = partition.sensor_indices[datasource_index] + other_sensor_id = node_to_id(partition.allocated_elements_to_nodes[other_sensor_element], partition.pid) + self.logger.warn("Datasource %s had already been assigned to sensor %s, which will now be unassigned." % (datasource, other_sensor_id)) + + partition.sensor_indices[datasource_index] = sensor_element + + @property def current_step(self): return self._step @@ -189,10 +204,7 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No self.names = {} # map of data sources to string node IDs - #self.sensormap = {} - - # map of string node IDs to data sources - #self.inverted_sensor_map = {} + self.sensormap = {} # map of data targets to string node IDs self.actuatormap = {} @@ -319,7 +331,7 @@ def save(self, filename): metadata['positions'] = self.positions metadata['names'] = self.names metadata['actuatormap'] = self.actuatormap - #metadata['sensormap'] = self.sensormap + metadata['sensormap'] = self.sensormap metadata['nodes'] = self.construct_native_modules_and_comments_dict() metadata['monitors'] = self.construct_monitors_dict() metadata['modulators'] = self.construct_modulators_dict() @@ -369,9 +381,6 @@ def load(self, filename): # was saved). self.reload_native_modules(self.native_module_definitions) - #for sensor, id_list in self.sensormap.items(): - # for id in id_list: - # self.inverted_sensor_map[id] = sensor for actuator, id_list in self.actuatormap.items(): for id in id_list: self.inverted_actuator_map[id] = actuator @@ -408,8 +417,8 @@ def initialize_nodenet(self, initfrom): self.positions[key] = (self.positions[key] + [0] * 3)[:3] if 'actuatormap' in initfrom: self.actuatormap = initfrom['actuatormap'] - #if 'sensormap' in initfrom: - # self.sensormap = initfrom['sensormap'] + if 'sensormap' in initfrom: + self.sensormap = initfrom['sensormap'] if 'current_step' in initfrom: self._step = initfrom['current_step'] @@ -731,13 +740,8 @@ def delete_node(self, uid): partition.delete_node(node_id) # remove sensor association if there should be one - #if uid in self.inverted_sensor_map: - # sensor = self.inverted_sensor_map[uid] - # del self.inverted_sensor_map[uid] - # if sensor in self.sensormap: - # self.sensormap[sensor].remove(uid) - # if len(self.sensormap[sensor]) == 0: - # del self.sensormap[sensor] + if uid in self.sensor_map.values(): + self.sensormap = { k:v for k, v in self.sensormap.items() if v is not uid } # remove actuator association if there should be one if uid in self.inverted_actuator_map: @@ -961,14 +965,14 @@ def clear_supplements(self, uid): def get_sensors(self, nodespace=None, datasource=None): sensors = {} sensorlist = [] - #if datasource is None: - # for ds_sensors in self.sensormap.values(): - # sensorlist.extend(ds_sensors) - #elif datasource in self.sensormap: - # sensorlist = self.sensormap[datasource] - #for uid in sensorlist: - # if nodespace is None or self.get_partition(uid).allocated_node_parents[node_from_id(uid)] == nodespace_from_id(nodespace): - # sensors[uid] = self.get_node(uid) + if datasource is None: + for ds_sensors in self.sensormap.values(): + sensorlist.extend(ds_sensors) + elif datasource in self.sensormap: + sensorlist = self.sensormap[datasource] + for uid in sensorlist: + if nodespace is None or self.get_partition(uid).allocated_node_parents[node_from_id(uid)] == nodespace_from_id(nodespace): + sensors[uid] = self.get_node(uid) return sensors def get_actors(self, nodespace=None, datatarget=None): From 2b131162916c6f5ff4d90c66c35ed8709e6d210c Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 18:26:48 +0100 Subject: [PATCH 030/306] Removing obsolete snapshotting --- .../nodenet/dict_engine/dict_nodenet.py | 3 --- micropsi_core/nodenet/nodefunctions.py | 4 ++-- .../nodenet/theano_engine/theano_nodenet.py | 3 --- .../theano_engine/theano_stepoperators.py | 13 +++-------- micropsi_core/world/worldadapter.py | 22 ++++++++----------- 5 files changed, 14 insertions(+), 31 deletions(-) diff --git a/micropsi_core/nodenet/dict_engine/dict_nodenet.py b/micropsi_core/nodenet/dict_engine/dict_nodenet.py index 91f93de7..08149973 100644 --- a/micropsi_core/nodenet/dict_engine/dict_nodenet.py +++ b/micropsi_core/nodenet/dict_engine/dict_nodenet.py @@ -471,9 +471,6 @@ def merge_data(self, nodenet_data, keep_uids=False): def step(self): """perform a simulation step""" - if self.worldadapter_instance: - self.worldadapter_instance.snapshot() - with self.netlock: self._step += 1 diff --git a/micropsi_core/nodenet/nodefunctions.py b/micropsi_core/nodenet/nodefunctions.py index 66a0b6ed..b18245c1 100644 --- a/micropsi_core/nodenet/nodefunctions.py +++ b/micropsi_core/nodenet/nodefunctions.py @@ -20,7 +20,7 @@ def register(netapi, node=None, **params): def sensor(netapi, node=None, datasource=None, **params): if datasource in netapi.worldadapter.get_available_datasources(): - datasource_value = netapi.worldadapter.get_datasource(datasource) + datasource_value = netapi.worldadapter.get_datasource_value(datasource) else: datasource_value = netapi.get_modulator(datasource) node.activation = datasource_value @@ -33,7 +33,7 @@ def actor(netapi, node=None, datatarget=None, **params): activation_to_set = node.get_slot("gen").activation if datatarget in netapi.worldadapter.get_available_datatargets(): netapi.worldadapter.add_to_datatarget(datatarget, activation_to_set) - feedback = netapi.worldadapter.get_datatarget_feedback(datatarget) + feedback = netapi.worldadapter.get_datatarget_feedback_value(datatarget) else: netapi.set_modulator(datatarget, activation_to_set) feedback = 1 diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 2cf41039..4adf5274 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -557,9 +557,6 @@ def merge_nodespace_data(self, nodespace_uid, data, uidmap, keep_uids=False): uidmap[nodespace_uid] = newuid def step(self): - if self.worldadapter_instance is not None: - self.worldadapter_instance.snapshot() - with self.netlock: self._step += 1 diff --git a/micropsi_core/nodenet/theano_engine/theano_stepoperators.py b/micropsi_core/nodenet/theano_engine/theano_stepoperators.py index 6919a40a..69cba170 100644 --- a/micropsi_core/nodenet/theano_engine/theano_stepoperators.py +++ b/micropsi_core/nodenet/theano_engine/theano_stepoperators.py @@ -44,16 +44,9 @@ def read_sensors_and_actuator_feedback(self): if self.worldadapter is None: return - sensor_values = self.worldadapter.get_datasources() - #datasource_to_value_map = {} - #for datasource in self.worldadapter.get_available_datasources(): - # datasource_to_value_map[datasource] = self.worldadapter.get_datasource(datasource) - - datatarget_to_value_map = {} - for datatarget in self.worldadapter.get_available_datatargets(): - datatarget_to_value_map[datatarget] = self.worldadapter.get_datatarget_feedback(datatarget) - - self.nodenet.set_sensors_and_actuator_feedback_to_values(sensor_values, []) + self.nodenet.set_sensors_and_actuator_feedback_to_values( + self.worldadapter.get_datasource_values(), + self.worldadapter.get_datatarget_feedback_values()) def write_actuators(self): if self.worldadapter is None: diff --git a/micropsi_core/world/worldadapter.py b/micropsi_core/world/worldadapter.py index f9c917ea..7b011280 100644 --- a/micropsi_core/world/worldadapter.py +++ b/micropsi_core/world/worldadapter.py @@ -35,9 +35,7 @@ def __init__(self, world, uid=None, **data): self.datatargets = {} self.datatarget_feedback = {} self.datasource_lock = Lock() - self.datasource_snapshots = {} WorldObject.__init__(self, world, category='agents', uid=uid, **data) - self.snapshot() def initialize_worldobject(self, data): for key in self.datasources: @@ -48,12 +46,6 @@ def initialize_worldobject(self, data): self.datatargets[key] = data['datatargets'][key] self.datatarget_feedback[key] = 0 - # agent facing methods: - def snapshot(self): - """called by the agent every netstep to create a consistent set of sensory input""" - with self.datasource_lock: - self.datasource_snapshots = self.datasources.copy() - def get_available_datasources(self): """returns a list of identifiers of the datasources available for this world adapter""" return list(self.datasources.keys()) @@ -62,23 +54,27 @@ def get_available_datatargets(self): """returns a list of identifiers of the datatargets available for this world adapter""" return list(self.datatargets.keys()) - def get_datasource(self, key): + def get_datasource_value(self, key): """allows the agent to read a value from a datasource""" - return self.datasource_snapshots.get(key) + return self.datasources.get(key) - def get_datasources(self): + def get_datasource_values(self): """allows the agent to read all datasource values""" - return [float(x) for x in self.datasource_snapshots.values()] + return [float(x) for x in self.datasources.values()] def add_to_datatarget(self, key, value): """allows the agent to write a value to a datatarget""" if key in self.datatargets: self.datatargets[key] += value - def get_datatarget_feedback(self, key): + def get_datatarget_feedback_value(self, key): """get feedback whether the actor-induced action succeeded""" return self.datatarget_feedback.get(key, 0) + def get_datatarget_feedback_values(self): + """allows the agent to read all datasource values""" + return [float(x) for x in self.datatarget_feedback.values()] + def set_datatarget_feedback(self, key, value): """set feedback for the given datatarget""" self.datatarget_feedback[key] = value From 10e4dd86cc8cabfbd3a678d1a14e626ba31ed2ea Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 18:52:51 +0100 Subject: [PATCH 031/306] Introducing the BlippingWorldAdapter --- micropsi_core/world/worldadapter.py | 71 ++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/micropsi_core/world/worldadapter.py b/micropsi_core/world/worldadapter.py index 7b011280..922b9276 100644 --- a/micropsi_core/world/worldadapter.py +++ b/micropsi_core/world/worldadapter.py @@ -99,7 +99,9 @@ def is_alive(self): class Default(WorldAdapter): - + """ + The default world adapter + """ def __init__(self, world, uid=None, **data): self.datasources = dict((s, 0) for s in ['static_on', 'random', 'static_off']) self.datatargets = {'echo': 0} @@ -110,3 +112,70 @@ def update_data_sources_and_targets(self): self.datatarget_feedback['echo'] = self.datatargets['echo'] self.datasources['static_on'] = 1 self.datasources['random'] = random.uniform(0, 1) + + +class BlippingWorldAdapter(WorldAdapter): + """ + The BlippingWorldAdapter base class allows to avoid python dictionaries and loops for transmitting values + to nodenet engines. + Engines that bulk-query values, such as the theano_engine, will be faster. + Numpy arrays can be passed directly into the engine. + """ + def __init__(self, world, uid=None, **data): + self.datasource_values = [] + self.datatarget_values = [] + self.datatarget_feedback_values = [] + + def get_datasource_value(self, key): + """allows the agent to read a value from a datasource""" + index = self.get_available_datasources().index(key) + return self.datasource_values[index] + + def get_datasource_values(self): + """allows the agent to read all datasource values""" + return self.datasource_values + + def add_to_datatarget(self, key, value): + """allows the agent to write a value to a datatarget""" + index = self.get_available_datasources().index(key) + self.datatarget_values[index] += value + + def get_datatarget_feedback_value(self, key): + """get feedback whether the actor-induced action succeeded""" + index = self.get_available_datatargets().index(key) + return self.datatarget_feedback_values[index] + + def get_datatarget_feedback_values(self): + """allows the agent to read all datasource values""" + return self.datatarget_feedback_values + + def set_datatarget_feedback(self, key, value): + """set feedback for the given datatarget""" + index = self.get_available_datatargets().index(key) + self.datatarget_feedback_values[index] = value + + def get_available_datasources(self): + """ + must be implemented by the concrete world adapater and return a list of datasource name strings, + in the same order as values returned by get_datasource_values() + """ + pass + + def get_available_datatargets(self): + """ + must be implemented by the concrete world adapater and return a list of datatarget name strings, + in the same order as values returned by get_datatarget_feedback_values() + """ + pass + + def update_data_sources_and_targets(self): + """ + must be implemented by concrete world adapters to read and set the following arrays: + datasource_values + datatarget_values + datatarget_feedback_values + + Arrays sizes need to be equal to the corresponding dict objects. + The values of the dict objects will be bypassed and ignored. + """ + pass From 926465123f98320449c866dd7097267bf433c70e Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 20:20:02 +0100 Subject: [PATCH 032/306] Blipping for actuators --- .../nodenet/theano_engine/theano_node.py | 33 +++++--- .../nodenet/theano_engine/theano_nodenet.py | 81 ++++++------------- .../nodenet/theano_engine/theano_partition.py | 5 ++ 3 files changed, 54 insertions(+), 65 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index 60f656d7..2000fc43 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -262,15 +262,23 @@ def set_parameter(self, parameter, value): self._nodenet.sensormap[value] = self.uid self._partition.sensor_indices[datasource_index] = sensor_element elif self.type == "Actor" and parameter == "datatarget": - if self.uid in self._nodenet.inverted_actuator_map: - olddatatarget = self._nodenet.inverted_actuator_map[self.uid] # first, clear old data target association - if self.uid in self._nodenet.actuatormap.get(olddatatarget, []): - self._nodenet.actuatormap.get(olddatatarget, []).remove(self.uid) - - connectedactuators = self._nodenet.actuatormap.get(value, []) # then, set the new one - connectedactuators.append(self.uid) - self._nodenet.actuatormap[value] = connectedactuators - self._nodenet.inverted_actuator_map[self.uid] = value + if value is not None and value != "": + actuator_element = self._partition.allocated_node_offsets[self._id] + GEN + old_datatarget_index = np.where(self._partition.actuator_indices == actuator_element)[0] + + self._partition.actuator_indices[old_datatarget_index] = 0 + datatarget_index = self._nodenet.worldadapter_instance.get_available_datatargets().index(value) + + if self._partition.actuator_indices[datatarget_index] != actuator_element and \ + self._partition.actuator_indices[datatarget_index] > 0: + + other_actuator_element = self._partition.actuator_indices[datatarget_index] + other_actuator_id = node_to_id(self._partition.allocated_elements_to_nodes[other_actuator_element], self._partition.pid) + + self.logger.warn("Datatarget %s had already been assigned to actuator %s, which will now be unassigned." % (value, other_actuator_id)) + + self._nodenet.actuatormap[value] = self.uid + self._partition.actuator_indices[datatarget_index] = actuator_element elif self.type == "Activator" and parameter == "type": if value != "sampling": self._nodenet.set_nodespace_gatetype_activator(self.parent_nodespace, value, self.uid) @@ -306,7 +314,12 @@ def clone_parameters(self): else: parameters['datasource'] = self._nodenet.worldadapter_instance.get_available_datasources()[datasource_index] elif self.type == "Actor": - parameters['datatarget'] = self._nodenet.inverted_actuator_map.get(self.uid, None) + actuator_element = self._partition.allocated_node_offsets[self._id] + GEN + datatarget_index = np.where(self._partition.actuator_indices == actuator_element)[0] + if len(datatarget_index) == 0: + parameters['datatarget'] = None + else: + parameters['datatarget'] = self._nodenet.worldadapter_instance.get_available_datatargets()[datatarget_index] elif self.type == "Activator": activator_type = None if self._id in self._partition.allocated_nodespaces_por_activators: diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 4adf5274..35d67fe2 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -174,8 +174,23 @@ def worldadapter_instance(self): def worldadapter_instance(self, _worldadapter_instance): self._worldadapter_instance = _worldadapter_instance for partition in self.partitions.values(): + partition.actuator_indices = np.zeros(len(_worldadapter_instance.get_available_datatargets()), np.int32) partition.sensor_indices = np.zeros(len(_worldadapter_instance.get_available_datasources()), np.int32) + for datatarget, node_id in self.actuatormap.items(): + if not isinstance(node_id, str): + node_id = node_id[0] + if self.get_partition(node_id) == partition: + actuator_element = partition.allocated_node_offsets[node_from_id(node_id)] + GEN + datatarget_index = _worldadapter_instance.get_available_datatargets().index(datatarget) + + if partition.sensor_indices[datatarget_index] != sensor_element and partition.sensor_indices[datatarget_index] > 0: + other_actuator_element = partition.actuator_indices[datatarget_index] + other_actuator_id = node_to_id(partition.allocated_elements_to_nodes[other_actuator_element], partition.pid) + self.logger.warn("Datatarget %s had already been assigned to actuator %s, which will now be unassigned." % (datatarget, other_actuator_id)) + + partition.actuator_indices[datatarget_index] = actuator_element + for datasource, node_id in self.sensormap.items(): if not isinstance(node_id, str): node_id = node_id[0] @@ -190,7 +205,6 @@ def worldadapter_instance(self, _worldadapter_instance): partition.sensor_indices[datasource_index] = sensor_element - @property def current_step(self): return self._step @@ -209,9 +223,6 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No # map of data targets to string node IDs self.actuatormap = {} - # map of string node IDs to data targets - self.inverted_actuator_map = {} - super(TheanoNodenet, self).__init__(name, worldadapter, world, owner, uid) precision = settings['theano']['precision'] @@ -643,18 +654,10 @@ def create_node(self, nodetype, nodespace_uid, position, name=None, uid=None, pa if nodetype == "Sensor": if 'datasource' in parameters: - datasource = parameters['datasource'] - if datasource is not None and datasource != "": - datasource_index = self.worldadapter_instance.get_available_datasources().index(datasource) - partition.sensor_indices[datasource_index] = partition.allocated_node_offsets[id] + GEN + self.get_node(uid).set_parameter("datasource", parameters['datasource']) elif nodetype == "Actor": if 'datatarget' in parameters: - datatarget = parameters['datatarget'] - if datatarget is not None: - connectedactuators = self.actuatormap.get(datatarget, []) - connectedactuators.append(uid) - self.actuatormap[datatarget] = connectedactuators - self.inverted_actuator_map[uid] = datatarget + self.get_node(uid).set_parameter("datatarget", parameters['datatarget']) return uid @@ -738,16 +741,11 @@ def delete_node(self, uid): # remove sensor association if there should be one if uid in self.sensor_map.values(): - self.sensormap = { k:v for k, v in self.sensormap.items() if v is not uid } + self.sensormap = {k:v for k, v in self.sensormap.items() if v is not uid } # remove actuator association if there should be one - if uid in self.inverted_actuator_map: - actuator = self.inverted_actuator_map[uid] - del self.inverted_actuator_map[uid] - if actuator in self.actuatormap: - self.actuatormap[actuator].remove(uid) - if len(self.actuatormap[actuator]) == 0: - del self.actuatormap[actuator] + if uid in self.actuator_map.values(): + self.actuatormap = {k:v for k, v in self.actuatormap.items() if v is not uid } self.clear_supplements(uid) @@ -1315,51 +1313,24 @@ def set_sensors_and_actuator_feedback_to_values(self, sensor_values, actuator_fe """ Sets the sensors for the given data sources to the given values """ - for partition in self.partitions.values(): a_array = partition.a.get_value(borrow=True) - a_array[partition.sensor_indices] = sensor_values - - #for datasource in datasource_to_value_map: - # value = datasource_to_value_map.get(datasource) - # sensor_uids = self.sensormap.get(datasource, []) - - # for sensor_uid in sensor_uids: - # if self.get_partition(sensor_uid).pid == partition.pid: - # a_array[partition.allocated_node_offsets[node_from_id(sensor_uid)] + GEN] = value - - #for datatarget in datatarget_to_value_map: - # value = datatarget_to_value_map.get(datatarget) - # actuator_uids = self.actuatormap.get(datatarget, []) - - # for actuator_uid in actuator_uids: - # if self.get_partition(actuator_uid).pid == partition.pid: - # a_array[partition.allocated_node_offsets[node_from_id(actuator_uid)] + GEN] = value - + a_array[partition.actuator_indices] = actuator_feedback_values partition.a.set_value(a_array, borrow=True) def read_actuators(self): """ - Returns a map of datatargets to values for writing back to the world adapter + Returns a list of datatarget values for writing back to the world adapter """ + if len(self.actuatormap) == 0: + return [] - actuator_values_to_write = {} + actuator_values_to_write = np.zeros(len(self.actuatormap), np.int32) for partition in self.partitions.values(): a_array = partition.a.get_value(borrow=True) - - for datatarget in self.actuatormap: - if datatarget not in actuator_values_to_write: - actuator_values_to_write[datatarget] = 0 - actuator_node_activations = 0 - for actuator_uid in self.actuatormap[datatarget]: - if self.get_partition(actuator_uid).pid == partition.pid: - actuator_node_activations += a_array[partition.allocated_node_offsets[node_from_id(actuator_uid)] + GEN] - - actuator_values_to_write[datatarget] += actuator_node_activations - - partition.a.set_value(a_array, borrow=True) + actuator_values_to_write += a_array[partition.actuator_indices] return actuator_values_to_write diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index b827b440..91331995 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -238,6 +238,7 @@ def __init__(self, nodenet, pid, sparse=True, initial_number_of_nodes=2000, aver self.allocated_elements_to_activators = np.zeros(self.NoE, dtype=np.int32) self.sensor_indices = np.zeros(0, dtype=np.int32) + self.actuator_indices = np.zeros(0, dtype=np.int32) self.inlinks = {} @@ -1435,6 +1436,10 @@ def delete_node(self, node_id): sensor_index = np.where(self.sensor_indices == node_id)[0] self.sensor_indices[sensor_index] = 0 + if type == ACTUATOR: + actuator_index = np.where(self.actuator_indices == node_id)[0] + self.actuator_indices[actuator_index] = 0 + if type == PIPE: n_function_selector_array = self.n_function_selector.get_value(borrow=True) n_function_selector_array[offset + GEN] = NFPG_PIPE_NON From cc12eed49a5f1719fde3716ae4bd7b5b4b3530f8 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 20:23:34 +0100 Subject: [PATCH 033/306] unsetting sensor/actuator assignments when unsetting world adapter --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 35d67fe2..6176964f 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -173,7 +173,13 @@ def worldadapter_instance(self): @worldadapter_instance.setter def worldadapter_instance(self, _worldadapter_instance): self._worldadapter_instance = _worldadapter_instance + for partition in self.partitions.values(): + if _worldadapter_instance is None: + partition.actuator_indices = np.zeros(0, np.int32) + partition.sensor_indices = np.zeros(0, np.int32) + continue + partition.actuator_indices = np.zeros(len(_worldadapter_instance.get_available_datatargets()), np.int32) partition.sensor_indices = np.zeros(len(_worldadapter_instance.get_available_datasources()), np.int32) From b41da442b34a06f8d40779bdd61582c5c9af7dad Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 20:37:04 +0100 Subject: [PATCH 034/306] Handling bad parameters --- micropsi_core/nodenet/theano_engine/theano_node.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index 2000fc43..e07d0179 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -249,6 +249,10 @@ def set_parameter(self, parameter, value): old_datasource_index = np.where(self._partition.sensor_indices == sensor_element)[0] self._partition.sensor_indices[old_datasource_index] = 0 + if value not in self._nodenet.worldadapter_instance.get_available_datasources(): + self.logger.warn("Datasource %s not known in world adapter %s, will not be assigned." % (value, self._nodenet.worldadapter)) + return + datasource_index = self._nodenet.worldadapter_instance.get_available_datasources().index(value) if self._partition.sensor_indices[datasource_index] != sensor_element and \ @@ -267,6 +271,10 @@ def set_parameter(self, parameter, value): old_datatarget_index = np.where(self._partition.actuator_indices == actuator_element)[0] self._partition.actuator_indices[old_datatarget_index] = 0 + if value not in self._nodenet.worldadapter_instance.get_available_datatargets(): + self.logger.warn("Datatarget %s not known in world adapter %s, whill not be assigned." % (value, self._nodenet.worldadapter)) + return + datatarget_index = self._nodenet.worldadapter_instance.get_available_datatargets().index(value) if self._partition.actuator_indices[datatarget_index] != actuator_element and \ From 4f918e040319af86c7873f36d64b85dbbb7169b3 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 20:37:20 +0100 Subject: [PATCH 035/306] Avoiding code duplication assignments --- .../nodenet/theano_engine/theano_nodenet.py | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 6176964f..af8b71ec 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -187,29 +187,13 @@ def worldadapter_instance(self, _worldadapter_instance): if not isinstance(node_id, str): node_id = node_id[0] if self.get_partition(node_id) == partition: - actuator_element = partition.allocated_node_offsets[node_from_id(node_id)] + GEN - datatarget_index = _worldadapter_instance.get_available_datatargets().index(datatarget) - - if partition.sensor_indices[datatarget_index] != sensor_element and partition.sensor_indices[datatarget_index] > 0: - other_actuator_element = partition.actuator_indices[datatarget_index] - other_actuator_id = node_to_id(partition.allocated_elements_to_nodes[other_actuator_element], partition.pid) - self.logger.warn("Datatarget %s had already been assigned to actuator %s, which will now be unassigned." % (datatarget, other_actuator_id)) - - partition.actuator_indices[datatarget_index] = actuator_element + self.get_node(node_id).set_parameter("datatarget", datatarget) for datasource, node_id in self.sensormap.items(): if not isinstance(node_id, str): node_id = node_id[0] if self.get_partition(node_id) == partition: - sensor_element = partition.allocated_node_offsets[node_from_id(node_id)] + GEN - datasource_index = _worldadapter_instance.get_available_datasources().index(datasource) - - if partition.sensor_indices[datasource_index] != sensor_element and partition.sensor_indices[datasource_index] > 0: - other_sensor_element = partition.sensor_indices[datasource_index] - other_sensor_id = node_to_id(partition.allocated_elements_to_nodes[other_sensor_element], partition.pid) - self.logger.warn("Datasource %s had already been assigned to sensor %s, which will now be unassigned." % (datasource, other_sensor_id)) - - partition.sensor_indices[datasource_index] = sensor_element + self.get_node(node_id).set_parameter("datasource", datasource) @property def current_step(self): From 43078e2eb51074837cff1c947afcac981ddf8249 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 10 Feb 2016 20:43:55 +0100 Subject: [PATCH 036/306] sensor and actuator list querying --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index af8b71ec..85ef5c19 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -951,10 +951,9 @@ def get_sensors(self, nodespace=None, datasource=None): sensors = {} sensorlist = [] if datasource is None: - for ds_sensors in self.sensormap.values(): - sensorlist.extend(ds_sensors) + sensorlist = self.sensormap.values() elif datasource in self.sensormap: - sensorlist = self.sensormap[datasource] + sensorlist.append(self.sensormap[datasource]) for uid in sensorlist: if nodespace is None or self.get_partition(uid).allocated_node_parents[node_from_id(uid)] == nodespace_from_id(nodespace): sensors[uid] = self.get_node(uid) @@ -964,10 +963,9 @@ def get_actors(self, nodespace=None, datatarget=None): actuators = {} actuatorlist = [] if datatarget is None: - for dt_actuators in self.actuatormap.values(): - actuatorlist.extend(dt_actuators) + actuatorlist = self.actuatormap.values() elif datatarget in self.actuatormap: - actuatorlist = self.actuatormap[datatarget] + actuatorlist.append(self.actuatormap[datatarget]) for uid in actuatorlist: if nodespace is None or self.get_partition(uid).allocated_node_parents[node_from_id(uid)] == nodespace_from_id(nodespace): actuators[uid] = self.get_node(uid) From cb3bb039489e7b133eb1bb2f42fb64f189c5ddbe Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Thu, 11 Feb 2016 13:10:18 +0100 Subject: [PATCH 037/306] Fixing actuators --- .../nodenet/theano_engine/theano_node.py | 4 ++-- .../nodenet/theano_engine/theano_nodenet.py | 11 ++++------ .../theano_engine/theano_stepoperators.py | 4 +--- micropsi_core/world/worldadapter.py | 21 +++++++++++++++---- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index e07d0179..23d5f9b9 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -244,7 +244,7 @@ def set_parameter(self, parameter, value): else: value = None if self.type == "Sensor" and parameter == "datasource": - if value is not None and value != "": + if value is not None and value != "" and self._nodenet.worldadapter_instance is not None: sensor_element = self._partition.allocated_node_offsets[self._id] + GEN old_datasource_index = np.where(self._partition.sensor_indices == sensor_element)[0] @@ -266,7 +266,7 @@ def set_parameter(self, parameter, value): self._nodenet.sensormap[value] = self.uid self._partition.sensor_indices[datasource_index] = sensor_element elif self.type == "Actor" and parameter == "datatarget": - if value is not None and value != "": + if value is not None and value != "" and self._nodenet.worldadapter_instance is not None: actuator_element = self._partition.allocated_node_offsets[self._id] + GEN old_datatarget_index = np.where(self._partition.actuator_indices == actuator_element)[0] diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 85ef5c19..a7d684db 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -730,11 +730,11 @@ def delete_node(self, uid): partition.delete_node(node_id) # remove sensor association if there should be one - if uid in self.sensor_map.values(): + if uid in self.sensormap.values(): self.sensormap = {k:v for k, v in self.sensormap.items() if v is not uid } # remove actuator association if there should be one - if uid in self.actuator_map.values(): + if uid in self.actuatormap.values(): self.actuatormap = {k:v for k, v in self.actuatormap.items() if v is not uid } self.clear_supplements(uid) @@ -1311,14 +1311,11 @@ def read_actuators(self): """ Returns a list of datatarget values for writing back to the world adapter """ - if len(self.actuatormap) == 0: - return [] - - actuator_values_to_write = np.zeros(len(self.actuatormap), np.int32) + actuator_values_to_write = np.zeros_like(self.rootpartition.actuator_indices) for partition in self.partitions.values(): a_array = partition.a.get_value(borrow=True) - actuator_values_to_write += a_array[partition.actuator_indices] + actuator_values_to_write = actuator_values_to_write + a_array[partition.actuator_indices] return actuator_values_to_write diff --git a/micropsi_core/nodenet/theano_engine/theano_stepoperators.py b/micropsi_core/nodenet/theano_engine/theano_stepoperators.py index 69cba170..c031f78e 100644 --- a/micropsi_core/nodenet/theano_engine/theano_stepoperators.py +++ b/micropsi_core/nodenet/theano_engine/theano_stepoperators.py @@ -52,9 +52,7 @@ def write_actuators(self): if self.worldadapter is None: return - values_to_write = self.nodenet.read_actuators() - for datatarget in values_to_write: - self.worldadapter.add_to_datatarget(datatarget, values_to_write[datatarget]) + self.worldadapter.set_datatarget_values(self.nodenet.read_actuators()) def count_success_and_failure(self, nodenet): nays = 0 diff --git a/micropsi_core/world/worldadapter.py b/micropsi_core/world/worldadapter.py index 922b9276..350bab93 100644 --- a/micropsi_core/world/worldadapter.py +++ b/micropsi_core/world/worldadapter.py @@ -48,11 +48,11 @@ def initialize_worldobject(self, data): def get_available_datasources(self): """returns a list of identifiers of the datasources available for this world adapter""" - return list(self.datasources.keys()) + return sorted(list(self.datasources.keys())) def get_available_datatargets(self): """returns a list of identifiers of the datatargets available for this world adapter""" - return list(self.datatargets.keys()) + return sorted(list(self.datatargets.keys())) def get_datasource_value(self, key): """allows the agent to read a value from a datasource""" @@ -60,20 +60,25 @@ def get_datasource_value(self, key): def get_datasource_values(self): """allows the agent to read all datasource values""" - return [float(x) for x in self.datasources.values()] + return [float(self.datasources[x]) for x in self.get_available_datasources()] def add_to_datatarget(self, key, value): """allows the agent to write a value to a datatarget""" if key in self.datatargets: self.datatargets[key] += value + def set_datatarget_values(self, values): + """allows the agent to write a list of value to the datatargets""" + for i, key in enumerate(self.get_available_datatargets()): + self.datatargets[key] = values[i] + def get_datatarget_feedback_value(self, key): """get feedback whether the actor-induced action succeeded""" return self.datatarget_feedback.get(key, 0) def get_datatarget_feedback_values(self): """allows the agent to read all datasource values""" - return [float(x) for x in self.datatarget_feedback.values()] + return [float(self.datatarget_feedback[x]) for x in self.get_available_datatargets()] def set_datatarget_feedback(self, key, value): """set feedback for the given datatarget""" @@ -154,6 +159,14 @@ def set_datatarget_feedback(self, key, value): index = self.get_available_datatargets().index(key) self.datatarget_feedback_values[index] = value + def set_datatarget_values(self, values): + """allows the agent to write a list of value to the datatargets""" + self.datatarget_values = values + + def reset_datatargets(self): + """ resets (zeros) the datatargets """ + self.datatarget_values[:] = 0 + def get_available_datasources(self): """ must be implemented by the concrete world adapater and return a list of datasource name strings, From a74e699ee41c070cfb181b357c8586c4f448564a Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Thu, 11 Feb 2016 13:24:36 +0100 Subject: [PATCH 038/306] Removing test that explicitly excluded the new behavior --- .../tests/test_runtime_world_basics.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/micropsi_core/tests/test_runtime_world_basics.py b/micropsi_core/tests/test_runtime_world_basics.py index 707122fa..1997bcfc 100644 --- a/micropsi_core/tests/test_runtime_world_basics.py +++ b/micropsi_core/tests/test_runtime_world_basics.py @@ -166,29 +166,6 @@ def test_reset_datatargets(test_world, test_nodenet): assert world.agents[test_nodenet].datatargets['engine_r'] == 0 -def test_actuators_do_not_reset_each_others_datatarget(test_world, test_nodenet): - world = runtime.worlds[test_world] - nodenet = runtime.get_nodenet(test_nodenet) - runtime.load_nodenet(test_nodenet) - nodenet.world = test_world - runtime.set_runner_properties(200, 1) - runtime.set_nodenet_properties(nodenet.uid, worldadapter='Braitenberg', world_uid=world.uid) - actor1 = nodenet.netapi.create_node("Actor", None) - actor2 = nodenet.netapi.create_node("Actor", None) - actor1.set_parameter('datatarget', 'engine_r') - actor2.set_parameter('datatarget', 'engine_r') - reg1 = nodenet.netapi.create_node("Register", None) - reg2 = nodenet.netapi.create_node("Register", None) - nodenet.netapi.link(reg1, 'gen', actor1, 'gen') - nodenet.netapi.link(reg2, 'gen', actor2, 'gen') - reg1.activation = 0.7 - reg2.activation = 0.3 - mock_reset = mock.Mock(return_value=None) - world.agents[test_nodenet].reset_datatargets = mock_reset - runtime.step_nodenet(test_nodenet) - assert world.agents[test_nodenet].datatargets['engine_r'] == 1 - - def test_worldadapter_update_calls_reset_datatargets(test_world, test_nodenet): world = runtime.worlds[test_world] nodenet = runtime.get_nodenet(test_nodenet) From b2175b7f15003969b8ee35003eb7c3610049387a Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Thu, 11 Feb 2016 13:24:43 +0100 Subject: [PATCH 039/306] minor --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index a7d684db..6f15b697 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -731,11 +731,11 @@ def delete_node(self, uid): # remove sensor association if there should be one if uid in self.sensormap.values(): - self.sensormap = {k:v for k, v in self.sensormap.items() if v is not uid } + self.sensormap = {k:v for k, v in self.sensormap.items() if v is not uid} # remove actuator association if there should be one if uid in self.actuatormap.values(): - self.actuatormap = {k:v for k, v in self.actuatormap.items() if v is not uid } + self.actuatormap = {k:v for k, v in self.actuatormap.items() if v is not uid} self.clear_supplements(uid) From ed4a851ae0414669a4f5da2b275a9eba87d58d0f Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Thu, 11 Feb 2016 13:35:13 +0100 Subject: [PATCH 040/306] Fixing tests --- micropsi_server/tests/test_json_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/micropsi_server/tests/test_json_api.py b/micropsi_server/tests/test_json_api.py index cf775209..d6aedf4b 100644 --- a/micropsi_server/tests/test_json_api.py +++ b/micropsi_server/tests/test_json_api.py @@ -993,8 +993,9 @@ def test_get_available_datatargets(app, test_nodenet, test_world): assert 'engine_r' in response.json_body['data'] -def test_bind_datasource_to_sensor(app, test_nodenet): +def test_bind_datasource_to_sensor(app, test_nodenet, test_world): app.set_auth() + response = app.post_json('/rpc/set_nodenet_properties', params=dict(nodenet_uid=test_nodenet, world_uid=test_world, worldadapter="Braitenberg")) response = app.post_json('/rpc/add_node', params={ 'nodenet_uid': test_nodenet, 'type': 'Sensor', @@ -1012,8 +1013,9 @@ def test_bind_datasource_to_sensor(app, test_nodenet): assert response.json_body['data']['parameters']['datasource'] == 'brightness_l' -def test_bind_datatarget_to_actor(app, test_nodenet): +def test_bind_datatarget_to_actor(app, test_nodenet, test_world): app.set_auth() + response = app.post_json('/rpc/set_nodenet_properties', params=dict(nodenet_uid=test_nodenet, world_uid=test_world, worldadapter="Braitenberg")) response = app.post_json('/rpc/add_node', params={ 'nodenet_uid': test_nodenet, 'type': 'Actor', From 016fb5dbc573326c2c00be5f540fae17cf204930 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Thu, 11 Feb 2016 14:21:23 +0100 Subject: [PATCH 041/306] Using the new ArrayWorldAdapter for time series --- micropsi_core/world/timeseries/timeseries.py | 20 ++++++++++++++------ micropsi_core/world/worldadapter.py | 16 +++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 3ecff76f..16b860c0 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -4,7 +4,7 @@ import os.path from configuration import config as cfg from micropsi_core.world.world import World -from micropsi_core.world.worldadapter import WorldAdapter +from micropsi_core.world.worldadapter import WorldAdapter, ArrayWorldAdapter import numpy as np @@ -89,14 +89,22 @@ def state(self): return self.timeseries[:, t] -class TimeSeriesRunner(WorldAdapter): +class TimeSeriesRunner(ArrayWorldAdapter): def __init__(self, world, uid=None, **data): super().__init__(world, uid, **data) + + self.available_datatargets = [] + self.available_datasources = [] + for idx, ID in enumerate(self.world.ids): - self.datasources[str(ID)] = 0 + self.available_datasources.append(str(ID)) + + def get_available_datasources(self): + return self.available_datasources + + def get_available_datatargets(self): + return self.available_datatargets def update_data_sources_and_targets(self): - state = self.world.state - for idx, ID in enumerate(self.world.ids): - self.datasources[str(ID)] = state[idx] + self.datasource_values = self.world.state \ No newline at end of file diff --git a/micropsi_core/world/worldadapter.py b/micropsi_core/world/worldadapter.py index 350bab93..4b86aa9a 100644 --- a/micropsi_core/world/worldadapter.py +++ b/micropsi_core/world/worldadapter.py @@ -21,9 +21,10 @@ from threading import Lock from micropsi_core.world.worldobject import WorldObject +from abc import ABCMeta, abstractmethod -class WorldAdapter(WorldObject): +class WorldAdapter(WorldObject, metaclass=ABCMeta): """Transmits data between agent and environment. The agent writes activation values into data targets, and receives it from data sources. The world adapter @@ -94,6 +95,7 @@ def reset_datatargets(self): for datatarget in self.datatargets: self.datatargets[datatarget] = 0 + @abstractmethod def update_data_sources_and_targets(self): """must be implemented by concrete world adapters to read datatargets and fill datasources""" pass @@ -119,9 +121,9 @@ def update_data_sources_and_targets(self): self.datasources['random'] = random.uniform(0, 1) -class BlippingWorldAdapter(WorldAdapter): +class ArrayWorldAdapter(WorldAdapter, metaclass=ABCMeta): """ - The BlippingWorldAdapter base class allows to avoid python dictionaries and loops for transmitting values + The ArrayWorldAdapter base class allows to avoid python dictionaries and loops for transmitting values to nodenet engines. Engines that bulk-query values, such as the theano_engine, will be faster. Numpy arrays can be passed directly into the engine. @@ -167,6 +169,7 @@ def reset_datatargets(self): """ resets (zeros) the datatargets """ self.datatarget_values[:] = 0 + @abstractmethod def get_available_datasources(self): """ must be implemented by the concrete world adapater and return a list of datasource name strings, @@ -174,6 +177,7 @@ def get_available_datasources(self): """ pass + @abstractmethod def get_available_datatargets(self): """ must be implemented by the concrete world adapater and return a list of datatarget name strings, @@ -181,6 +185,7 @@ def get_available_datatargets(self): """ pass + @abstractmethod def update_data_sources_and_targets(self): """ must be implemented by concrete world adapters to read and set the following arrays: @@ -188,7 +193,8 @@ def update_data_sources_and_targets(self): datatarget_values datatarget_feedback_values - Arrays sizes need to be equal to the corresponding dict objects. - The values of the dict objects will be bypassed and ignored. + Arrays sizes need to be equal to the corresponding responses of get_available_datasources() and + get_available_datatargets(). + Values of the superclass' dict objects will be bypassed and ignored. """ pass From 75e2132cf43876115ef7d2aa0401cbc4650eb83b Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 11 Feb 2016 14:32:37 +0100 Subject: [PATCH 042/306] also explore subclasses even if this class lies in a different module --- micropsi_core/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micropsi_core/tools.py b/micropsi_core/tools.py index 7eba4824..2349a401 100644 --- a/micropsi_core/tools.py +++ b/micropsi_core/tools.py @@ -226,5 +226,5 @@ def itersubclasses(cls, folder=None, _seen=None): if folder is None or sub.__module__.startswith(folder): _seen.add(sub) yield sub - for sub in itersubclasses(sub, folder=folder, _seen=_seen): - yield sub + for sub in itersubclasses(sub, folder=folder, _seen=_seen): + yield sub From b9fbf2270a397fdfdff6346d445964fea6a97ee2 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 11 Feb 2016 14:33:28 +0100 Subject: [PATCH 043/306] world gets a config param in init now --- micropsi_core/world/timeseries/timeseries.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 16b860c0..804d05c4 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -22,8 +22,8 @@ class TimeSeries(World): """ supported_worldadapters = ['TimeSeriesRunner'] - def __init__(self, filename, world_type="Island", name="", owner="", engine=None, uid=None, version=1): - World.__init__(self, filename, world_type=world_type, name=name, owner=owner, uid=uid, version=version) + def __init__(self, filename, world_type="Island", name="", owner="", engine=None, uid=None, version=1, config={}): + World.__init__(self, filename, world_type=world_type, name=name, owner=owner, uid=uid, version=version, config=config) path = os.path.join(cfg['micropsi2']['data_directory'], 'timeseries.npz') print("loading timeseries from", path, "for world", uid) From 67b1bb40b55fe8ba25f7c929455e3bc8904373c2 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Thu, 11 Feb 2016 14:56:20 +0100 Subject: [PATCH 044/306] Fixes for running TimeSeriesRunner --- micropsi_core/_runtime_api_world.py | 4 ++-- micropsi_core/world/worldadapter.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index dc21d4e9..c8a9ed1d 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -59,8 +59,8 @@ def get_worldadapters(world_uid, nodenet_uid=None): data[name] = {'description': worldadapter.__doc__} if nodenet_uid and nodenet_uid in world.agents: agent = world.agents[nodenet_uid] - data[agent.__class__.__name__]['datasources'] = sorted(agent.datasources.keys()) - data[agent.__class__.__name__]['datatargets'] = sorted(agent.datatargets.keys()) + data[agent.__class__.__name__]['datasources'] = agent.get_available_datasources() + data[agent.__class__.__name__]['datatargets'] = agent.get_available_datatargets() return data diff --git a/micropsi_core/world/worldadapter.py b/micropsi_core/world/worldadapter.py index 4b86aa9a..4a0b5758 100644 --- a/micropsi_core/world/worldadapter.py +++ b/micropsi_core/world/worldadapter.py @@ -129,6 +129,7 @@ class ArrayWorldAdapter(WorldAdapter, metaclass=ABCMeta): Numpy arrays can be passed directly into the engine. """ def __init__(self, world, uid=None, **data): + WorldAdapter.__init__(self, world, duid=uid) self.datasource_values = [] self.datatarget_values = [] self.datatarget_feedback_values = [] @@ -167,7 +168,7 @@ def set_datatarget_values(self, values): def reset_datatargets(self): """ resets (zeros) the datatargets """ - self.datatarget_values[:] = 0 + pass @abstractmethod def get_available_datasources(self): From 6d90c042db2d76cdd5227c465ab8637928cee5fd Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 11 Feb 2016 15:53:10 +0100 Subject: [PATCH 045/306] scan whole data directory for nodetypes, recipes annotate native_modules and recipes with category, annotate native_modules with a path to their nodefunction-file --- micropsi_core/nodenet/node.py | 9 +++++---- micropsi_core/runtime.py | 35 +++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/micropsi_core/nodenet/node.py b/micropsi_core/nodenet/node.py index 21774398..9d6fb475 100644 --- a/micropsi_core/nodenet/node.py +++ b/micropsi_core/nodenet/node.py @@ -599,22 +599,22 @@ def nodefunction_name(self, nodefunction_name): self._nodefunction_name = nodefunction_name try: if self.path: - module = SourceFileLoader("nodefunctions", os.path.join(RESOURCE_PATH, self.path, "nodefunctions.py")).load_module() + module = SourceFileLoader("nodefunctions", self.path).load_module() self.nodefunction = getattr(module, nodefunction_name) else: from micropsi_core.nodenet import nodefunctions if hasattr(nodefunctions, nodefunction_name): self.nodefunction = getattr(nodefunctions, nodefunction_name) else: - self.logger.warn("Can not find definition of nodefunction %s" % nodefunction_name) + self.logger.warning("Can not find definition of nodefunction %s" % nodefunction_name) except (ImportError, AttributeError) as err: - self.logger.warn("Import error while importing node function: nodefunctions.%s %s" % (nodefunction_name, err)) + self.logger.warning("Import error while importing node function: nodefunctions.%s %s" % (nodefunction_name, err)) raise err def __init__(self, name, nodenet, slottypes=None, gatetypes=None, parameters=None, nodefunction_definition=None, nodefunction_name=None, parameter_values=None, gate_defaults=None, - symbol=None, shape=None, engine=None, parameter_defaults=None, path=''): + symbol=None, shape=None, engine=None, parameter_defaults=None, path='', category=''): """Initializes or creates a nodetype. Arguments: @@ -634,6 +634,7 @@ def __init__(self, name, nodenet, slottypes=None, gatetypes=None, parameters=Non self.gatetypes = gatetypes or {} self.path = path + self.category = category self.logger = nodenet.logger diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 9f8f8561..0d9fdfef 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -37,6 +37,8 @@ WORLD_DIRECTORY = "worlds" RESOURCE_PATH = cfg['paths']['resource_path'] +sys.path.append(RESOURCE_PATH) + configs = config.ConfigurationManager(cfg['paths']['server_settings_path']) worlds = {} @@ -1140,7 +1142,8 @@ def get_available_recipes(): recipes[name] = { 'name': name, 'parameters': data['parameters'], - 'docstring': data['docstring'] + 'docstring': data['docstring'], + 'category': data['category'] } return recipes @@ -1287,9 +1290,10 @@ def parse_native_module_file(path): try: with open(path) as fp: modules = json.load(fp) - diffpath = os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) + category = os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) for key in modules: - modules[key]['path'] = diffpath + modules[key]['path'] = os.path.join(os.path.dirname(path), 'nodefunctions.py') + modules[key]['category'] = category if key in native_modules: logging.getLogger("system").warn("Native module names must be unique. %s is not." % key) native_modules[key] = modules[key] @@ -1300,18 +1304,20 @@ def parse_native_module_file(path): def parse_recipe_file(path, reload=False): global custom_recipes - import importlib.machinery + import importlib import inspect - diffpath = os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) - loader = importlib.machinery.SourceFileLoader("recipes", path) - recipes = loader.load_module() - # import recipes - all_modules = inspect.getmembers(recipes, inspect.ismodule) - for name, module in all_modules: + category = os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) + relpath = os.path.relpath(path, start=RESOURCE_PATH) + pyname = relpath.replace(os.path.sep, '.')[:-3] + + recipes = __import__(pyname, fromlist=['recipes']) + importlib.reload(sys.modules[pyname]) + + for name, module in inspect.getmembers(recipes, inspect.ismodule): if module.__file__.startswith(RESOURCE_PATH): - loader = importlib.machinery.SourceFileLoader(name, module.__file__) - loader.load_module() + module = importlib.reload(module) + all_functions = inspect.getmembers(recipes, inspect.isfunction) for name, func in all_functions: argspec = inspect.getargspec(func) @@ -1335,7 +1341,8 @@ def parse_recipe_file(path, reload=False): 'parameters': params, 'function': func, 'docstring': inspect.getdoc(func), - 'path': diffpath + 'category': category, + 'path': path } @@ -1374,7 +1381,7 @@ def reload_native_modules(): load_definitions() init_worlds(world_data) -load_user_files() +reload_native_modules() # initialize runners # Initialize the threads for the continuous simulation of nodenets and worlds From 202d1ca22159d8c0beb5987e4b30602041d1cc59 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 11 Feb 2016 15:54:25 +0100 Subject: [PATCH 046/306] ship gradient-descent nativemodules with toolkit --- micropsi_core/nodenet/native_modules.py | 1206 +++++++++++++++++++++++ micropsi_core/runtime.py | 2 + 2 files changed, 1208 insertions(+) create mode 100644 micropsi_core/nodenet/native_modules.py diff --git a/micropsi_core/nodenet/native_modules.py b/micropsi_core/nodenet/native_modules.py new file mode 100644 index 00000000..91ab0cd3 --- /dev/null +++ b/micropsi_core/nodenet/native_modules.py @@ -0,0 +1,1206 @@ +""" +Builtin native modules + +Currently contains + * GradientDescent for 3 layers (input, hidden, outpu) + * GradientDescent for LSTMS +""" + +import os + +nodetypes = {} + +try: + import numpy as np + import theano + numpy_installed = True +except ImportError: + numpy_installed = False + + +if numpy_installed: + # only register these native modules if we + # have theano and numpy installed. + nodetypes["GradientDescentLSTM"] = { + "name": "GradientDescentLSTM", + "engine": "theano_engine", + "slottypes": ["trigger", "debug"], + "gatetypes": ["e"], + "nodefunction_name": "gradient_descent_lstm", + "symbol": "↺", + "category": "backward_pass", + "path": os.path.abspath(__file__), + "parameters": [ + "adadelta_rho", + "adadelta_epsilon", + "sequence_length", + "links_io", + "links_porpor", + "links_porgin", + "links_porgou", + "links_porgfg", + "bias_gin", + "bias_gou", + "bias_gfg", + "group_t_nodes", + "group_t_gates", + "group_i_nodes", + "group_i_gates", + "group_c_nodes", + "group_o_nodes", + "group_o_gates" + ], + "parameter_values": { + "links_io": ["true", "false"], + "links_porpor": ["true", "false"], + "links_porgin": ["true", "false"], + "links_porgou": ["true", "false"], + "links_porgfg": ["true", "false"], + "bias_gin": ["true", "false"], + "bias_gou": ["true", "false"], + "bias_gfg": ["true", "false"] + }, + "parameter_defaults": { + "adadelta_rho": "0.95", + "adadelta_epsilon": "0.000001", + "sequence_length": "5", + "links_io": "true", + "links_porpor": "true", + "links_porgin": "true", + "links_porgou": "true", + "links_porgfg": "true", + "bias_gin": "true", + "bias_gou": "true", + "bias_gfg": "true", + "group_t_nodes": "target", + "group_t_gates": "gen", + "group_i_nodes": "input", + "group_i_gates": "gen", + "group_c_nodes": "lstm", + "group_o_nodes": "output", + "group_o_gates": "gen" + } + } + + nodetypes["GradientDescent"] = { + "name": "GradientDescent", + "engine": "theano_engine", + "slottypes": ["gen"], + "gatetypes": ["gen"], + "nodefunction_name": "gradient_descent", + "symbol": "☲", + "category": "backward_pass", + "path": os.path.abspath(__file__), + "parameters": [ + "input_nodesace", "hidden_nodespace", "output_nodespace" + "ae_type", "learning_rate", "sparsity_value", "sparsity_penalty", + "weight_decay", "adadelta_rho", "adadelta_eps", "t", "tied_weights", + "ctr", "check_grad", "input_layer", "hidden_layer", "output_layer" + ], + "parameter_values": { + "ae_type": ["sparse", "denoising"], + "tied_weights": ["True", "False"], + "check_grad": ["yes", "no"] + }, + "parameter_defaults": { + "ae_type": "denoising", + "tied_weights": "True", + "input_layer": "fov__", + "hidden_layer": "hidden_1", + "output_layer": "output_1" + } + } + + +def gradient_descent_lstm(netapi, node=None, **params): + """ + Gradient Descent for LSTMs + + The following assumes a three-layer architecture, with hidden LSTM nodes. + There is always a single LSTM cell per block (no multi-block cells are implemented). + + The following sets of weights are defined: + input -> output + input -> cell + input -> input gate + input -> output gate + input -> forget gate + cell -> output + cell -> input gate + cell -> output gate + cell -> forget gate + + The cell's constant error carousel link is explicitly modelled (as a gen loop). + Note that input, output and forget gate links aren't updated right now. + + Variable naming and implementation follows: + Gers & al. 1999, Learning to Forget - Continual Prediction with LSTM + + Other helpful papers: + Hochreiter & al. 1997, Long Short-Term Memory (introduces naming convention and most of the math) + Graves & al. 2005, Framewise Phoneme Classification with Bidrectional LSTM and Other NN Architectures + + For the Graves paper, a minimal, almost readable python implementation can be found at: + https://gist.github.com/neubig/ff2f97d91c9bed820c15 + + The ADADELTA implemetation follows the original ADADELTA paper: + Zeiler 2012, ADADELTA: An Adaptive Learning Rate Method + + A nice theano adadelta implementation is here: + https://blog.wtf.sg/2014/08/28/implementing-adadelta/ + """ + + from numbers import Number + from theano import tensor as T + + SEQUENCE_LENGTH = 3 + sequence_length_string = node.get_parameter("sequence_length") + if sequence_length_string is not None: + SEQUENCE_LENGTH = int(sequence_length_string) + + target = node.get_parameter("group_t_nodes") + target_gate = node.get_parameter("group_t_gates") + output = node.get_parameter("group_o_nodes") + output_gate = node.get_parameter("group_o_gates") + input = node.get_parameter("group_i_nodes") + input_gate = node.get_parameter("group_i_gates") + lstm = node.get_parameter("group_c_nodes") + lstm_gen = "%s_gen" % lstm + lstm_por = "%s_por" % lstm + lstm_gin = "%s_gin" % lstm + lstm_gou = "%s_gou" % lstm + lstm_gfg = "%s_gfg" % lstm + + nodespace = node.parent_nodespace + + if not hasattr(node, 'initialized'): + + # create the groups + netapi.group_nodes_by_names(nodespace, node_name_prefix=target, gate=target_gate) + netapi.group_nodes_by_names(nodespace, node_name_prefix=output, gate=output_gate) + netapi.group_nodes_by_names(nodespace, node_name_prefix=input, gate=input_gate) + + netapi.group_nodes_by_names(nodespace, node_name_prefix=lstm, gate="gen", group_name=lstm_gen) + netapi.group_nodes_by_names(nodespace, node_name_prefix=lstm, gate="por", group_name=lstm_por) + netapi.group_nodes_by_names(nodespace, node_name_prefix=lstm, gate="gin", group_name=lstm_gin) + netapi.group_nodes_by_names(nodespace, node_name_prefix=lstm, gate="gou", group_name=lstm_gou) + netapi.group_nodes_by_names(nodespace, node_name_prefix=lstm, gate="gfg", group_name=lstm_gfg) + + len_output = len(netapi.get_activations(nodespace, output)) + len_input = len(netapi.get_activations(nodespace, input)) + len_hidden = len(netapi.get_activations(nodespace, lstm_por)) + + # define a single LSTM-style backpropagation through time step, to be scanned over by theano + def bpttstep( + s, tgt, y_k, y_i, y_c, net_in, net_out, net_phi, + error, drv_ci_prev, drv_cc_prev, drv_ini_prev, drv_inc_prev, drv_in1_prev, drv_phii_prev, drv_phic_prev, drv_phi1_prev, + delta_w_ki, delta_w_kc, delta_w_outi, delta_w_outc, delta_w_ci, delta_w_cc, delta_w_ini, delta_w_inc, delta_w_phii, delta_w_phic, + delta_theta_i, delta_theta_k, delta_theta_in, delta_theta_out, delta_theta_phi, + w_kc, w_ci, w_cc, w_outc, w_outi, w_ini, w_inc, w_phii, w_phic): + + # calculate error + e_k = tgt - y_k # (12) error per output element + E = T.sum(T.square(e_k)) / 2. # (12) squared sum to be minimized + + # Part I: standard (truncated) BPTT for links to output registers and lstm output gate slots + # cell -> output + # cell -> output gate + # input -> output + # input -> output gate + + # functions and derivatives + y_in = T.nnet.sigmoid(net_in) # (3) y_in = f(net_in) + y_out = T.nnet.sigmoid(net_out) # (3) y_out = f(net_out) + y_phi = T.nnet.sigmoid(net_phi) # (3) y_phi = f(net_phi) + + h_s = 2 * T.nnet.sigmoid(s) - 1 # (8) + + f_primed_net_k = y_k * (1. - y_k) # f'(net_k) = f(net_k) * (1 - f(net_k)), f(net_k) provided as y_k + f_primed_net_out = y_out * (1. - y_out) + f_primed_net_in = y_in * (1. - y_in) + f_primed_net_phi = y_phi * (1. - y_phi) + # f_primed_net_i = y_i * (1. - y_i) + h_primed_s = (2 * T.exp(s)) / T.square(T.exp(s) + 1) + + delta_k = f_primed_net_k * e_k # (14) delta per output element + delta_out = f_primed_net_out * h_s * T.sum(w_kc * T.reshape(delta_k, (len_output, 1)), axis=0) # (15) delta per output gate + + # we use y_c and y_i here instead of y_i_prev because we have "flattened snapshots" to work with + # i.e. the partial derivative of net_k(t) with respect to w_kc is delta_k(t) * y_c(t) + # (y_c is what was propagated and created net_k) + delta_w_kc += T.dot(T.reshape(delta_k, (len_output, 1)), T.reshape(y_c, (1, len_hidden))) # (13) m = c + delta_w_ki += T.dot(T.reshape(delta_k, (len_output, 1)), T.reshape(y_i, (1, len_input))) # (13) m = i + delta_w_outi += T.dot(T.reshape(delta_out, (len_hidden, 1)), T.reshape(y_i, (1, len_input))) # (13) m = c + delta_w_outc += T.dot(T.reshape(delta_out, (len_hidden, 1)), T.reshape(y_c, (1, len_hidden))) # (13) m = i + + delta_theta_k += delta_k + delta_theta_out += delta_out + + # Part II: RTRL-style updates + # input -> cell + # cell -> cell + # input -> input gate + # cell -> input gate + # input -> forget gate + # cell -> forget gate + + net_c = T.dot(w_ci, y_i) # ugly re-calculation of forward pass for net_c + g_net_c = 4 * T.nnet.sigmoid(net_c) - 2 # (5) + g_primed_net_c = (4 * T.exp(net_c)) / T.square(T.exp(net_c) + 1) + + e_s = y_out * h_primed_s * T.sum(w_kc * T.reshape(delta_k, (len_output, 1)), axis=0) # (17) + + drv_ci = drv_ci_prev * T.reshape(y_phi, (len_hidden, 1)) \ + + T.dot(T.reshape(g_primed_net_c * y_in, (len_hidden, 1)), T.reshape(y_i, (1, len_input))) # (19) m = i + drv_cc = drv_cc_prev * T.reshape(y_phi, (len_hidden, 1)) \ + + T.dot(T.reshape(g_primed_net_c * y_in, (len_hidden, 1)), T.reshape(y_c, (1, len_hidden))) # (19) m = i + + drv_ini = drv_ini_prev * T.reshape(y_phi, (len_hidden, 1)) \ + + T.dot(T.reshape(g_net_c * f_primed_net_in, (len_hidden, 1)), T.reshape(y_i, (1, len_input))) # (20) m = i + drv_inc = drv_inc_prev * T.reshape(y_phi, (len_hidden, 1)) \ + + T.dot(T.reshape(g_net_c * f_primed_net_in, (len_hidden, 1)), T.reshape(y_c, (1, len_hidden))) # (20) m = c + drv_in1 = drv_in1_prev * y_phi + g_net_c * f_primed_net_in + + drv_phii = drv_phii_prev * T.reshape(y_phi, (len_hidden, 1)) \ + + T.dot(T.reshape(h_s * f_primed_net_phi, (len_hidden, 1)), T.reshape(y_i, (1, len_input))) # (21) m = i + drv_phic = drv_phic_prev * T.reshape(y_phi, (len_hidden, 1)) \ + + T.dot(T.reshape(h_s * f_primed_net_phi, (len_hidden, 1)), T.reshape(y_c, (1, len_hidden))) # (21) m = c + drv_phi1 = drv_phi1_prev * y_phi + h_s * f_primed_net_phi + + delta_w_ci += T.reshape(e_s, (len_hidden, 1)) * drv_ci + delta_w_cc += T.reshape(e_s, (len_hidden, 1)) * drv_cc + + delta_w_ini += T.reshape(e_s, (len_hidden, 1)) * drv_ini + delta_w_inc += T.reshape(e_s, (len_hidden, 1)) * drv_inc + + delta_w_phii += T.reshape(e_s, (len_hidden, 1)) * drv_phii + delta_w_phic += T.reshape(e_s, (len_hidden, 1)) * drv_phic + + # delta_theta_i += 0 + delta_theta_in += e_s * drv_in1 + delta_theta_phi += e_s * drv_phi1 + + error = E + + return error, drv_ci, drv_cc, drv_ini, drv_inc, drv_in1, drv_phii, drv_phic, drv_phi1, \ + delta_w_ki, delta_w_kc, delta_w_outi, delta_w_outc, delta_w_ci, delta_w_cc, delta_w_ini, delta_w_inc, delta_w_phii, delta_w_phic, \ + delta_theta_i, delta_theta_k, delta_theta_in, delta_theta_out, delta_theta_phi # cumulate + + node.set_state('current_error', 0.) + node.set_state('error', 0.) + node.set_state('updates', 0) + node.t = -1 + node.samples = 0 + + t_a_i_matrix = node.t_a_i_matrix = np.zeros(shape=(SEQUENCE_LENGTH, len_input)).astype(T.config.floatX) + t_a_t_matrix = node.t_a_t_matrix = np.zeros(shape=(SEQUENCE_LENGTH, len_output)).astype(T.config.floatX) + t_a_o_matrix = node.t_a_o_matrix = np.zeros(shape=(SEQUENCE_LENGTH, len_output)).astype(T.config.floatX) + t_a_h_gen_matrix = node.t_a_h_gen_matrix = np.zeros(shape=(SEQUENCE_LENGTH, len_hidden)).astype(T.config.floatX) + t_a_h_por_matrix = node.t_a_h_por_matrix = np.zeros(shape=(SEQUENCE_LENGTH, len_hidden)).astype(T.config.floatX) + t_a_h_gin_matrix = node.t_a_h_gin_matrix = np.zeros(shape=(SEQUENCE_LENGTH, len_hidden)).astype(T.config.floatX) + t_a_h_gou_matrix = node.t_a_h_gou_matrix = np.zeros(shape=(SEQUENCE_LENGTH, len_hidden)).astype(T.config.floatX) + t_a_h_gfg_matrix = node.t_a_h_gfg_matrix = np.zeros(shape=(SEQUENCE_LENGTH, len_hidden)).astype(T.config.floatX) + + w_oh_por_array = netapi.get_link_weights(nodespace, lstm_por, nodespace, output) + w_oi_array = netapi.get_link_weights(nodespace, input, nodespace, output) + w_h_por_i_array = netapi.get_link_weights(nodespace, input, nodespace, lstm_por) + w_h_gou_h_por_array = netapi.get_link_weights(nodespace, lstm_por, nodespace, lstm_gou) + w_h_gou_i_array = netapi.get_link_weights(nodespace, input, nodespace, lstm_gou) + w_h_por_h_por_array = netapi.get_link_weights(nodespace, lstm_por, nodespace, lstm_por) + w_h_gin_i_array = netapi.get_link_weights(nodespace, input, nodespace, lstm_gin) + w_h_gin_h_por_array = netapi.get_link_weights(nodespace, lstm_por, nodespace, lstm_gin) + w_h_gfg_i_array = netapi.get_link_weights(nodespace, input, nodespace, lstm_gfg) + w_h_gfg_h_por_array = netapi.get_link_weights(nodespace, lstm_por, nodespace, lstm_gfg) + + theta_input_array = netapi.get_thetas(nodespace, input) + theta_output_array = netapi.get_thetas(nodespace, output) + theta_lstm_gin_array = netapi.get_thetas(nodespace, lstm_gin) + theta_lstm_gou_array = netapi.get_thetas(nodespace, lstm_gou) + theta_lstm_gfg_array = netapi.get_thetas(nodespace, lstm_gfg) + + steps = T.iscalar("steps") + + # adadelta hyperparameters + rho = T.scalar("rho") + epsilon = T.scalar("epsilon") + + # activations -- post node/gatefunction, i.e. post-nonlinearities: y + # tgt t(t) + tgt = node.tgt = theano.shared(value=t_a_t_matrix.astype(T.config.floatX), name="tgt", borrow=False) + # output k(t) + y_k = node.y_k = theano.shared(value=t_a_o_matrix.astype(T.config.floatX), name="y_k", borrow=False) + # input i(t) + y_i = node.y_i = theano.shared(value=t_a_i_matrix.astype(T.config.floatX), name="y_i", borrow=False) + # cell state c(t) + y_c = node.y_c = theano.shared(value=t_a_h_por_matrix.astype(T.config.floatX), name="y_c", borrow=False) + # cell internal state (cec) s(t) + s = node.s = theano.shared(value=t_a_h_gen_matrix.astype(T.config.floatX), name="s", borrow=False) + + # for the LSTM gates, no node/gatefunction has been calculated, so we get net sums, not post-nonlinearity values + # output gate out(t) + net_out = node.net_out = theano.shared(value=t_a_h_gou_matrix.astype(T.config.floatX), name="net_out", borrow=False) + # input gate in(t) + net_in = node.net_in = theano.shared(value=t_a_h_gin_matrix.astype(T.config.floatX), name="net_in", borrow=False) + # forget gate phi(t) + net_phi = node.net_phi = theano.shared(value=t_a_h_gfg_matrix.astype(T.config.floatX), name="net_phi", borrow=False) + + # weight sets to be updated + # cell (c) -> output (k) + w_kc = node.w_kc = theano.shared(value=w_oh_por_array.astype(T.config.floatX), name="w_kc", borrow=False) + # input (i) -> output (k) + w_ki = node.w_ki = theano.shared(value=w_oi_array.astype(T.config.floatX), name="w_ki", borrow=False) + # cell (c) -> output gate (out) + w_outc = node.w_outc = theano.shared(value=w_h_gou_h_por_array.astype(T.config.floatX), name="w_outc", borrow=False) + # input (i) -> output gate (out) + w_outi = node.w_outi = theano.shared(value=w_h_gou_i_array.astype(T.config.floatX), name="w_outi", borrow=False) + # input (i) -> cell (c) + w_ci = node.w_ci = theano.shared(value=w_h_por_i_array.astype(T.config.floatX), name="w_ci", borrow=False) + # input (i) -> cell (c) + w_cc = node.w_cc = theano.shared(value=w_h_por_h_por_array.astype(T.config.floatX), name="w_cc", borrow=False) + # input (i) -> input gate (in) + w_ini = node.w_ini = theano.shared(value=w_h_gin_i_array.astype(T.config.floatX), name="w_ini", borrow=False) + # cell (c) -> input gate (in) + w_inc = node.w_inc = theano.shared(value=w_h_gin_h_por_array.astype(T.config.floatX), name="w_inc", borrow=False) + # input (i) -> forget gate (phi) + w_phii = node.w_phii = theano.shared(value=w_h_gfg_i_array.astype(T.config.floatX), name="w_phii", borrow=False) + # cell (c) -> forget gate (phi) + w_phic = node.w_phic = theano.shared(value=w_h_gfg_h_por_array.astype(T.config.floatX), name="w_phic", borrow=False) + + # bias sets to be updated + theta_i = node.theta_i = theano.shared(value=theta_input_array.astype(T.config.floatX), name="theta_i", borrow=False) + theta_k = node.theta_k = theano.shared(value=theta_output_array.astype(T.config.floatX), name="theta_k", borrow=False) + theta_in = node.theta_in = theano.shared(value=theta_lstm_gin_array.astype(T.config.floatX), name="theta_in", borrow=False) + theta_out = node.theta_out = theano.shared(value=theta_lstm_gou_array.astype(T.config.floatX), name="theta_out", borrow=False) + theta_phi = node.theta_phi = theano.shared(value=theta_lstm_gfg_array.astype(T.config.floatX), name="theta_phi", borrow=False) + + # adadelta gradients and delta accumulation variables + node.accu_grad_w_kc = theano.shared(value=np.zeros_like(w_oh_por_array), name="accu_grad_w_kc", borrow=True) + node.accu_delta_w_kc = theano.shared(value=np.zeros_like(w_oh_por_array), name="accu_delta_w_kc", borrow=True) + node.accu_grad_w_ki = theano.shared(value=np.zeros_like(w_oi_array), name="accu_grad_w_ki", borrow=True) + node.accu_delta_w_ki = theano.shared(value=np.zeros_like(w_oi_array), name="accu_delta_w_ki", borrow=True) + node.accu_grad_w_outc = theano.shared(value=np.zeros_like(w_h_gou_h_por_array), name="accu_grad_w_outc", borrow=True) + node.accu_delta_w_outc = theano.shared(value=np.zeros_like(w_h_gou_h_por_array), name="accu_delta_w_outc", borrow=True) + node.accu_grad_w_outi = theano.shared(value=np.zeros_like(w_h_gou_i_array), name="accu_grad_w_outi", borrow=True) + node.accu_delta_w_outi = theano.shared(value=np.zeros_like(w_h_gou_i_array), name="accu_delta_w_outi", borrow=True) + node.accu_grad_w_ci = theano.shared(value=np.zeros_like(w_h_por_i_array), name="accu_grad_w_ci", borrow=True) + node.accu_delta_w_ci = theano.shared(value=np.zeros_like(w_h_por_i_array), name="accu_delta_w_ci", borrow=True) + node.accu_grad_w_cc = theano.shared(value=np.zeros_like(w_h_por_h_por_array), name="accu_grad_w_cc", borrow=True) + node.accu_delta_w_cc = theano.shared(value=np.zeros_like(w_h_por_h_por_array), name="accu_delta_w_cc", borrow=True) + node.accu_grad_w_ini = theano.shared(value=np.zeros_like(w_h_gin_i_array), name="accu_grad_w_ini", borrow=True) + node.accu_delta_w_ini = theano.shared(value=np.zeros_like(w_h_gin_i_array), name="accu_delta_w_ini", borrow=True) + node.accu_grad_w_inc = theano.shared(value=np.zeros_like(w_h_gin_h_por_array), name="accu_grad_w_inc", borrow=True) + node.accu_delta_w_inc = theano.shared(value=np.zeros_like(w_h_gin_h_por_array), name="accu_delta_w_inc", borrow=True) + node.accu_grad_w_phii = theano.shared(value=np.zeros_like(w_h_gfg_i_array), name="accu_grad_w_phii", borrow=True) + node.accu_delta_w_phii = theano.shared(value=np.zeros_like(w_h_gfg_i_array), name="accu_delta_w_phii", borrow=True) + node.accu_grad_w_phic = theano.shared(value=np.zeros_like(w_h_gfg_h_por_array), name="accu_grad_w_phic", borrow=True) + node.accu_delta_w_phic = theano.shared(value=np.zeros_like(w_h_gfg_h_por_array), name="accu_delta_w_phic", borrow=True) + node.accu_grad_theta_k = theano.shared(value=np.zeros_like(theta_output_array), name="accu_grad_theta_k", borrow=True) + node.accu_delta_theta_k = theano.shared(value=np.zeros_like(theta_output_array), name="accu_delta_theta_k", borrow=True) + node.accu_grad_theta_out = theano.shared(value=np.zeros_like(theta_lstm_gou_array), name="accu_grad_theta_out", borrow=True) + node.accu_delta_theta_out = theano.shared(value=np.zeros_like(theta_lstm_gou_array), name="accu_delta_theta_out", borrow=True) + node.accu_grad_theta_in = theano.shared(value=np.zeros_like(theta_lstm_gin_array), name="accu_grad_theta_in", borrow=True) + node.accu_delta_theta_in = theano.shared(value=np.zeros_like(theta_lstm_gin_array), name="accu_delta_theta_in", borrow=True) + node.accu_grad_theta_phi = theano.shared(value=np.zeros_like(theta_lstm_gfg_array), name="accu_grad_theta_phi", borrow=True) + node.accu_delta_theta_phi = theano.shared(value=np.zeros_like(theta_lstm_gfg_array), name="accu_delta_theta_phi", borrow=True) + + [errors, + deriv_ci_prev, + deriv_cc_prev, + deriv_ini_prev, + deriv_inc_prev, + deriv_in1_prev, + deriv_phii_prev, + deriv_phic_prev, + deriv_phi1_prev, + grad_w_ki, + grad_w_kc, + grad_w_outi, + grad_w_outc, + grad_w_ci, + grad_w_cc, + grad_w_ini, + grad_w_inc, + grad_w_phii, + grad_w_phic, + grad_theta_i, + grad_theta_k, + grad_theta_in, + grad_theta_out, + grad_theta_phi], updates = theano.scan( + fn=bpttstep, + sequences=[dict(input=s, taps=[-0]), + dict(input=tgt, taps=[-0]), + dict(input=y_k, taps=[-0]), + dict(input=y_i, taps=[-0]), + dict(input=y_c, taps=[-0]), + dict(input=net_in, taps=[-0]), + dict(input=net_out, taps=[-0]), + dict(input=net_phi, taps=[-0])], + outputs_info=[0., # error + T.zeros_like(w_ci, dtype=T.config.floatX), # deriv_ci_prev + T.zeros_like(w_cc, dtype=T.config.floatX), # deriv_cc_prev + T.zeros_like(w_ini, dtype=T.config.floatX), # deriv_ini_prev + T.zeros_like(w_inc, dtype=T.config.floatX), # deriv_inc_prev + T.zeros_like(theta_in, dtype=T.config.floatX), # deriv_in1_prev + T.zeros_like(w_phii, dtype=T.config.floatX), # deriv_phii_prev + T.zeros_like(w_phic, dtype=T.config.floatX), # deriv_phic_prev + T.zeros_like(theta_phi, dtype=T.config.floatX), # deriv_phi1_prev + T.zeros_like(w_ki, dtype=T.config.floatX), # delta_w_ki + T.zeros_like(w_kc, dtype=T.config.floatX), # delta_w_kc + T.zeros_like(w_outi, dtype=T.config.floatX), # delta_w_outi + T.zeros_like(w_outc, dtype=T.config.floatX), # delta_w_outc + T.zeros_like(w_ci, dtype=T.config.floatX), # delta_w_ci + T.zeros_like(w_cc, dtype=T.config.floatX), # delta_w_cc + T.zeros_like(w_ini, dtype=T.config.floatX), # delta_w_ini + T.zeros_like(w_inc, dtype=T.config.floatX), # delta_w_inc + T.zeros_like(w_phii, dtype=T.config.floatX), # delta_w_phii + T.zeros_like(w_phic, dtype=T.config.floatX), # delta_w_phic + T.zeros_like(theta_i, dtype=T.config.floatX), # delta_theta_i + T.zeros_like(theta_k, dtype=T.config.floatX), # delta_theta_k + T.zeros_like(theta_in, dtype=T.config.floatX), # delta_theta_in + T.zeros_like(theta_out, dtype=T.config.floatX), # delta_theta_out + T.zeros_like(theta_phi, dtype=T.config.floatX)], # delta_theta_phi + non_sequences=[w_kc, + w_ci, + w_cc, + w_outc, + w_outi, + w_ini, + w_inc, + w_phii, + w_phic], + go_backwards=True, + n_steps=steps, + strict=True) + + # adadelta momentum + accu_grad_w_kc = rho * node.accu_grad_w_kc + (1. - rho) * (grad_w_kc[SEQUENCE_LENGTH - 1]**2) + delta_w_kc = (T.sqrt(node.accu_delta_w_kc + epsilon) / T.sqrt(accu_grad_w_kc + epsilon)) * grad_w_kc[SEQUENCE_LENGTH - 1] + accu_delta_w_kc = rho * node.accu_delta_w_kc + (1. - rho) * (delta_w_kc**2) + + accu_grad_w_ki = rho * node.accu_grad_w_ki + (1. - rho) * (grad_w_ki[SEQUENCE_LENGTH - 1]**2) + delta_w_ki = (T.sqrt(node.accu_delta_w_ki + epsilon) / T.sqrt(accu_grad_w_ki + epsilon)) * grad_w_ki[SEQUENCE_LENGTH - 1] + accu_delta_w_ki = rho * node.accu_delta_w_ki + (1. - rho) * (delta_w_ki**2) + + accu_grad_w_outc = rho * node.accu_grad_w_outc + (1. - rho) * (grad_w_outc[SEQUENCE_LENGTH - 1]**2) + delta_w_outc = (T.sqrt(node.accu_delta_w_outc + epsilon) / T.sqrt(accu_grad_w_outc + epsilon)) * grad_w_outc[SEQUENCE_LENGTH - 1] + accu_delta_w_outc = rho * node.accu_delta_w_outc + (1. - rho) * (delta_w_outc**2) + + accu_grad_w_outi = rho * node.accu_grad_w_outi + (1. - rho) * (grad_w_outi[SEQUENCE_LENGTH - 1]**2) + delta_w_outi = (T.sqrt(node.accu_delta_w_outi + epsilon) / T.sqrt(accu_grad_w_outi + epsilon)) * grad_w_outi[SEQUENCE_LENGTH - 1] + accu_delta_w_outi = rho * node.accu_delta_w_outi + (1. - rho) * (delta_w_outi**2) + + accu_grad_w_ci = rho * node.accu_grad_w_ci + (1. - rho) * (grad_w_ci[SEQUENCE_LENGTH - 1]**2) + delta_w_ci = (T.sqrt(node.accu_delta_w_ci + epsilon) / T.sqrt(accu_grad_w_ci + epsilon)) * grad_w_ci[SEQUENCE_LENGTH - 1] + accu_delta_w_ci = rho * node.accu_delta_w_ci + (1. - rho) * (delta_w_ci**2) + + accu_grad_w_cc = rho * node.accu_grad_w_cc + (1. - rho) * (grad_w_cc[SEQUENCE_LENGTH - 1]**2) + delta_w_cc = (T.sqrt(node.accu_delta_w_cc + epsilon) / T.sqrt(accu_grad_w_cc + epsilon)) * grad_w_cc[SEQUENCE_LENGTH - 1] + accu_delta_w_cc = rho * node.accu_delta_w_cc + (1. - rho) * (delta_w_cc**2) + + accu_grad_w_ini = rho * node.accu_grad_w_ini + (1. - rho) * (grad_w_ini[SEQUENCE_LENGTH - 1]**2) + delta_w_ini = (T.sqrt(node.accu_delta_w_ini + epsilon) / T.sqrt(accu_grad_w_ini + epsilon)) * grad_w_ini[SEQUENCE_LENGTH - 1] + accu_delta_w_ini = rho * node.accu_delta_w_ini + (1. - rho) * (delta_w_ini**2) + + accu_grad_w_inc = rho * node.accu_grad_w_inc + (1. - rho) * (grad_w_inc[SEQUENCE_LENGTH - 1]**2) + delta_w_inc = (T.sqrt(node.accu_delta_w_inc + epsilon) / T.sqrt(accu_grad_w_inc + epsilon)) * grad_w_inc[SEQUENCE_LENGTH - 1] + accu_delta_w_inc = rho * node.accu_delta_w_inc + (1. - rho) * (delta_w_inc**2) + + accu_grad_w_phii = rho * node.accu_grad_w_phii + (1. - rho) * (grad_w_phii[SEQUENCE_LENGTH - 1]**2) + delta_w_phii = (T.sqrt(node.accu_delta_w_phii + epsilon) / T.sqrt(accu_grad_w_phii + epsilon)) * grad_w_phii[SEQUENCE_LENGTH - 1] + accu_delta_w_phii = rho * node.accu_delta_w_phii + (1. - rho) * (delta_w_phii**2) + + accu_grad_w_phic = rho * node.accu_grad_w_phic + (1. - rho) * (grad_w_phic[SEQUENCE_LENGTH - 1]**2) + delta_w_phic = (T.sqrt(node.accu_delta_w_phic + epsilon) / T.sqrt(accu_grad_w_phic + epsilon)) * grad_w_phic[SEQUENCE_LENGTH - 1] + accu_delta_w_phic = rho * node.accu_delta_w_phic + (1. - rho) * (delta_w_phic**2) + + accu_grad_theta_k = rho * node.accu_grad_theta_k + (1. - rho) * (grad_theta_k[SEQUENCE_LENGTH - 1]**2) + delta_theta_k = (T.sqrt(node.accu_delta_theta_k + epsilon) / T.sqrt(accu_grad_theta_k + epsilon)) * grad_theta_k[SEQUENCE_LENGTH - 1] + accu_delta_theta_k = rho * node.accu_delta_theta_k + (1. - rho) * (delta_theta_k**2) + + accu_grad_theta_out = rho * node.accu_grad_theta_out + (1. - rho) * (grad_theta_out[SEQUENCE_LENGTH - 1]**2) + delta_theta_out = (T.sqrt(node.accu_delta_theta_out + epsilon) / T.sqrt(accu_grad_theta_out + epsilon)) * grad_theta_out[SEQUENCE_LENGTH - 1] + accu_delta_theta_out = rho * node.accu_delta_theta_out + (1. - rho) * (delta_theta_out**2) + + accu_grad_theta_in = rho * node.accu_grad_theta_in + (1. - rho) * (grad_theta_in[SEQUENCE_LENGTH - 1]**2) + delta_theta_in = (T.sqrt(node.accu_delta_theta_in + epsilon) / T.sqrt(accu_grad_theta_in + epsilon)) * grad_theta_in[SEQUENCE_LENGTH - 1] + accu_delta_theta_in = rho * node.accu_delta_theta_in + (1. - rho) * (delta_theta_in**2) + + accu_grad_theta_phi = rho * node.accu_grad_theta_phi + (1. - rho) * (grad_theta_phi[SEQUENCE_LENGTH - 1]**2) + delta_theta_phi = (T.sqrt(node.accu_delta_theta_phi + epsilon) / T.sqrt(accu_grad_theta_phi + epsilon)) * grad_theta_phi[SEQUENCE_LENGTH - 1] + accu_delta_theta_phi = rho * node.accu_delta_theta_phi + (1. - rho) * (delta_theta_phi**2) + + # update weights + w_kc += delta_w_kc + w_ki += delta_w_ki + w_outc += delta_w_outc + w_outi += delta_w_outi + w_ci += delta_w_ci + w_cc += delta_w_cc + w_ini += delta_w_ini + w_inc += delta_w_inc + w_phii += delta_w_phii + w_phic += delta_w_phic + + # update biases + # theta_i += delta_theta_i + theta_k += delta_theta_k + theta_out += delta_theta_out + theta_in += delta_theta_in + theta_phi += delta_theta_phi + + # this will provide new w values to be written back to the node net, + # as well as deriv_lm_prev values to be used in the next step + node.get_updated_parameters = theano.function([rho, epsilon, steps], + errors, + updates=[(node.w_kc, w_kc), + (node.w_ki, w_ki), + (node.w_outc, w_outc), + (node.w_outi, w_outi), + (node.w_ci, w_ci), + (node.w_cc, w_cc), + (node.w_ini, w_ini), + (node.w_inc, w_inc), + (node.w_phii, w_phii), + (node.w_phic, w_phic), + (node.theta_i, theta_i), + (node.theta_k, theta_k), + (node.theta_in, theta_in), + (node.theta_out, theta_out), + (node.theta_phi, theta_phi), + (node.accu_grad_w_kc, accu_grad_w_kc), + (node.accu_delta_w_kc, accu_delta_w_kc), + (node.accu_grad_w_ki, accu_grad_w_ki), + (node.accu_delta_w_ki, accu_delta_w_ki), + (node.accu_grad_w_outc, accu_grad_w_outc), + (node.accu_delta_w_outc, accu_delta_w_outc), + (node.accu_grad_w_outi, accu_grad_w_outi), + (node.accu_delta_w_outi, accu_delta_w_outi), + (node.accu_grad_w_ci, accu_grad_w_ci), + (node.accu_delta_w_ci, accu_delta_w_ci), + (node.accu_grad_w_cc, accu_grad_w_cc), + (node.accu_delta_w_cc, accu_delta_w_cc), + (node.accu_grad_w_ini, accu_grad_w_ini), + (node.accu_delta_w_ini, accu_delta_w_ini), + (node.accu_grad_w_inc, accu_grad_w_inc), + (node.accu_delta_w_inc, accu_delta_w_inc), + (node.accu_grad_w_phii, accu_grad_w_phii), + (node.accu_delta_w_phii, accu_delta_w_phii), + (node.accu_grad_w_phic, accu_grad_w_phic), + (node.accu_delta_w_phic, accu_delta_w_phic), + (node.accu_grad_theta_k, accu_grad_theta_k), + (node.accu_delta_theta_k, accu_delta_theta_k), + (node.accu_grad_theta_out, accu_grad_theta_out), + (node.accu_delta_theta_out, accu_delta_theta_out), + (node.accu_grad_theta_in, accu_grad_theta_in), + (node.accu_delta_theta_in, accu_delta_theta_in), + (node.accu_grad_theta_phi, accu_grad_theta_phi), + (node.accu_delta_theta_phi, accu_delta_theta_phi) + ], + on_unused_input='warn') + + node.get_error = theano.function([], T.sum(T.square(tgt[SEQUENCE_LENGTH] - y_k[SEQUENCE_LENGTH])) / 2.) + + node.initialized = True + + # every step + + error_prev = node.get_state("current_error") + if error_prev is None: + error_prev = 0. + node.get_gate('e').gate_function(error_prev) + + if netapi.step % 3 == 0 and node.get_slot("debug").activation > 0.5: + netapi.logger.debug("%10i: lstm sample step" % netapi.step) + + if netapi.step % 3 != 1: + return + # every three steps, sample activation from LSTMs + + node.t += 1 + if node.t >= SEQUENCE_LENGTH: + node.t = 0 + + # roll time snapshots to the left + node.t_a_i_matrix = np.roll(node.t_a_i_matrix, -1, 0) + node.t_a_t_matrix = np.roll(node.t_a_t_matrix, -1, 0) + node.t_a_o_matrix = np.roll(node.t_a_o_matrix, -1, 0) + node.t_a_h_gen_matrix = np.roll(node.t_a_h_gen_matrix, -1, 0) + node.t_a_h_por_matrix = np.roll(node.t_a_h_por_matrix, -1, 0) + node.t_a_h_gin_matrix = np.roll(node.t_a_h_gin_matrix, -1, 0) + node.t_a_h_gou_matrix = np.roll(node.t_a_h_gou_matrix, -1, 0) + node.t_a_h_gfg_matrix = np.roll(node.t_a_h_gfg_matrix, -1, 0) + + # insert new snapshot at the end + node.t_a_i_matrix[SEQUENCE_LENGTH - 1, :] = netapi.get_activations(nodespace, input) + node.t_a_t_matrix[SEQUENCE_LENGTH - 1, :] = netapi.get_activations(nodespace, target) + node.t_a_o_matrix[SEQUENCE_LENGTH - 1, :] = netapi.get_activations(nodespace, output) + node.t_a_h_gen_matrix[SEQUENCE_LENGTH - 1, :] = netapi.get_activations(nodespace, lstm_gen) + node.t_a_h_por_matrix[SEQUENCE_LENGTH - 1, :] = netapi.get_activations(nodespace, lstm_por) + node.t_a_h_gin_matrix[SEQUENCE_LENGTH - 1, :] = netapi.get_activations(nodespace, lstm_gou) + node.t_a_h_gou_matrix[SEQUENCE_LENGTH - 1, :] = netapi.get_activations(nodespace, lstm_gin) + node.t_a_h_gfg_matrix[SEQUENCE_LENGTH - 1, :] = netapi.get_activations(nodespace, lstm_gfg) + node.samples += 1 + + if node.get_slot("debug").activation > 0.5: + netapi.logger.debug("%10i: bp sample #%i t, i, c, k data: t[0]=%.6f i[0]=%.6f c[0]=%.6f k[0]=%.6f" + % (netapi.step, node.t, node.t_a_t_matrix[node.t, 0], node.t_a_i_matrix[node.t, 0], + node.t_a_h_por_matrix[node.t, 0], node.t_a_o_matrix[node.t, 0])) + + if node.t != SEQUENCE_LENGTH - 1 or node.samples < 3: + return + # every sequence length samples, do backpropagation-through-time for the sampled sequence + + # netapi.logger.debug("t=%.6f o=%.6f s=%.6f c=%.6f i=%.6f" % (node.t_a_t_matrix[0, 0], node.t_a_o_matrix[0, 0], + # node.t_a_h_gen_matrix[0, 0], node.t_a_h_por_matrix[0, 0], node.t_a_i_matrix[0, 0])) + # netapi.logger.debug("t=%.6f o=%.6f s=%.6f c=%.6f i=%.6f" % (node.t_a_t_matrix[1, 0], node.t_a_o_matrix[1, 0], + # node.t_a_h_gen_matrix[1, 0], node.t_a_h_por_matrix[1, 0], node.t_a_i_matrix[1, 0])) + # netapi.logger.debug("t=%.6f o=%.6f s=%.6f c=%.6f i=%.6f" % (node.t_a_t_matrix[2, 0], node.t_a_o_matrix[2, 0], + # node.t_a_h_gen_matrix[2, 0], node.t_a_h_por_matrix[2, 0], node.t_a_i_matrix[2, 0])) + + # fill w and a variables with values from the Node Net + node.w_kc.set_value(netapi.get_link_weights(nodespace, lstm_por, nodespace, output), borrow=True) + node.w_ki.set_value(netapi.get_link_weights(nodespace, input, nodespace, output), borrow=True) + node.w_outc.set_value(netapi.get_link_weights(nodespace, lstm_por, nodespace, lstm_gou), borrow=True) + node.w_outi.set_value(netapi.get_link_weights(nodespace, input, nodespace, lstm_gou), borrow=True) + node.w_ci.set_value(netapi.get_link_weights(nodespace, input, nodespace, lstm_por), borrow=True) + node.w_cc.set_value(netapi.get_link_weights(nodespace, lstm_por, nodespace, lstm_por), borrow=True) + node.w_ini.set_value(netapi.get_link_weights(nodespace, input, nodespace, lstm_gin), borrow=True) + node.w_inc.set_value(netapi.get_link_weights(nodespace, lstm_por, nodespace, lstm_gin), borrow=True) + node.w_phii.set_value(netapi.get_link_weights(nodespace, input, nodespace, lstm_gfg), borrow=True) + node.w_phic.set_value(netapi.get_link_weights(nodespace, lstm_por, nodespace, lstm_gfg), borrow=True) + + node.theta_i.set_value(netapi.get_thetas(nodespace, input), borrow=True) + node.theta_k.set_value(netapi.get_thetas(nodespace, output), borrow=True) + node.theta_in.set_value(netapi.get_thetas(nodespace, lstm_gin), borrow=True) + node.theta_out.set_value(netapi.get_thetas(nodespace, lstm_gou), borrow=True) + node.theta_phi.set_value(netapi.get_thetas(nodespace, lstm_gfg), borrow=True) + + node.tgt.set_value(node.t_a_t_matrix, borrow=True) + node.y_k.set_value(node.t_a_o_matrix, borrow=True) + node.y_i.set_value(node.t_a_i_matrix, borrow=True) + node.y_c.set_value(node.t_a_h_por_matrix, borrow=True) + node.s.set_value(node.t_a_h_gen_matrix, borrow=True) + node.net_out.set_value(node.t_a_h_gou_matrix, borrow=True) + node.net_in.set_value(node.t_a_h_gin_matrix, borrow=True) + node.net_phi.set_value(node.t_a_h_gfg_matrix, borrow=True) + + rho = float(node.get_parameter('adadelta_rho')) + if not isinstance(rho, Number): + rho = 0.95 + node.set_parameter('adadelta_rho', rho) + + epsilon = float(node.get_parameter('adadelta_epsilon')) + if not isinstance(epsilon, Number): + epsilon = 0.000001 + node.set_parameter('adadelta_epsilon', epsilon) + + len_output = len(netapi.get_activations(nodespace, output)) + len_input = len(netapi.get_activations(nodespace, input)) + len_hidden = len(netapi.get_activations(nodespace, lstm_por)) + + # update the weights, all derivatives and weight update sums are 0 for the first step + errors = node.get_updated_parameters(rho, epsilon, node.t + 1) + + if node.get_slot("debug").activation > 0.5: + netapi.logger.debug("%10i: bp with error %.4f" % (netapi.step, errors[SEQUENCE_LENGTH - 1])) + + # write back changed weights to node net + + # netapi.set_thetas(nodespace, input, node.theta_i.get_value(borrow=True)) + if node.get_parameter("bias_gin") == "true": + netapi.set_thetas(nodespace, lstm_gin, node.theta_in.get_value(borrow=True)) + if node.get_parameter("bias_gou") == "true": + netapi.set_thetas(nodespace, lstm_gou, node.theta_out.get_value(borrow=True)) + if node.get_parameter("bias_gfg") == "true": + netapi.set_thetas(nodespace, lstm_gfg, node.theta_phi.get_value(borrow=True)) + + netapi.set_link_weights(nodespace, input, nodespace, lstm_gou, node.w_outi.get_value(borrow=True)) + netapi.set_link_weights(nodespace, input, nodespace, lstm_por, node.w_ci.get_value(borrow=True)) + netapi.set_link_weights(nodespace, input, nodespace, lstm_gin, node.w_ini.get_value(borrow=True)) + netapi.set_link_weights(nodespace, input, nodespace, lstm_gfg, node.w_phii.get_value(borrow=True)) + netapi.set_link_weights(nodespace, lstm_por, nodespace, output, node.w_kc.get_value(borrow=True)) + + if node.get_parameter("links_io") == "true": + netapi.set_link_weights(nodespace, input, nodespace, output, node.w_ki.get_value(borrow=True)) + if node.get_parameter("links_porpor") == "true": + netapi.set_link_weights(nodespace, lstm_por, nodespace, lstm_por, node.w_cc.get_value(borrow=True)) + if node.get_parameter("links_porgin") == "true": + netapi.set_link_weights(nodespace, lstm_por, nodespace, lstm_gin, node.w_inc.get_value(borrow=True)) + if node.get_parameter("links_porgou") == "true": + netapi.set_link_weights(nodespace, lstm_por, nodespace, lstm_gou, node.w_outc.get_value(borrow=True)) + if node.get_parameter("links_porgfg") == "true": + netapi.set_link_weights(nodespace, lstm_por, nodespace, lstm_gfg, node.w_phic.get_value(borrow=True)) + + node.set_state('current_error', errors[SEQUENCE_LENGTH - 1]) + node.set_state('error', node.get_state('error') + errors[SEQUENCE_LENGTH - 1]) + if node.get_state('updates') % 100 == 0: + netapi.logger.debug("Number of lstm backprop steps computed %d" % node.get_state('updates')) + netapi.logger.debug("Error %.6f (Latest from loop: 0=%.6f)" % ((node.get_state('error') / 100), errors[SEQUENCE_LENGTH - 1])) + node.set_state('error', 0.0) + + # after weight updates, reset gen loops of lstms + netapi.substitute_activations(nodespace, lstm_gen, np.zeros_like(netapi.get_activations(nodespace, lstm_gen))) + # netapi.substitute_activations(nodespace, "lstm_por", np.zeros_like(a_h_por_array)) + + node.set_state('updates', node.get_state('updates') + 1) + + +def gradient_descent(netapi, node=None, **params): + """ + Online gradient descent with backpropagation for three layers (input, hidden, + and output layer) and AdaDelta for adapting the learning rate per parameter. + + References: + [1] Werbos, PJ. "Beyond Regression: New Tools for Prediction and Analysis + in the Behavioral Sciences." (1974). + [2] Zeiler, MD. "ADADELTA: An adaptive learning rate method." (2012). + [3] Vincent, P. "Extracting and Composing Robust Features with Denoising + Autoencoders." (2008). + """ + + # To be able to switch this native module on and off, require positive + # activation on the gen slot for its code to be run. + if node.get_slot('gen').activation > 0: + + import theano + import theano.tensor as T + + # get shared name prefix of nodes in input, hidden, and output layers + input_ = node.get_parameter('input_layer') + hidden = node.get_parameter('hidden_layer') + output = node.get_parameter('output_layer') + + # get nodespaces, default to parent nodespace + ns_input_uid = params.get("input_nodespace") or node.parent_nodespace + ns_hidden_uid = params.get("hidden_nodespace") or node.parent_nodespace + ns_output_uid = params.get("output_nodespace") or node.parent_nodespace + + # initialization + if not hasattr(node, 'initialized'): + + sparse = str(node.get_parameter('ae_type')) == "sparse" + # denoising = str(node.get_parameter('ae_type')) == "denoising" + tied_weights = str(node.get_parameter('tied_weights')) == "True" + + # group nodes + netapi.group_nodes_by_names(ns_input_uid, node_name_prefix=input_) + netapi.group_nodes_by_names(ns_hidden_uid, node_name_prefix=hidden) + netapi.group_nodes_by_names(ns_output_uid, node_name_prefix=output) + + # get activation values + a_i_array = netapi.get_activations(ns_input_uid, input_) + a_h_array = netapi.get_activations(ns_hidden_uid, hidden) + a_o_array = netapi.get_activations(ns_output_uid, output) + + node.set_parameter('error', 0.0) # store error values to observe how training develops + + len_input = len(a_i_array) + len_hidden = len(a_h_array) + len_output = len(a_o_array) + + if len_input == 0: + netapi.logger.warn("Node net has no input nodes whose names start with '%s'", input_) + node.set_parameter('ctr', 0) + return + elif len_hidden == 0: + netapi.logger.warn("Node net has no hidden nodes whose names start with '%s'.", hidden) + node.set_parameter('ctr', 0) + return + elif len_output == 0: + netapi.logger.warn("Node net has no output names whose names start with '%s'.", output) + node.set_parameter('ctr', 0) + return + else: + netapi.logger.info("Initializing theano-based autoencoder backprop with layout: %i -> %i -> %i", + len_input, len_hidden, len_output) + + # get parameter values from node net + b_h_array = netapi.get_thetas(ns_hidden_uid, hidden) + b_o_array = netapi.get_thetas(ns_output_uid, output) + w_hi_array = netapi.get_link_weights(ns_input_uid, input_, ns_hidden_uid, hidden) + w_oh_array = netapi.get_link_weights(ns_hidden_uid, hidden, ns_output_uid, output) + + # declare shared variables ( shared b/w theano and node nets ) + a_i = node.a_i = theano.shared(value=a_i_array.astype(T.config.floatX), name="a_i", borrow=False) + a_h = node.a_h = theano.shared(value=a_h_array.astype(T.config.floatX), name="a_h", borrow=False) + a_o = node.a_o = theano.shared(value=a_o_array.astype(T.config.floatX), name="a_o", borrow=False) + b_h = node.b_h = theano.shared(value=b_h_array.astype(T.config.floatX), name="b_h", borrow=False) + b_o = node.b_o = theano.shared(value=b_o_array.astype(T.config.floatX), name="b_o", borrow=False) + w_hi = node.w_hi = theano.shared(value=w_hi_array.astype(T.config.floatX), name="w_hi", borrow=False) + w_oh = node.w_oh = theano.shared(value=w_oh_array.astype(T.config.floatX), name="w_oh", borrow=False) + + # write initial parameter values to shared variables + node.b_h.set_value(b_h_array, borrow=True) + node.b_o.set_value(b_o_array, borrow=True) + node.w_hi.set_value(w_hi_array, borrow=True) + node.w_oh.set_value(w_oh_array, borrow=True) + + # initialize accumulation variables for AdaDelta, ie. mean square gradients and mean square deltas + ms_grad_b_o = node.ms_grad_b_o = theano.shared(value=np.zeros_like(b_o_array), name="ms_grad_b_o", borrow=True) + ms_grad_w_oh = node.ms_grad_w_oh = theano.shared(value=np.zeros_like(w_oh_array), name="ms_grad_w_oh", borrow=True) + ms_grad_b_h = node.ms_grad_b_h = theano.shared(value=np.zeros_like(b_h_array), name="ms_grad_b_h", borrow=True) + ms_grad_w_hi = node.ms_grad_w_hi = theano.shared(value=np.zeros_like(w_hi_array), name="ms_grad_w_hi", borrow=True) + + ms_delta_b_o = node.ms_delta_b_o = theano.shared(value=np.zeros_like(b_o_array), name="ms_delta_b_o", borrow=True) + ms_delta_w_oh = node.ms_delta_w_oh = theano.shared(value=np.zeros_like(w_oh_array), name="ms_delta_w_oh", borrow=True) + ms_delta_b_h = node.ms_delta_b_h = theano.shared(value=np.zeros_like(b_h_array), name="ms_delta_b_h", borrow=True) + ms_delta_w_hi = node.ms_delta_w_hi = theano.shared(value=np.zeros_like(w_hi_array), name="ms_delta_w_hi", borrow=True) + + # make function parameters theano compatible + weight_decay = T.scalar("weight_decay", dtype=T.config.floatX) + sparsity_value = T.scalar("sparsity_value", dtype=T.config.floatX) + sparsity_penalty = T.scalar("sparsity_penalty", dtype=T.config.floatX) + ada_rho = T.scalar("ada_rho", dtype=T.config.floatX) + ada_eps = T.scalar("ada_eps", dtype=T.config.floatX) + + # declare the reconstruction error + error_term = T.sum(T.square(a_o - a_i)) / 2. # squared error + # error_term = -T.sum(a_i * T.log(a_o) + (1. - a_i) * T.log(1. - a_o)) # cross-entropy + + # use a weight constraint as a regularizer + weight_constraint = (weight_decay / 2.) * (T.sum(T.square(w_hi)) + T.sum(T.square(w_oh))) + + if sparse: # training criterion for a sparse autoencoder + + # save the average activation of hidden units; initialize to first activation received + avg_a_h = node.avg_a_h = theano.shared(value=a_h_array, name="avg_a_h", borrow=False) + new_avg_a_h = 0.95 * avg_a_h + (1 - 0.95) * a_h # for gradient checking, set new_avg_a_h = a_h + + rho = sparsity_value + information_gain = rho * T.log(rho / new_avg_a_h) + (1. - rho) * T.log((1. - rho) / (1. - new_avg_a_h)) + + sparsity_constraint = sparsity_penalty * T.sum(information_gain) + cost = error_term + weight_constraint + sparsity_constraint + + else: # training criterion for a denoising autoencoder + + cost = error_term + weight_constraint + + node.cost = theano.function([weight_decay, sparsity_value, sparsity_penalty], cost, on_unused_input='ignore') + node.error = theano.function([], error_term) + + # compute gradients + sigmoid_deriv_a_o = a_o * (1. - a_o) + grad_o = (a_o - a_i) * sigmoid_deriv_a_o # squared error # T.grad(cost, z_o) + # grad_o = ((a_i - a_o) / (a_o - a_o**2)) * sigmoid_deriv_a_o # cross-entropy + + sigmoid_deriv_a_h = a_h * (1. - a_h) + + if sparse: + + grad_w_oh = T.dot(T.reshape(grad_o, (len_input, 1)), T.reshape(a_h, (1, len_hidden))) + weight_decay * w_oh + grad_sparsity = (- rho / new_avg_a_h + (1. - rho) / (1. - new_avg_a_h)).T + grad_h = (T.dot(w_oh.T, grad_o) + sparsity_penalty * grad_sparsity) * sigmoid_deriv_a_h + grad_w_hi = T.dot(T.reshape(grad_h, (len_hidden, 1)), T.reshape(a_i, (1, len_input))) + weight_decay * w_hi + + else: # denoising + + grad_w_oh = T.dot(T.reshape(grad_o, (len_input, 1)), T.reshape(a_h, (1, len_hidden))) + weight_decay * w_oh + grad_h = T.dot(w_oh.T, grad_o) * sigmoid_deriv_a_h + grad_w_hi = T.dot(T.reshape(grad_h, (len_hidden, 1)), T.reshape(a_i, (1, len_input))) + weight_decay * w_hi + + if tied_weights: + grad_w_oh = grad_w_oh + grad_w_hi.T + gradients = [grad_o, grad_w_oh, grad_h] + ms_grad = [ms_grad_b_o, ms_grad_w_oh, ms_grad_b_h] + ms_delta = [ms_delta_b_o, ms_delta_w_oh, ms_delta_b_h] + else: + gradients = [grad_o, grad_w_oh, grad_h, grad_w_hi] + ms_grad = [ms_grad_b_o, ms_grad_w_oh, ms_grad_b_h, ms_grad_w_hi] + ms_delta = [ms_delta_b_o, ms_delta_w_oh, ms_delta_b_h, ms_delta_w_hi] + + # update accumulation variables for AdaDelta and compute new deltas + # compute an exponentially decaying average of squared gradients + # ie. recent gradients are more important and the quantity doesn't continue to grow + # thereby allowing the learning rate to grow or shrink as time progresses ( rather than just shrink as in AdaGrad ) + new_ms_grad = [ada_rho * ms_g + (1 - ada_rho) * (g**2) for ms_g, g in zip(ms_grad, gradients)] + # Note: the square root of the mean squared gradients plus epsilon is effectively the RMS of the gradients + # epsilon is added ~"to start off the first iteration and to ensure progress when previous updates become small" + deltas = [(T.sqrt(ms_d + ada_eps) / T.sqrt(ms_g + ada_eps)) * g for ms_d, ms_g, g in zip(ms_delta, new_ms_grad, gradients)] + # compute an exponentially decaying average of squared deltas -- this is to ensure correct units + new_ms_delta = [ada_rho * ms_d + (1 - ada_rho) * (d**2) for ms_d, d in zip(ms_delta, deltas)] + + # update parameters, ie. old_value - learning_rate * delta_value + if tied_weights: + new_b_o, new_w_oh, new_b_h = (old - update for old, update in zip([b_o, w_oh, b_h], deltas)) + new_w_hi = new_w_oh.T + new_ms_grad.append(new_ms_grad[1].T) + new_ms_delta.append(new_ms_delta[1].T) + gradients.append(gradients[1].T) + else: + new_b_o, new_w_oh, new_b_h, new_w_hi = (old - update for old, update in zip([b_o, w_oh, b_h, w_hi], deltas)) + + if sparse: + + update_function = theano.function([weight_decay, sparsity_value, sparsity_penalty, ada_rho, ada_eps], + None, + updates=[(b_o, new_b_o), + (w_oh, new_w_oh), + (b_h, new_b_h), + (w_hi, new_w_hi), + (avg_a_h, new_avg_a_h), + (ms_grad_b_o, new_ms_grad[0]), + (ms_grad_w_oh, new_ms_grad[1]), + (ms_grad_b_h, new_ms_grad[2]), + (ms_grad_w_hi, new_ms_grad[3]), + (ms_delta_b_o, new_ms_delta[0]), + (ms_delta_w_oh, new_ms_delta[1]), + (ms_delta_b_h, new_ms_delta[2]), + (ms_delta_w_hi, new_ms_delta[3])], + on_unused_input='ignore') + + else: # denoising + + update_function = theano.function([weight_decay, sparsity_value, sparsity_penalty, ada_rho, ada_eps], + None, + updates=[(b_o, new_b_o), + (w_oh, new_w_oh), + (b_h, new_b_h), + (w_hi, new_w_hi), + (ms_grad_b_o, new_ms_grad[0]), + (ms_grad_w_oh, new_ms_grad[1]), + (ms_grad_b_h, new_ms_grad[2]), + (ms_grad_w_hi, new_ms_grad[3]), + (ms_delta_b_o, new_ms_delta[0]), + (ms_delta_w_oh, new_ms_delta[1]), + (ms_delta_b_h, new_ms_delta[2]), + (ms_delta_w_hi, new_ms_delta[3])], + on_unused_input='ignore') + + node.get_updated_parameters = update_function + + # for gradient checking use the following function: + node.get_gradients = theano.function([weight_decay, sparsity_value, sparsity_penalty, ada_rho, ada_eps], + [gradients[0], gradients[1], gradients[2], gradients[3]], on_unused_input='ignore') + + node.initialized = True + + # get activations from node net + a_i_array = netapi.get_activations(ns_input_uid, input_) + a_h_array = netapi.get_activations(ns_hidden_uid, hidden) + a_o_array = netapi.get_activations(ns_output_uid, output) + + # learn only if activation on the input layer has been persistent for as many steps as your neural net has layers + # Note: since we're currently using denoising autoencoders, this means persistent up to Bernoulli noise + try: + # check if activation has changed since the last step ( by testing if there's any different activation value ) + bool_idx = node.prev_a_i != a_i_array + input_changed = np.any(bool_idx) + + # if deviating activations were 0 ( i.e most likely the effect of Bernoulli noising ), assume no change + is_zero = node.prev_a_i[bool_idx] == 0 + # if is_zero contains elements but not all input activations and their values are all zero, assume no change + if len(is_zero) and len(is_zero) < len(a_i_array) and np.all(is_zero): + input_changed = False + except: + input_changed = True + + node.prev_a_i = a_i_array + + if input_changed: + node.set_parameter('ctr', 1) + else: + node.set_parameter('ctr', int(node.get_parameter('ctr')) + 1) + + # until counter equals number of layers, ie. the same activation has reached all layers, don't compute + if node.get_parameter('ctr') < 3: + return + + # define learning parameters + param = node.get_parameter('weight_decay') + if param is None: + weight_decay = netapi.floatX(4e-06) # 0.0001 . 1e-07 assuming batches of size 1000 . 4e-06 assuming batches of size 256 + node.set_parameter('weight_decay', str(weight_decay)) # store as regular float to appease the serializer + else: + weight_decay = netapi.floatX(param) + + param = node.get_parameter('sparsity_value') + if param is None: + sparsity_value = netapi.floatX(0.05) + node.set_parameter('sparsity_value', str(sparsity_value)) + else: + sparsity_value = netapi.floatX(param) + + param = node.get_parameter('sparsity_penalty') + if param is None: + sparsity_penalty = netapi.floatX(0.001) # 3.0 . 0.003 assuming batches of size 1000 . 0.01 assuming batches of size 256 + node.set_parameter('sparsity_penalty', str(sparsity_penalty)) + else: + sparsity_penalty = netapi.floatX(param) + + param = node.get_parameter('adadelta_rho') + if param is None: + ada_rho = netapi.floatX(0.95) + node.set_parameter('adadelta_rho', str(ada_rho)) + else: + ada_rho = netapi.floatX(param) + + param = node.get_parameter('adadelta_eps') + if param is None: + ada_eps = netapi.floatX(1e-6) + node.set_parameter('adadelta_eps', str(ada_eps)) + else: + ada_eps = netapi.floatX(param) + + param = node.get_parameter('ae_type') + if param is None: + ae_type = 'sparse' # options: 'sparse', 'denoising' + node.set_parameter('ae_type', 'sparse') + else: + ae_type = str(param) + + param = node.get_parameter('t') + if param is None: + t = 0 + node.set_parameter('t', t) + else: + t = int(param) + + # gradient checking + # Note: use double precision when running gradient checks + if node.get_parameter('check_grad') == 'yes': + + # get values of biases and weights from node net + b_h_array = netapi.get_thetas(ns_hidden_uid, hidden) + b_o_array = netapi.get_thetas(ns_output_uid, output) + w_hi_array = netapi.get_link_weights(ns_input_uid, input_, ns_hidden_uid, hidden) + w_oh_array = netapi.get_link_weights(ns_hidden_uid, hidden, ns_output_uid, output) + + # compute the analytical gradient + anal_grad = compute_analytic_gradient( + netapi, node, a_i_array, a_h_array, a_o_array, b_h_array, b_o_array, w_hi_array, w_oh_array, + weight_decay, sparsity_value, sparsity_penalty, ada_rho, ada_eps) + + # compute the numerical gradient + num_grad = compute_numeric_gradient( + netapi, node, a_i_array, a_h_array, a_o_array, b_h_array, b_o_array, w_hi_array, w_oh_array, + weight_decay, sparsity_value, sparsity_penalty) + + # compare them + diff = np.linalg.norm(num_grad - anal_grad) / np.linalg.norm(num_grad + anal_grad) + print("Gradient difference: %e" % diff) # %.10f" % diff + print("The norm of the difference between numerical and analytical gradient should be < 1e-9\n") + + # write values to shared variables + node.a_i.set_value(a_i_array, borrow=True) + node.a_h.set_value(a_h_array, borrow=True) + node.a_o.set_value(a_o_array, borrow=True) + + # save current error as node parameter + node.set_parameter('error', float(node.error())) + # # print average reconstruction error + # node.set_parameter('error', node.get_parameter('error') + node.error()) + # if node.get_parameter('t') % 100 == 0: + # netapi.logger.debug("Number of backprop steps computed %d" % node.get_parameter('t')) + # netapi.logger.debug("Reconstruction error %f" % (node.get_parameter('error') / 100)) + # node.set_parameter('error', 0.0) + + # update values in shared variables ( using backpropgation of the gradients ) + node.get_updated_parameters(weight_decay, sparsity_value, sparsity_penalty, ada_rho, ada_eps) + + # write new parameter values to node net + netapi.set_thetas(ns_output_uid, output, node.b_o.get_value(borrow=True)) + netapi.set_link_weights(ns_hidden_uid, hidden, ns_output_uid, output, node.w_oh.get_value(borrow=True)) + netapi.set_thetas(ns_hidden_uid, hidden, node.b_h.get_value(borrow=True)) + netapi.set_link_weights(ns_input_uid, input_, ns_hidden_uid, hidden, node.w_hi.get_value(borrow=True)) + + # reset counter after successful backprop step; cf. must wait for new sensory activation to reach output layer + node.set_parameter('ctr', 0) + node.set_parameter('t', int(node.get_parameter('t')) + 1) + + +def sigmoid(z): + """ The sigmoid ( activation ) function. """ + return 1. / (1. + np.exp(-z)) + + +def compute_analytic_gradient(netapi, node, a_i, a_h, a_o, b_h, b_o, w_hi, w_oh, weight_decay, + sparsity_value, sparsity_penalty, ada_rho, ada_eps): + + # make sure borrow is False here because otherwise the buffers are overwritten and + # compute_numerical_gradient(..) still needs these same input values for proper comparison + node.a_i.set_value(a_i, borrow=False) + node.a_h.set_value(a_h, borrow=False) + node.a_o.set_value(a_o, borrow=False) + node.b_h.set_value(b_h, borrow=False) + node.b_o.set_value(b_o, borrow=False) + node.w_hi.set_value(w_hi, borrow=False) + node.w_oh.set_value(w_oh, borrow=False) + + delta_o, delta_w_oh, delta_h, delta_w_hi = \ + node.get_gradients(weight_decay, sparsity_value, sparsity_penalty, ada_rho, ada_eps) + + gradient = np.concatenate((delta_o, np.ravel(delta_w_oh), delta_h, np.ravel(delta_w_hi))) + + return gradient + + +def compute_numeric_gradient(netapi, node, a_i, a_h, a_o, b_h, b_o, w_hi, w_oh, weight_decay, sparsity_value, sparsity_penalty): + """ Compute numerical gradient for validating backprop implementation above. """ + + from copy import deepcopy + + # helper variables + epsilon = netapi.floatX(1e-4) + ni = len(b_o) + nh = len(b_h) + nih = ni * nh + + theta = np.concatenate((b_o, np.ravel(w_oh), b_h, np.ravel(w_hi))) + + n = theta.shape[0] + I = np.eye(n, dtype=netapi.floatX) + gradient = np.zeros(theta.shape, dtype=netapi.floatX) + + for i in range(n): + + eps_vec = np.array(I[:, i] * epsilon, dtype=netapi.floatX) + eps_plus = theta + eps_vec + eps_minus = theta - eps_vec + + # split theta into parts, recompute activations, update shared variables, compute cost + b_o_plus = eps_plus[: ni] + w_oh_plus = eps_plus[ni: ni + nih].reshape((ni, nh)) + b_h_plus = eps_plus[ni + nih: ni + nih + nh] + w_hi_plus = eps_plus[ni + nih + nh:].reshape((nh, ni)) + a_i_plus = deepcopy(a_i) + a_h_plus = np.ravel(sigmoid(w_hi_plus.dot(a_i_plus) + b_h_plus)) + a_o_plus = np.ravel(sigmoid(w_oh_plus.dot(a_h_plus) + b_o_plus)) + + node.a_i.set_value(a_i_plus, borrow=True) + node.a_h.set_value(a_h_plus, borrow=True) + node.a_o.set_value(a_o_plus, borrow=True) + node.b_h.set_value(b_h_plus, borrow=True) + node.b_o.set_value(b_o_plus, borrow=True) + node.w_hi.set_value(w_hi_plus, borrow=True) + node.w_oh.set_value(w_oh_plus, borrow=True) + + cost = node.cost(weight_decay, sparsity_value, sparsity_penalty) + + # split theta into parts, recompute activations, update shared variables, compute cost + b_o_minus = eps_minus[: ni] + w_oh_minus = eps_minus[ni: ni + nih].reshape((ni, nh)) + b_h_minus = eps_minus[ni + nih: ni + nih + nh] + w_hi_minus = eps_minus[ni + nih + nh:].reshape((nh, ni)) + a_i_minus = deepcopy(a_i) + a_h_minus = np.ravel(sigmoid(w_hi_minus.dot(a_i_minus) + b_h_minus)) + a_o_minus = np.ravel(sigmoid(w_oh_minus.dot(a_h_minus) + b_o_minus)) + + node.a_i.set_value(a_i_minus, borrow=True) + node.a_h.set_value(a_h_minus, borrow=True) + node.a_o.set_value(a_o_minus, borrow=True) + node.b_h.set_value(b_h_minus, borrow=True) + node.b_o.set_value(b_o_minus, borrow=True) + node.w_hi.set_value(w_hi_minus, borrow=True) + node.w_oh.set_value(w_oh_minus, borrow=True) + + cost_ = node.cost(weight_decay, sparsity_value, sparsity_penalty) + + # compute cost difference + gradient[i] = (cost - cost_) / (2. * epsilon) + + if i % 1000 == 0: + print("Computed numeric gradient for %d parameters" % i) + + return gradient diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 0d9fdfef..bd364bca 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1364,6 +1364,8 @@ def reload_native_modules(): native_modules = {} custom_recipes = {} runners = {} + from micropsi_core.nodenet.native_modules import nodetypes + native_modules.update(nodetypes) for uid in nodenets: if nodenets[uid].is_active: runners[uid] = True From ef49e0a8c859a1865f141a9667e8afd4f13e1f7e Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 11 Feb 2016 15:56:22 +0100 Subject: [PATCH 047/306] frontend: hierarchical dropdown for nativemodule creation --- .../static/css/micropsi-styles.css | 2 +- micropsi_server/static/js/nodenet.js | 49 +++++++++++++++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/micropsi_server/static/css/micropsi-styles.css b/micropsi_server/static/css/micropsi-styles.css index 2164adfa..af1b453f 100644 --- a/micropsi_server/static/css/micropsi-styles.css +++ b/micropsi_server/static/css/micropsi-styles.css @@ -274,7 +274,7 @@ graphs margin-top: -30px; } -.dropdown-menu li:hover .sub-menu { +.dropdown-menu li:hover > .sub-menu { visibility: visible; display: block; } diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index 89e42d3f..006bea48 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -110,6 +110,7 @@ selectionBox.name = "selectionBox"; nodetypes = {}; native_modules = {}; +native_module_categories = {}; available_gatetypes = []; nodespaces = {}; sorted_nodetypes = []; @@ -288,9 +289,27 @@ function setCurrentNodenet(uid, nodespace, changed){ if(a > b) return 1; return 0; }); + + function buildTreeRecursive(item, path, idx){ + if (idx < path.length){ + name = path[idx]; + if (!item[name]){ + item[name] = {}; + } + buildTreeRecursive(item[name], path, idx + 1); + } + } + + categories = []; for(var key in native_modules){ nodetypes[key] = native_modules[key]; + categories.push(native_modules[key].category.split('/')); + } + native_module_categories = {} + for(var i =0; i < categories.length; i++){ + buildTreeRecursive(native_module_categories, categories[i], 0); } + available_gatetypes = []; for(var key in nodetypes){ $.merge(available_gatetypes, nodetypes[key].gatetypes || []); @@ -2552,6 +2571,32 @@ function initializeDialogs(){ var clickPosition = null; +function buildNativeModuleDropdown(cat, html, current_category){ + if(!current_category){ + current_category=''; + } + var catentries = [] + for(var key in cat){ + catentries.push(key); + } + for(var i = 0; i < catentries.length; i++){ + var newcategory = current_category || ''; + if(current_category == '') newcategory += catentries[i] + else newcategory += '/'+catentries[i]; + html += '
  • '+catentries[i]+''; + html += '
      ' + html += buildNativeModuleDropdown(cat[catentries[i]], '', newcategory); + html += '
    • '; + } + for(var idx in sorted_native_modules){ + key = sorted_native_modules[idx]; + if(native_modules[key].category == current_category){ + html += '
  • '+ native_modules[key].name +'
    • '; + } + } + return html +} + function openContextMenu(menu_id, event) { event.cancelBubble = true; if(!currentNodenet){ @@ -2573,9 +2618,7 @@ function openContextMenu(menu_id, event) { if(Object.keys(native_modules).length){ html += '
  • Create Native Module'; html += '
    • '; } html += '
  • Autoalign Nodes
    • '; From 98a6251f317864dfc8862dc6466c04f7a475e4ee Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 11 Feb 2016 15:56:48 +0100 Subject: [PATCH 048/306] frontend: typeahead for recipe selection --- micropsi_server/static/js/dialogs.js | 16 ++++++++-------- micropsi_server/view/dialogs.tpl | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/micropsi_server/static/js/dialogs.js b/micropsi_server/static/js/dialogs.js index da3335d5..aea70605 100644 --- a/micropsi_server/static/js/dialogs.js +++ b/micropsi_server/static/js/dialogs.js @@ -524,12 +524,10 @@ $(function() { $('#recipe_modal .docstring').show(); $('#recipe_modal .docstring').html(recipes[name].docstring); $('#recipe_modal .btn-primary').show(); - $('#recipe_modal form').show(); } else { $('#recipe_modal .default_explanation').show(); $('#recipe_modal .docstring').hide(); $('#recipe_modal .btn-primary').hide(); - $('#recipe_modal form').hide(); } if(name in recipes){ var html = ''; @@ -621,13 +619,15 @@ $(function() { $('#recipe_modal button').prop('disabled', false); api.call('get_available_recipes', {}, function(data){ recipes = data; - var options = ''; - var items = Object.values(data); - var sorted = items.sort(sortByName); - for(var idx in sorted){ - options += ''; + var options = []; + for(var key in data){ + options.push(data[key].name); } - recipe_name_input.html(options); + recipe_name_input.typeahead({'source': options, highlighter: function(item){ + var search = recipe_name_input.val(); + var text = item.replace(search, ""+search+""); + return ''+text+' ('+recipes[item].category+')'; + }}); recipe_name_input.focus(); update_parameters_for_recipe(); }); diff --git a/micropsi_server/view/dialogs.tpl b/micropsi_server/view/dialogs.tpl index 68daa322..f74ccde9 100644 --- a/micropsi_server/view/dialogs.tpl +++ b/micropsi_server/view/dialogs.tpl @@ -120,7 +120,7 @@
    - +
    From bf814da4072ab5d61d315700f90d72f1e934884d Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Fri, 12 Feb 2016 13:14:30 +0100 Subject: [PATCH 049/306] Better set_link_weights performance --- .../nodenet/theano_engine/theano_nodenet.py | 12 ++++--- .../nodenet/theano_engine/theano_partition.py | 31 +++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 6f15b697..3b861165 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -1433,12 +1433,14 @@ def set_link_weights(self, nodespace_from_uid, group_from, nodespace_to_uid, gro else: partition_from.set_link_weights(nodespace_from_uid, group_from, nodespace_to_uid, group_to, new_w) - uids_to_invalidate = self.get_node_uids(nodespace_from_uid, group_from) - uids_to_invalidate.extend(self.get_node_uids(nodespace_to_uid, group_to)) + self.proxycache.clear() - for uid in uids_to_invalidate: - if uid in self.proxycache: - del self.proxycache[uid] + #uids_to_invalidate = self.get_node_uids(nodespace_from_uid, group_from) + #uids_to_invalidate.extend(self.get_node_uids(nodespace_to_uid, group_to)) + + #for uid in uids_to_invalidate: + # if uid in self.proxycache: + # del self.proxycache[uid] def get_available_gatefunctions(self): return ["identity", "absolute", "sigmoid", "tanh", "rect", "one_over_x"] diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index 91331995..1bad7554 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -1771,14 +1771,14 @@ def get_link_weights(self, nodespace_from_uid, group_from, nodespace_to_uid, gro return w_matrix[rows,cols] def set_link_weights(self, nodespace_from_uid, group_from, nodespace_to_uid, group_to, new_w): - if nodespace_from_uid not in self.nodegroups or group_from not in self.nodegroups[nodespace_from_uid]: - raise ValueError("Group %s does not exist in nodespace %s." % (group_from, nodespace_from_uid)) - if nodespace_to_uid not in self.nodegroups or group_to not in self.nodegroups[nodespace_to_uid]: - raise ValueError("Group %s does not exist in nodespace %s." % (group_to, nodespace_to_uid)) - if len(self.nodegroups[nodespace_from_uid][group_from]) != new_w.shape[1]: - raise ValueError("group_from %s has length %i, but new_w.shape[1] is %i" % (group_from, len(self.nodegroups[nodespace_from_uid][group_from]), new_w.shape[1])) - if len(self.nodegroups[nodespace_to_uid][group_to]) != new_w.shape[0]: - raise ValueError("group_to %s has length %i, but new_w.shape[0] is %i" % (group_to, len(self.nodegroups[nodespace_to_uid][group_to]), new_w.shape[0])) + #if nodespace_from_uid not in self.nodegroups or group_from not in self.nodegroups[nodespace_from_uid]: + # raise ValueError("Group %s does not exist in nodespace %s." % (group_from, nodespace_from_uid)) + #if nodespace_to_uid not in self.nodegroups or group_to not in self.nodegroups[nodespace_to_uid]: + # raise ValueError("Group %s does not exist in nodespace %s." % (group_to, nodespace_to_uid)) + #if len(self.nodegroups[nodespace_from_uid][group_from]) != new_w.shape[1]: + # raise ValueError("group_from %s has length %i, but new_w.shape[1] is %i" % (group_from, len(self.nodegroups[nodespace_from_uid][group_from]), new_w.shape[1])) + #if len(self.nodegroups[nodespace_to_uid][group_to]) != new_w.shape[0]: + # raise ValueError("group_to %s has length %i, but new_w.shape[0] is %i" % (group_to, len(self.nodegroups[nodespace_to_uid][group_to]), new_w.shape[0])) w_matrix = self.w.get_value(borrow=True) grp_from = self.nodegroups[nodespace_from_uid][group_from] @@ -1787,16 +1787,13 @@ def set_link_weights(self, nodespace_from_uid, group_from, nodespace_to_uid, gro w_matrix[rows, cols] = new_w self.w.set_value(w_matrix, borrow=True) - for id in self.allocated_elements_to_nodes[grp_from]: - self.nodes_last_changed[id] = self.nodenet.current_step - self.nodespaces_contents_last_changed[self.allocated_node_parents[id]] = self.nodenet.current_step - for id in self.allocated_elements_to_nodes[grp_to]: - self.nodes_last_changed[id] = self.nodenet.current_step - self.nodespaces_contents_last_changed[self.allocated_node_parents[id]] = self.nodenet.current_step + cstep = self.nodenet.current_step + self.nodes_last_changed[self.allocated_elements_to_nodes[grp_from]] = cstep + self.nodespaces_contents_last_changed[self.allocated_node_parents[self.allocated_elements_to_nodes[grp_from]]] = cstep + self.nodes_last_changed[self.allocated_elements_to_nodes[grp_to]] = cstep + self.nodespaces_contents_last_changed[self.allocated_node_parents[self.allocated_elements_to_nodes[grp_to]]] = cstep - # todo: only set this if one of the groups is por/ret relevant - if self.has_pipes: - self.por_ret_dirty = True + self.por_ret_dirty = self.has_pipes def set_inlink_weights(self, partition_from_spid, new_from_elements, new_to_elements, new_weights): if partition_from_spid in self.inlinks: From 1dc25bb25824191f478b28296b8861e3cdd67ec6 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Fri, 12 Feb 2016 13:39:38 +0100 Subject: [PATCH 050/306] Faster parameter querying for native modules --- micropsi_core/nodenet/theano_engine/theano_node.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index 23d5f9b9..958eba02 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -235,7 +235,10 @@ def get_associated_node_uids(self): return ids def get_parameter(self, parameter): - return self.clone_parameters().get(parameter, None) + if self.type in self._nodenet.native_modules: + return self.parameters.get(parameter, self.nodetype.parameter_defaults.get(parameter, None)) + else: + return self.clone_parameters().get(parameter, None) def set_parameter(self, parameter, value): if value == '' or value is None: From 42570a3f67cb0f87ed8a63b95f241c2de6de70cd Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 12 Feb 2016 19:04:35 +0100 Subject: [PATCH 051/306] adjust the tests --- conftest.py | 68 ++++------------- .../tests/test_runtime_nodenet_basics.py | 27 ++++--- micropsi_core/tests/test_runtime_nodes.py | 75 ++++++++++++------- micropsi_core/tests/test_vizapi.py | 9 ++- micropsi_server/tests/test_json_api.py | 56 +++++++++----- 5 files changed, 125 insertions(+), 110 deletions(-) diff --git a/conftest.py b/conftest.py index ec3e6998..0948cda4 100644 --- a/conftest.py +++ b/conftest.py @@ -36,10 +36,6 @@ world_uid = 'WorldOfPain' nn_uid = 'Testnet' -nodetype_file = os.path.join(config['paths']['resource_path'], 'nodetypes.json') -nodefunc_file = os.path.join(config['paths']['resource_path'], 'nodefunctions.py') -recipes_file = os.path.join(config['paths']['resource_path'], 'recipes.py') - try: import theano @@ -82,25 +78,19 @@ def pytest_runtest_setup(item): engine_marker = engine_marker.args[0] if engine_marker != item.callspec.params['engine']: pytest.skip("test requires engine %s" % engine_marker) + for uid in list(micropsi.nodenets.keys()): + micropsi.delete_nodenet(uid) + for uid in list(micropsi.worlds.keys()): + micropsi.delete_world(uid) + shutil.rmtree(config['paths']['resource_path']) + os.mkdir(config['paths']['resource_path']) + os.mkdir(os.path.join(config['paths']['resource_path'], 'nodenets')) + os.mkdir(os.path.join(config['paths']['resource_path'], 'worlds')) + # default native module container + os.mkdir(os.path.join(config['paths']['resource_path'], 'Test')) + micropsi.reload_native_modules() micropsi.logger.clear_logs() - - -def pytest_runtest_teardown(item, nextitem): - if nextitem is None: - print("DELETING ALL STUFF") - shutil.rmtree(config['paths']['resource_path']) - else: - uids = list(micropsi.nodenets.keys()) - for uid in uids: - micropsi.delete_nodenet(uid) - if os.path.isfile(nodetype_file): - os.remove(nodetype_file) - if os.path.isfile(nodefunc_file): - os.remove(nodefunc_file) - if os.path.isfile(recipes_file): - os.remove(recipes_file) - micropsi.reload_native_modules() - set_logging_levels() + set_logging_levels() @pytest.fixture(scope="session") @@ -108,46 +98,18 @@ def resourcepath(): return config['paths']['resource_path'] -@pytest.fixture() -def nodetype_def(): - return nodetype_file - - -@pytest.fixture -def nodefunc_def(): - return nodefunc_file - - -@pytest.fixture -def recipes_def(): - return recipes_file - - @pytest.fixture(scope="function") def test_world(request): global world_uid - worlds = micropsi.get_available_worlds("Pytest User") - if world_uid not in worlds: - success, world_uid = micropsi.new_world("World of Pain", "Island", "Pytest User", uid=world_uid) - - def fin(): - try: - micropsi.delete_world(world_uid) - except: - pass # world was deleted in test - request.addfinalizer(fin) + success, world_uid = micropsi.new_world("World of Pain", "Island", "Pytest User", uid=world_uid) return world_uid @pytest.fixture(scope="function") def test_nodenet(request, test_world, engine): global nn_uid - nodenets = micropsi.get_available_nodenets("Pytest User") or {} - if nn_uid not in nodenets: - success, nn_uid = micropsi.new_nodenet("Testnet", engine=engine, owner="Pytest User", uid='Testnet') - micropsi.save_nodenet(nn_uid) - if nn_uid not in micropsi.nodenets: - micropsi.load_nodenet(nn_uid) + success, nn_uid = micropsi.new_nodenet("Testnet", engine=engine, owner="Pytest User", uid='Testnet') + micropsi.save_nodenet(nn_uid) return nn_uid diff --git a/micropsi_core/tests/test_runtime_nodenet_basics.py b/micropsi_core/tests/test_runtime_nodenet_basics.py index dcc227a4..ca49b894 100644 --- a/micropsi_core/tests/test_runtime_nodenet_basics.py +++ b/micropsi_core/tests/test_runtime_nodenet_basics.py @@ -49,8 +49,11 @@ def test_nodenet_data_gate_parameters(fixed_nodenet): assert data == {'gen': {'threshold': 1}} -def test_user_prompt(fixed_nodenet, nodetype_def, nodefunc_def): - with open(nodetype_def, 'w') as fp: +def test_user_prompt(fixed_nodenet, resourcepath): + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ @@ -61,7 +64,7 @@ def test_user_prompt(fixed_nodenet, nodetype_def, nodefunc_def): "testparam": 13\ }\ }}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") micropsi.reload_native_modules() @@ -241,8 +244,11 @@ def test_modulators(fixed_nodenet): assert nodenet.netapi.get_modulator("test_modulator") == -1 -def test_node_parameters(fixed_nodenet, nodetype_def, nodefunc_def): - with open(nodetype_def, 'w') as fp: +def test_node_parameters(fixed_nodenet, resourcepath): + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ @@ -258,7 +264,7 @@ def test_node_parameters(fixed_nodenet, nodetype_def, nodefunc_def): "protocol_mode": "all_active"\ }}\ }') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") assert micropsi.reload_native_modules() @@ -294,15 +300,18 @@ def test_delete_linked_nodes(fixed_nodenet): netapi.delete_node(evil_two) -def test_multiple_nodenet_interference(engine, nodetype_def, nodefunc_def): - with open(nodetype_def, 'w') as fp: +def test_multiple_nodenet_interference(engine, resourcepath): + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ "gatetypes": ["gen", "foo", "bar"],\ "nodefunction_name": "testnodefunc"\ }}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n node.get_gate('gen').gate_function(17)") micropsi.reload_native_modules() diff --git a/micropsi_core/tests/test_runtime_nodes.py b/micropsi_core/tests/test_runtime_nodes.py index bea5c2c3..7fa75617 100644 --- a/micropsi_core/tests/test_runtime_nodes.py +++ b/micropsi_core/tests/test_runtime_nodes.py @@ -171,10 +171,13 @@ def hashlink(l): # performance reasons, changed defaults will only affect newly created nodes. # This test will have to be replaced when the generic solution proposed in TOL-90 has been # implemented. -def test_gate_defaults_change_with_nodetype(fixed_nodenet, resourcepath, nodetype_def, nodefunc_def): +def test_gate_defaults_change_with_nodetype(fixed_nodenet, resourcepath,): # gate_parameters are a property of the nodetype, and should change with # the nodetype definition if not explicitly overwritten for a given node - with open(nodetype_def, 'w') as fp: + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ @@ -186,11 +189,11 @@ def test_gate_defaults_change_with_nodetype(fixed_nodenet, resourcepath, nodetyp "amplification": 13\ }\ }}}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") micropsi.reload_native_modules() res, uid = micropsi.add_node(fixed_nodenet, "Testnode", [10, 10], name="Testnode") - with open(nodetype_def, 'w') as fp: + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ @@ -233,9 +236,11 @@ def test_ignore_links(test_nodenet): assert 'links' not in nodespace["nodes"][nodes['a']] -def test_remove_and_reload_native_module(fixed_nodenet, resourcepath, nodetype_def, nodefunc_def): - from os import remove - with open(nodetype_def, 'w') as fp: +def test_remove_and_reload_native_module(fixed_nodenet, resourcepath): + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ @@ -247,20 +252,23 @@ def test_remove_and_reload_native_module(fixed_nodenet, resourcepath, nodetype_d "amplification": 13\ }\ }}}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") micropsi.reload_native_modules() res, uid = micropsi.add_node(fixed_nodenet, "Testnode", [10, 10, 10], name="Testnode") - remove(nodetype_def) - remove(nodefunc_def) + os.remove(nodetype_file) + os.remove(nodefunc_file) micropsi.reload_native_modules() - assert micropsi.get_available_native_module_types(fixed_nodenet) == {} + assert 'Testnode' not in micropsi.get_available_native_module_types(fixed_nodenet) @pytest.mark.engine("dict_engine") -def test_engine_specific_nodetype_dict(fixed_nodenet, resourcepath, nodetype_def, nodefunc_def): - with open(nodetype_def, 'w') as fp: +def test_engine_specific_nodetype_dict(fixed_nodenet, resourcepath): + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "engine": "theano_engine",\ "name": "Testnode",\ @@ -273,7 +281,7 @@ def test_engine_specific_nodetype_dict(fixed_nodenet, resourcepath, nodetype_def "amplification": 13\ }\ }}}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") micropsi.reload_native_modules() @@ -282,8 +290,11 @@ def test_engine_specific_nodetype_dict(fixed_nodenet, resourcepath, nodetype_def @pytest.mark.engine("theano_engine") -def test_engine_specific_nodetype_theano(fixed_nodenet, resourcepath, nodetype_def, nodefunc_def): - with open(nodetype_def, 'w') as fp: +def test_engine_specific_nodetype_theano(fixed_nodenet, resourcepath): + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "engine": "dict_engine",\ "name": "Testnode",\ @@ -296,7 +307,7 @@ def test_engine_specific_nodetype_theano(fixed_nodenet, resourcepath, nodetype_d "amplification": 13\ }\ }}}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") micropsi.reload_native_modules() @@ -313,8 +324,10 @@ def test_node_parameters_none_resets_to_default(fixed_nodenet): assert node.get_parameter('wait') == 0 -def test_get_recipes(fixed_nodenet, resourcepath, recipes_def): - with open(recipes_def, 'w') as fp: +def test_get_recipes(fixed_nodenet, resourcepath): + import os + recipe_file = os.path.join(resourcepath, 'Test', 'recipes.py') + with open(recipe_file, 'w') as fp: fp.write(""" def testfoo(netapi, count=23): return {'count':count} @@ -327,8 +340,10 @@ def testfoo(netapi, count=23): assert recipes['testfoo']['parameters'][0]['default'] == 23 -def test_run_recipe(fixed_nodenet, resourcepath, recipes_def): - with open(recipes_def, 'w') as fp: +def test_run_recipe(fixed_nodenet, resourcepath): + import os + recipe_file = os.path.join(resourcepath, 'Test', 'recipes.py') + with open(recipe_file, 'w') as fp: fp.write(""" def testfoo(netapi, count=23): return {'count':count} @@ -339,8 +354,11 @@ def testfoo(netapi, count=23): assert result['count'] == 42 -def test_node_parameter_defaults(fixed_nodenet, resourcepath, nodetype_def, nodefunc_def): - with open(nodetype_def, 'w') as fp: +def test_node_parameter_defaults(fixed_nodenet, resourcepath): + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ @@ -351,7 +369,7 @@ def test_node_parameter_defaults(fixed_nodenet, resourcepath, nodetype_def, node "testparam": 13\ }\ }}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") micropsi.reload_native_modules() @@ -361,8 +379,11 @@ def test_node_parameter_defaults(fixed_nodenet, resourcepath, nodetype_def, node assert node.get_parameter("testparam") == 13 -def test_node_parameters_from_persistence(fixed_nodenet, resourcepath, nodetype_def, nodefunc_def): - with open(nodetype_def, 'w') as fp: +def test_node_parameters_from_persistence(fixed_nodenet, resourcepath): + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ @@ -373,7 +394,7 @@ def test_node_parameters_from_persistence(fixed_nodenet, resourcepath, nodetype_ "testparam": 13\ }\ }}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") micropsi.reload_native_modules() res, uid = micropsi.add_node(fixed_nodenet, "Testnode", [10, 10, 10], name="Test") diff --git a/micropsi_core/tests/test_vizapi.py b/micropsi_core/tests/test_vizapi.py index 674c3e74..55ae1e1f 100644 --- a/micropsi_core/tests/test_vizapi.py +++ b/micropsi_core/tests/test_vizapi.py @@ -47,7 +47,7 @@ def test_save_file(test_nodenet, resourcepath): assert os.path.isfile(filepath) -def test_plot_from_nodefunc(test_nodenet, resourcepath, nodetype_def, nodefunc_def): +def test_plot_from_nodefunc(test_nodenet, resourcepath): import os from random import random from time import sleep @@ -61,14 +61,17 @@ def test_plot_from_nodefunc(test_nodenet, resourcepath, nodetype_def, nodefunc_d assert os.path.abspath(returnpath) == os.path.abspath(filepath) assert os.path.isfile(filepath) os.remove(filepath) - with open(nodetype_def, 'w') as fp: + os.mkdir(os.path.join(resourcepath, 'plotter')) + nodetype_file = os.path.join(resourcepath, "plotter", "nodetypes.json") + nodefunc_file = os.path.join(resourcepath, "plotter", "nodefunctions.py") + with open(nodetype_file, 'w') as fp: fp.write("""{"Plotter": { "name": "Plotter", "slottypes": [], "nodefunction_name": "plotfunc", "gatetypes": [], "parameters": ["plotpath"]}}""") - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write(""" def plotfunc(netapi, node=None, **params): import os diff --git a/micropsi_server/tests/test_json_api.py b/micropsi_server/tests/test_json_api.py index e6e3396e..148d4aec 100644 --- a/micropsi_server/tests/test_json_api.py +++ b/micropsi_server/tests/test_json_api.py @@ -77,10 +77,14 @@ def test_set_nodenet_properties(app, test_nodenet, test_world): assert data['worldadapter'] == 'Braitenberg' -def test_set_node_state(app, test_nodenet, nodetype_def, nodefunc_def): +def test_set_node_state(app, test_nodenet, resourcepath): + import os app.set_auth() # create a native module: - with open(nodetype_def, 'w') as fp: + + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ @@ -88,7 +92,7 @@ def test_set_node_state(app, test_nodenet, nodetype_def, nodefunc_def): "gatetypes": ["gen", "foo", "bar"],\ "symbol": "t"}}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") response = app.get_json('/rpc/reload_native_modules()') @@ -886,10 +890,13 @@ def test_get_available_node_types(app, test_nodenet): assert 'Sensor' in response.json_body['data'] -def test_get_available_native_module_types(app, test_nodenet): +def test_get_available_native_module_types(app, test_nodenet, engine): response = app.get_json('/rpc/get_available_native_module_types(nodenet_uid="%s")' % test_nodenet) assert_success(response) - assert response.json_body['data'] == {} + if engine == 'dict_engine': + assert response.json_body['data'] == {} + elif engine == 'theano_engine': + assert "GradientDescent" in response.json_body['data'] def test_set_node_parameters(app, test_nodenet): @@ -1077,17 +1084,20 @@ def test_delete_link(app, test_nodenet, node): data['nodes'][node]['links'] == {} -def test_reload_native_modules(app, test_nodenet, nodetype_def, nodefunc_def): +def test_reload_native_modules(app, test_nodenet, resourcepath): app.set_auth() # create a native module: - with open(nodetype_def, 'w') as fp: + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ "nodefunction_name": "testnodefunc",\ "gatetypes": ["gen", "foo", "bar"],\ "symbol": "t"}}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") response = app.get_json('/rpc/reload_native_modules()') assert_success(response) @@ -1099,17 +1109,20 @@ def test_reload_native_modules(app, test_nodenet, nodetype_def, nodefunc_def): assert data['name'] == 'Testnode' -def test_user_prompt_response(app, test_nodenet, nodetype_def, nodefunc_def): +def test_user_prompt_response(app, test_nodenet, resourcepath): app.set_auth() # create a native module: - with open(nodetype_def, 'w') as fp: + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ "nodefunction_name": "testnodefunc",\ "gatetypes": ["gen", "foo", "bar"],\ "symbol": "t"}}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") response = app.get_json('/rpc/reload_native_modules()') assert_success(response) @@ -1217,9 +1230,11 @@ def test_500(app): assert response.json_body['traceback'] is not None -def test_get_recipes(app, test_nodenet, recipes_def): +def test_get_recipes(app, test_nodenet, resourcepath): app.set_auth() - with open(recipes_def, 'w') as fp: + import os + recipe_file = os.path.join(resourcepath, 'Test', 'recipes.py') + with open(recipe_file, 'w') as fp: fp.write(""" def foobar(netapi, quatsch=23): return {'quatsch': quatsch} @@ -1233,9 +1248,11 @@ def foobar(netapi, quatsch=23): assert data['foobar']['parameters'][0]['default'] == 23 -def test_run_recipes(app, test_nodenet, recipes_def): +def test_run_recipes(app, test_nodenet, resourcepath): app.set_auth() - with open(recipes_def, 'w') as fp: + import os + recipe_file = os.path.join(resourcepath, 'Test', 'recipes.py') + with open(recipe_file, 'w') as fp: fp.write(""" def foobar(netapi, quatsch=23): return {'quatsch': quatsch} @@ -1258,16 +1275,19 @@ def test_get_agent_dashboard(app, test_nodenet, node): assert data['count_nodes'] == 1 -def test_nodenet_data_structure(app, test_nodenet, nodetype_def, nodefunc_def, node): +def test_nodenet_data_structure(app, test_nodenet, resourcepath, node): app.set_auth() - with open(nodetype_def, 'w') as fp: + import os + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + with open(nodetype_file, 'w') as fp: fp.write('{"Testnode": {\ "name": "Testnode",\ "slottypes": ["gen", "foo", "bar"],\ "nodefunction_name": "testnodefunc",\ "gatetypes": ["gen", "foo", "bar"],\ "symbol": "t"}}') - with open(nodefunc_def, 'w') as fp: + with open(nodefunc_file, 'w') as fp: fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") response = app.get_json('/rpc/reload_native_modules()') response = app.post_json('/rpc/add_nodespace', params={ From 9aa9aff7ca751d25b8789ff0aba7528f890d1fbb Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 12 Feb 2016 19:04:51 +0100 Subject: [PATCH 052/306] fix reloading --- micropsi_core/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index bd364bca..a7017177 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1276,7 +1276,7 @@ def load_user_files(path=RESOURCE_PATH, reload_nodefunctions=False): if not f.startswith('.') and f != '__pycache__': abspath = os.path.join(path, f) if os.path.isdir(abspath): - load_user_files(abspath) + load_user_files(path=abspath, reload_nodefunctions=reload_nodefunctions) elif f == 'nodetypes.json': parse_native_module_file(abspath) elif f == 'recipes.py': From 4e5484fe4ff0d8a872490f1e2503aae37895961f Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 12 Feb 2016 19:13:30 +0100 Subject: [PATCH 053/306] port changes from nodenet-repo --- micropsi_core/nodenet/native_modules.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/micropsi_core/nodenet/native_modules.py b/micropsi_core/nodenet/native_modules.py index 91ab0cd3..89dd58f1 100644 --- a/micropsi_core/nodenet/native_modules.py +++ b/micropsi_core/nodenet/native_modules.py @@ -767,14 +767,23 @@ def gradient_descent(netapi, node=None, **params): import theano.tensor as T # get shared name prefix of nodes in input, hidden, and output layers - input_ = node.get_parameter('input_layer') - hidden = node.get_parameter('hidden_layer') - output = node.get_parameter('output_layer') - - # get nodespaces, default to parent nodespace - ns_input_uid = params.get("input_nodespace") or node.parent_nodespace - ns_hidden_uid = params.get("hidden_nodespace") or node.parent_nodespace - ns_output_uid = params.get("output_nodespace") or node.parent_nodespace + input_ = node.get_parameter('input_prefix') + hidden = node.get_parameter('hidden_prefix') + output = node.get_parameter('output_prefix') + + # get the name of the nodespace where the input lives + ns_input_name = node.get_parameter('input_nodespace') + + # get nodespace uids of nodes in input, hidden, and output layers + # assumption: if the input layer consists of sensor nodes, they have their + # own nodespace, all other nodes are in this node's nodespace + ns_input_uid = None + for ns in netapi.get_nodespaces(): + if ns.name == ns_input_name: + ns_input_uid = ns.uid + break + ns_hidden_uid = node.parent_nodespace + ns_output_uid = node.parent_nodespace # initialization if not hasattr(node, 'initialized'): From 486188672dd4bbd364881199594f0293d72aab89 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 12 Feb 2016 19:55:42 +0100 Subject: [PATCH 054/306] test for the categories of nativemodules and recipes --- micropsi_core/tests/test_runtime_nodes.py | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/micropsi_core/tests/test_runtime_nodes.py b/micropsi_core/tests/test_runtime_nodes.py index 7fa75617..77b03deb 100644 --- a/micropsi_core/tests/test_runtime_nodes.py +++ b/micropsi_core/tests/test_runtime_nodes.py @@ -166,6 +166,30 @@ def hashlink(l): assert links_before == links_after +def test_native_module_and_recipe_categories(fixed_nodenet, resourcepath): + import os + os.mkdir(os.path.join(resourcepath, 'Test', 'Test2')) + nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') + nodefunc_file = os.path.join(resourcepath, 'Test', 'nodefunctions.py') + recipe_file = os.path.join(resourcepath, 'Test', 'Test2', 'recipes.py') + with open(nodetype_file, 'w') as fp: + fp.write('{"Testnode": {\ + "name": "Testnode",\ + "slottypes": ["gen", "foo", "bar"],\ + "nodefunction_name": "testnodefunc",\ + "gatetypes": ["gen", "foo", "bar"]\ + }}') + with open(nodefunc_file, 'w') as fp: + fp.write("def testnodefunc(netapi, node=None, **prams):\r\n return 17") + with open(recipe_file, 'w') as fp: + fp.write("def testrecipe(netapi):\r\n pass") + micropsi.reload_native_modules() + res = micropsi.get_available_native_module_types(fixed_nodenet) + assert res['Testnode']['category'] == 'Test' + res = micropsi.get_available_recipes() + assert res['testrecipe']['category'] == 'Test/Test2' + + @pytest.mark.engine("dict_engine") # This behavior is not available in theano_engine: Default inheritance at runtime is not implemented for # performance reasons, changed defaults will only affect newly created nodes. From 137d3850b5faf39805938b24d2a7bccfa32bfdfa Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 15 Feb 2016 15:16:11 +0100 Subject: [PATCH 055/306] fix defaulting engine to dict if no engine given (no engine given means most likely, this nodenet has been created when there was no theano engine. thus default missing values to dict) --- micropsi_core/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 6d8dbb29..7c92bf98 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -283,7 +283,7 @@ def load_nodenet(nodenet_uid): else: logging.getLogger("system").warn("World %s for nodenet %s not found" % (data.world, data.uid)) - engine = data.get('engine', 'dict_engine') + engine = data.get('engine') or 'dict_engine' logger.register_logger("agent.%s" % nodenet_uid, cfg['logging']['level_agent']) From 1316f2f746aba7e34e5edeeebb777af9bcd04f65 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 15 Feb 2016 15:17:20 +0100 Subject: [PATCH 056/306] fix netapi code generation --- micropsi_core/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 7c92bf98..8012053e 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -927,7 +927,7 @@ def generate_netapi_fragment(nodenet_uid, node_uids): # positions origin = [100, 100, 100] factor = [int(min(xpos)), int(min(ypos)), int(min(zpos))] - lines.append("origin_pos = (%d, %d)" % origin) + lines.append("origin_pos = (%d, %d)" % (origin[0], origin[1])) for node in nodes + nodespaces: x = int(node.position[0] - factor[0]) y = int(node.position[1] - factor[1]) From 13f4f75da0a92da936ac250333123d1b5f814a2a Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 15 Feb 2016 17:38:37 +0100 Subject: [PATCH 057/306] fix netapi_fragment part2 --- micropsi_core/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 8012053e..e237f3c0 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -927,7 +927,7 @@ def generate_netapi_fragment(nodenet_uid, node_uids): # positions origin = [100, 100, 100] factor = [int(min(xpos)), int(min(ypos)), int(min(zpos))] - lines.append("origin_pos = (%d, %d)" % (origin[0], origin[1])) + lines.append("origin_pos = (%d, %d, %d)" % (origin[0], origin[1], origin[2])) for node in nodes + nodespaces: x = int(node.position[0] - factor[0]) y = int(node.position[1] - factor[1]) From f107a9ef2bccc27a545affde53f47e17a8cb2488 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 15 Feb 2016 18:25:27 +0100 Subject: [PATCH 058/306] missing changes from `a28008ea` --- micropsi_core/nodenet/native_modules.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/micropsi_core/nodenet/native_modules.py b/micropsi_core/nodenet/native_modules.py index 89dd58f1..9397010a 100644 --- a/micropsi_core/nodenet/native_modules.py +++ b/micropsi_core/nodenet/native_modules.py @@ -92,10 +92,20 @@ "category": "backward_pass", "path": os.path.abspath(__file__), "parameters": [ - "input_nodesace", "hidden_nodespace", "output_nodespace" - "ae_type", "learning_rate", "sparsity_value", "sparsity_penalty", - "weight_decay", "adadelta_rho", "adadelta_eps", "t", "tied_weights", - "ctr", "check_grad", "input_layer", "hidden_layer", "output_layer" + "ae_type", + "adadelta_rho", + "adadelta_eps", + "check_grad", + "weight_decay", + "tied_weights", + "sparsity_value", + "sparsity_penalty", + "t", + "ctr", + "input_prefix", + "hidden_prefix", + "output_prefix", + "input_nodespace" ], "parameter_values": { "ae_type": ["sparse", "denoising"], @@ -105,9 +115,8 @@ "parameter_defaults": { "ae_type": "denoising", "tied_weights": "True", - "input_layer": "fov__", - "hidden_layer": "hidden_1", - "output_layer": "output_1" + "hidden_prefix": "hidden_1", + "output_prefix": "output_1" } } From 55a1c431b8a47e166b3a9ca447e77931865b767f Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 16 Feb 2016 11:47:46 +0100 Subject: [PATCH 059/306] rename category --- micropsi_core/nodenet/native_modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micropsi_core/nodenet/native_modules.py b/micropsi_core/nodenet/native_modules.py index 9397010a..2782093e 100644 --- a/micropsi_core/nodenet/native_modules.py +++ b/micropsi_core/nodenet/native_modules.py @@ -28,7 +28,7 @@ "gatetypes": ["e"], "nodefunction_name": "gradient_descent_lstm", "symbol": "↺", - "category": "backward_pass", + "category": "nn_learning", "path": os.path.abspath(__file__), "parameters": [ "adadelta_rho", @@ -89,7 +89,7 @@ "gatetypes": ["gen"], "nodefunction_name": "gradient_descent", "symbol": "☲", - "category": "backward_pass", + "category": "nn_learning", "path": os.path.abspath(__file__), "parameters": [ "ae_type", From 60ba01a07fe9dedadc5b8403df58e114a4e1cdd3 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 16 Feb 2016 12:42:33 +0100 Subject: [PATCH 060/306] sort native module categories --- micropsi_server/static/js/nodenet.js | 1 + 1 file changed, 1 insertion(+) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index 006bea48..25f5798d 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -2579,6 +2579,7 @@ function buildNativeModuleDropdown(cat, html, current_category){ for(var key in cat){ catentries.push(key); } + catentries.sort(); for(var i = 0; i < catentries.length; i++){ var newcategory = current_category || ''; if(current_category == '') newcategory += catentries[i] From 3cb62527894000fb12d5f3f3908d4cfaacf64294 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 16 Feb 2016 14:49:00 +0100 Subject: [PATCH 061/306] allow for omitting slottypes and gatetypes from native module definition --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 50a13b15..6adc7542 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -1046,7 +1046,7 @@ def reload_native_modules(self, native_modules): numeric_id = node_from_id(uid) number_of_elements = len(np.where(partition.allocated_elements_to_nodes == numeric_id)[0]) - new_numer_of_elements = max(len(native_modules[instance.type]['slottypes']), len(native_modules[instance.type]['gatetypes'])) + new_numer_of_elements = max(len(native_modules[instance.type].get('slottypes', [])), len(native_modules[instance.type].get('gatetypes', []))) if number_of_elements != new_numer_of_elements: self.logger.warn("Number of elements changed for node type %s from %d to %d, recreating instance %s" % (instance.type, number_of_elements, new_numer_of_elements, uid)) From 3b555a8be45e4a62ef8fb3f3af40765ec8a2daa9 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Tue, 16 Feb 2016 15:55:05 +0100 Subject: [PATCH 062/306] TimeseriesWorld now using configurable worlds facilities --- micropsi_core/world/timeseries/timeseries.py | 80 ++++++++++++++++---- 1 file changed, 64 insertions(+), 16 deletions(-) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 804d05c4..44259362 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -1,7 +1,7 @@ """ Worlds and bodies for agents whose habitats are ordered sequences of vectors. """ -import os.path +import os from configuration import config as cfg from micropsi_core.world.world import World from micropsi_core.world.worldadapter import WorldAdapter, ArrayWorldAdapter @@ -24,22 +24,34 @@ class TimeSeries(World): def __init__(self, filename, world_type="Island", name="", owner="", engine=None, uid=None, version=1, config={}): World.__init__(self, filename, world_type=world_type, name=name, owner=owner, uid=uid, version=version, config=config) - path = os.path.join(cfg['micropsi2']['data_directory'], 'timeseries.npz') - print("loading timeseries from", path, "for world", uid) - with np.load(path) as f: - self.timeseries = f['data'] - self.ids = f['ids'] - self.startdate = f['startdate'] - self.enddate = f['enddate'] + filename = config.get('time_series_data_file', "timeseries.npz") + if os.path.isabs(filename): + path = filename + else: + path = os.path.join(cfg['micropsi2']['data_directory'], filename) + self.logger.info("loading timeseries from %s for world %s" % (path, uid)) + + try: + with np.load(path) as f: + self.timeseries = f['data'] + self.ids = f['ids'] + self.startdate = f['startdate'] + self.enddate = f['enddate'] + except IOError as error: + self.logger.error("Could not load data file %s, error was: %s" % (path, str(error))) + return # todo use the new configurable world options. - dummydata = False # present the same random pattern in each step. - z_transform = True # for each ID, center on mean & normalize by standard deviation - clip_and_scale = False # for each ID, center on mean & clip to 4 standard deviations and rescale to [0,1]. - sigmoid = True # for each ID, z-transform and apply a sigmoid activation function - self.shuffle = True # randomize order of presentation - assert(not (clip_and_scale and sigmoid)) + dummydata = config('dummydata') == "True" + z_transform = config('z_transform') == "True" + clip_and_scale = config('clip_and_scale') == "True" + sigmoid = config('sigmoid') == "True" + self.shuffle = config('shuffle') == "True" + + if clip_and_scale and sigmoid: + self.logger.warn("clip_and_scale and sigmoid cannot both be configured, choosing sigmoid") + clip_and_scale = False def sigm(X): """ sigmoid that avoids float overflows for very small inputs. @@ -49,7 +61,6 @@ def sigm(X): X[np.nan_to_num(X) <= -cutoff] = -cutoff return 1. / (1. + np.exp(-X)) - if (z_transform or clip_and_scale or sigmoid) and not dummydata: data_z = np.empty_like(self.timeseries) data_z[:] = np.nan @@ -70,7 +81,7 @@ def sigm(X): self.timeseries = data_z if not sigmoid else sigm(data_z) if dummydata: - print("! Using dummy data") + self.logger.warn("! Using dummy data") n_ids = self.timeseries.shape[0] self.timeseries = np.tile(np.random.rand(n_ids,1),(1,10)) @@ -88,6 +99,43 @@ def state(self): t = self.permutation[t] return self.timeseries[:, t] + @staticmethod + def get_config_options(): + """ Returns a list of configuration-options for this world. + Expected format: + [{ + 'name': 'param1', + 'description': 'this is just an example', + 'options': ['value1', 'value2'], + 'default': 'value1' + }] + description, options and default are optional settings + """ + return [ + {'name': 'time_series_data_file', + 'description': 'The data file with the time series', + 'default': 'timeseries.npz'}, + {'name': 'shuffle', + 'description': 'Randomize order of presentation', + 'default': 'True', + 'options': ["True", "False"]}, + {'name': 'z_transform', + 'description': 'For each ID, center on mean & normalize by standard deviation', + 'default': 'True', + 'options': ["True", "False"]}, + {'name': 'clip_and_scale', + 'description': 'For each ID, center on mean & clip to 4 standard deviations and rescale to [0,1].', + 'default': 'False', + 'options': ["True", "False"]}, + {'name': 'sigmoid', + 'description': 'For each ID, z-transform and apply a sigmoid activation function', + 'default': 'True', + 'options': ["True", "False"]}, + {'name': 'dummy_data', + 'description': 'Present the same random pattern in each step (instead of the actual time series data)', + 'default': 'False', + 'options': ["True", "False"]} + ] class TimeSeriesRunner(ArrayWorldAdapter): From bd0d93bb4afb439518647ffd2f46677f02c46406 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 16 Feb 2016 16:53:20 +0100 Subject: [PATCH 063/306] better error reporting for malformed nodenet-data files --- micropsi_core/runtime.py | 75 +++++++++++++++++----------- micropsi_server/static/js/dialogs.js | 26 +++++++--- 2 files changed, 65 insertions(+), 36 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index d286919f..1eec4f0a 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1270,36 +1270,40 @@ def init_worlds(world_data): return worlds -def load_user_files(path=RESOURCE_PATH, reload_nodefunctions=False): +def load_user_files(path=RESOURCE_PATH, reload_nodefunctions=False, errors=[]): global native_modules, custom_recipes for f in os.listdir(path): if not f.startswith('.') and f != '__pycache__': abspath = os.path.join(path, f) + err = None if os.path.isdir(abspath): - load_user_files(path=abspath, reload_nodefunctions=reload_nodefunctions) + errors.extend(load_user_files(path=abspath, reload_nodefunctions=reload_nodefunctions, errors=[])) elif f == 'nodetypes.json': - parse_native_module_file(abspath) + err = parse_native_module_file(abspath) elif f == 'recipes.py': - parse_recipe_file(abspath, reload_nodefunctions) + err = parse_recipe_file(abspath, reload_nodefunctions) elif f == 'nodefunctions.py' and reload_nodefunctions: - reload_nodefunctions_file(abspath) + err = reload_nodefunctions_file(abspath) + if err: + errors.append(err) + return errors def parse_native_module_file(path): global native_modules - try: - with open(path) as fp: + with open(path) as fp: + category = os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) + try: modules = json.load(fp) - category = os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) - for key in modules: - modules[key]['path'] = os.path.join(os.path.dirname(path), 'nodefunctions.py') - modules[key]['category'] = category - if key in native_modules: - logging.getLogger("system").warn("Native module names must be unique. %s is not." % key) - native_modules[key] = modules[key] - sys.path.append(path) - except ValueError: - logging.getLogger('system').warn("Nodetype data in %s not well-formed." % path) + except ValueError: + return "Nodetype data in %s/nodetypes.json not well-formed." % category + for key in modules: + modules[key]['path'] = os.path.join(os.path.dirname(path), 'nodefunctions.py') + modules[key]['category'] = category + if key in native_modules: + logging.getLogger("system").warning("Native module names must be unique. %s is not." % key) + native_modules[key] = modules[key] + sys.path.append(path) def parse_recipe_file(path, reload=False): @@ -1311,8 +1315,11 @@ def parse_recipe_file(path, reload=False): relpath = os.path.relpath(path, start=RESOURCE_PATH) pyname = relpath.replace(os.path.sep, '.')[:-3] - recipes = __import__(pyname, fromlist=['recipes']) - importlib.reload(sys.modules[pyname]) + try: + recipes = __import__(pyname, fromlist=['recipes']) + importlib.reload(sys.modules[pyname]) + except SyntaxError as e: + return "%s in recipe file %s, line %d" % (e.__class__.__name__, relpath, e.lineno) for name, module in inspect.getmembers(recipes, inspect.ismodule): if module.__file__.startswith(RESOURCE_PATH): @@ -1335,7 +1342,7 @@ def parse_recipe_file(path, reload=False): 'default': default }) if name in custom_recipes: - logging.getLogger("system").warn("Recipe function names must be unique. %s is not." % name) + logging.getLogger("system").warning("Recipe function names must be unique. %s is not." % name) custom_recipes[name] = { 'name': name, 'parameters': params, @@ -1350,12 +1357,16 @@ def reload_nodefunctions_file(path): import importlib import inspect - loader = importlib.machinery.SourceFileLoader("nodefunctions", path) - nodefuncs = loader.load_module() - for name, module in inspect.getmembers(nodefuncs, inspect.ismodule): - if module.__file__.startswith(RESOURCE_PATH): - loader = importlib.machinery.SourceFileLoader(name, module.__file__) - loader.load_module() + try: + loader = importlib.machinery.SourceFileLoader("nodefunctions", path) + nodefuncs = loader.load_module() + for name, module in inspect.getmembers(nodefuncs, inspect.ismodule): + if module.__file__.startswith(RESOURCE_PATH): + loader = importlib.machinery.SourceFileLoader(name, module.__file__) + loader.load_module() + except SyntaxError as e: + relpath = os.path.relpath(path, start=RESOURCE_PATH) + return "%s in nodefunction file %s, line %d" % (e.__class__.__name__, relpath, e.lineno) def reload_native_modules(): @@ -1370,20 +1381,26 @@ def reload_native_modules(): if nodenets[uid].is_active: runners[uid] = True nodenets[uid].is_active = False - load_user_files(reload_nodefunctions=True) + errors = load_user_files(reload_nodefunctions=True, errors=[]) for nodenet_uid in nodenets: nodenets[nodenet_uid].reload_native_modules(filter_native_modules(nodenets[nodenet_uid].engine)) # restart previously active nodenets for uid in runners: nodenets[uid].is_active = True - return True + + if len(errors) == 0: + return True, [] + else: + return False, errors native_modules = {} custom_recipes = {} load_definitions() init_worlds(world_data) -reload_native_modules() +result, errors = reload_native_modules() +for e in errors: + logging.getLogger("system").error(e) # initialize runners # Initialize the threads for the continuous simulation of nodenets and worlds diff --git a/micropsi_server/static/js/dialogs.js b/micropsi_server/static/js/dialogs.js index aea70605..8d159303 100644 --- a/micropsi_server/static/js/dialogs.js +++ b/micropsi_server/static/js/dialogs.js @@ -173,14 +173,26 @@ var api = { if(data.status == 0){ msg = "Server not reachable."; } else { - try{ - error = JSON.parse(data.responseText); - var errtext = $('
    ').text(error.data).html(); - msg += '' + errtext + ''; - if(error.traceback){ - msg += '

    '+error.traceback+'

    '; + if(data.responseText){ + try{ + error = JSON.parse(data.responseText); + var errtext = $('
    ').text(error.data).html(); + msg += '' + errtext + ''; + if(error.traceback){ + msg += '

    '+error.traceback+'

    '; + } + } catch (err){} + } else if(data.data) { + if(typeof data.data == 'object'){ + msg = '
      '; + for(var i in data.data){ + msg += '
    • '+data.data[i]+'
      • '; + } + msg += '
    '; + } else { + msg = data } - } catch (err){} + } if(!msg){ msg = type || "serverside exception"; } From 58d26f6849c4d589cb2a43dcc22d393d30af62a8 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 17 Feb 2016 15:49:09 +0100 Subject: [PATCH 064/306] combobox for recipe selection --- micropsi_server/static/css/chosen-sprite.png | Bin 0 -> 538 bytes .../static/css/chosen-sprite@2x.png | Bin 0 -> 738 bytes micropsi_server/static/css/chosen.min.css | 3 ++ .../static/js/chosen.jquery.min.js | 2 + micropsi_server/static/js/chosen.min.css | 3 ++ micropsi_server/static/js/dialogs.js | 35 +++++++++++++----- micropsi_server/view/boilerplate.tpl | 2 + micropsi_server/view/dialogs.tpl | 5 ++- 8 files changed, 38 insertions(+), 12 deletions(-) create mode 100644 micropsi_server/static/css/chosen-sprite.png create mode 100644 micropsi_server/static/css/chosen-sprite@2x.png create mode 100755 micropsi_server/static/css/chosen.min.css create mode 100755 micropsi_server/static/js/chosen.jquery.min.js create mode 100755 micropsi_server/static/js/chosen.min.css diff --git a/micropsi_server/static/css/chosen-sprite.png b/micropsi_server/static/css/chosen-sprite.png new file mode 100644 index 0000000000000000000000000000000000000000..c57da70b4b5b1e08a6977ddde182677af0e5e1b8 GIT binary patch literal 538 zcmV+#0_FXQP)cz2)-WJLkv8J@4bb5L`rsE?Kc|FrXHkKz)ov z76MHYM&Apx%05P7orE!>9=yZ~6O0^V?1%{=1UASqa<2Pgnk7fs!OIs9gh{NCN+@=) z>Gfttd5uq;oeR{%NHjtqV~jEQeY?tDff=(jqx>~SZ_e+iN26HR*`0Q!Re)~HD85p> zbL()Mw}bI^#`7wp0+cv&7*LhrtOmR)?PK>(-BeLm#jL5Jfogv-QS(TBnUb;))Krqm zD}uDDeVLNhm1G*pFB`O?iA=dnWBEpqHk8Yh%Qu45EIG=&F-dDmt|;|nN@|3lOkVZ7>z*~a1?_t?U)c+&|JFJke1`&0-a z#PjhRlg?=$KTo4|rU@NyV_fzDy@>h!lVyKShsO8>V>$xyIXRbHK%H~^Aaz=s$Jz^V zlb?KfaKdZqu3^#m$mintvgJ15@j`sb2Zr%69Sn=xN01Tm5r)NQanT=jhwm7zqj2>O cEB}D~0$b-QdD7|v=>Px#07*qoM6N<$g6AXnUH||9 literal 0 HcmV?d00001 diff --git a/micropsi_server/static/css/chosen-sprite@2x.png b/micropsi_server/static/css/chosen-sprite@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..6b50545202cb4770039362c55025b0b9824663ad GIT binary patch literal 738 zcmV<80v-K{P)oJoIWh{eAG@xkM<0ryd(K3(} zP8JV&;uuIJ4nL%g8!wSG9E$P+3QVMGgj><+00}M5I5kMzaT<~M;uJ`UhLfbp9Ahdsrux5(g+(>Q*+9wU{AuYPH0}W_u4`|q(9c->{ zt>Jn|lbhH<_x5jU6prFi#S}&XMZ=~Y5VyC3+ZN%hXciz8 zPcLpJgbIK#a49e31-%wf2zh2F&&(Nq;AL%4zA(=QJRGq`sx3y3#0_cg9Fim739XTOu1NKKjlWs`52Q+3 Uja*K~(*OVf07*qoM6N<$g3mu-GXMYp literal 0 HcmV?d00001 diff --git a/micropsi_server/static/css/chosen.min.css b/micropsi_server/static/css/chosen.min.css new file mode 100755 index 00000000..9484b570 --- /dev/null +++ b/micropsi_server/static/css/chosen.min.css @@ -0,0 +1,3 @@ +/* Chosen v1.5.0 | (c) 2011-2016 by Harvest | MIT License, https://github.com/harvesthq/chosen/blob/master/LICENSE.md */ + +.chosen-container{position:relative;display:inline-block;vertical-align:middle;font-size:13px;-webkit-user-select:none;-moz-user-select:none;user-select:none}.chosen-container *{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.chosen-container .chosen-drop{position:absolute;top:100%;left:-9999px;z-index:1010;width:100%;border:1px solid #aaa;border-top:0;background:#fff;box-shadow:0 4px 5px rgba(0,0,0,.15)}.chosen-container.chosen-with-drop .chosen-drop{left:0}.chosen-container a{cursor:pointer}.chosen-container .search-choice .group-name,.chosen-container .chosen-single .group-name{margin-right:4px;overflow:hidden;white-space:nowrap;text-overflow:ellipsis;font-weight:400;color:#999}.chosen-container .search-choice .group-name:after,.chosen-container .chosen-single .group-name:after{content:":";padding-left:2px;vertical-align:top}.chosen-container-single .chosen-single{position:relative;display:block;overflow:hidden;padding:0 0 0 8px;height:25px;border:1px solid #aaa;border-radius:5px;background-color:#fff;background:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#fff),color-stop(50%,#f6f6f6),color-stop(52%,#eee),color-stop(100%,#f4f4f4));background:-webkit-linear-gradient(#fff 20%,#f6f6f6 50%,#eee 52%,#f4f4f4 100%);background:-moz-linear-gradient(#fff 20%,#f6f6f6 50%,#eee 52%,#f4f4f4 100%);background:-o-linear-gradient(#fff 20%,#f6f6f6 50%,#eee 52%,#f4f4f4 100%);background:linear-gradient(#fff 20%,#f6f6f6 50%,#eee 52%,#f4f4f4 100%);background-clip:padding-box;box-shadow:0 0 3px #fff inset,0 1px 1px rgba(0,0,0,.1);color:#444;text-decoration:none;white-space:nowrap;line-height:24px}.chosen-container-single .chosen-default{color:#999}.chosen-container-single .chosen-single span{display:block;overflow:hidden;margin-right:26px;text-overflow:ellipsis;white-space:nowrap}.chosen-container-single .chosen-single-with-deselect span{margin-right:38px}.chosen-container-single .chosen-single abbr{position:absolute;top:6px;right:26px;display:block;width:12px;height:12px;background:url(chosen-sprite.png) -42px 1px no-repeat;font-size:1px}.chosen-container-single .chosen-single abbr:hover{background-position:-42px -10px}.chosen-container-single.chosen-disabled .chosen-single abbr:hover{background-position:-42px -10px}.chosen-container-single .chosen-single div{position:absolute;top:0;right:0;display:block;width:18px;height:100%}.chosen-container-single .chosen-single div b{display:block;width:100%;height:100%;background:url(chosen-sprite.png) no-repeat 0 2px}.chosen-container-single .chosen-search{position:relative;z-index:1010;margin:0;padding:3px 4px;white-space:nowrap}.chosen-container-single .chosen-search input[type=text]{margin:1px 0;padding:4px 20px 4px 5px;width:100%;height:auto;outline:0;border:1px solid #aaa;background:#fff url(chosen-sprite.png) no-repeat 100% -20px;background:url(chosen-sprite.png) no-repeat 100% -20px;font-size:1em;font-family:sans-serif;line-height:normal;border-radius:0}.chosen-container-single .chosen-drop{margin-top:-1px;border-radius:0 0 4px 4px;background-clip:padding-box}.chosen-container-single.chosen-container-single-nosearch .chosen-search{position:absolute;left:-9999px}.chosen-container .chosen-results{color:#444;position:relative;overflow-x:hidden;overflow-y:auto;margin:0 4px 4px 0;padding:0 0 0 4px;max-height:240px;-webkit-overflow-scrolling:touch}.chosen-container .chosen-results li{display:none;margin:0;padding:5px 6px;list-style:none;line-height:15px;word-wrap:break-word;-webkit-touch-callout:none}.chosen-container .chosen-results li.active-result{display:list-item;cursor:pointer}.chosen-container .chosen-results li.disabled-result{display:list-item;color:#ccc;cursor:default}.chosen-container .chosen-results li.highlighted{background-color:#3875d7;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#3875d7),color-stop(90%,#2a62bc));background-image:-webkit-linear-gradient(#3875d7 20%,#2a62bc 90%);background-image:-moz-linear-gradient(#3875d7 20%,#2a62bc 90%);background-image:-o-linear-gradient(#3875d7 20%,#2a62bc 90%);background-image:linear-gradient(#3875d7 20%,#2a62bc 90%);color:#fff}.chosen-container .chosen-results li.no-results{color:#777;display:list-item;background:#f4f4f4}.chosen-container .chosen-results li.group-result{display:list-item;font-weight:700;cursor:default}.chosen-container .chosen-results li.group-option{padding-left:15px}.chosen-container .chosen-results li em{font-style:normal;text-decoration:underline}.chosen-container-multi .chosen-choices{position:relative;overflow:hidden;margin:0;padding:0 5px;width:100%;height:auto!important;height:1%;border:1px solid #aaa;background-color:#fff;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(1%,#eee),color-stop(15%,#fff));background-image:-webkit-linear-gradient(#eee 1%,#fff 15%);background-image:-moz-linear-gradient(#eee 1%,#fff 15%);background-image:-o-linear-gradient(#eee 1%,#fff 15%);background-image:linear-gradient(#eee 1%,#fff 15%);cursor:text}.chosen-container-multi .chosen-choices li{float:left;list-style:none}.chosen-container-multi .chosen-choices li.search-field{margin:0;padding:0;white-space:nowrap}.chosen-container-multi .chosen-choices li.search-field input[type=text]{margin:1px 0;padding:0;height:25px;outline:0;border:0!important;background:transparent!important;box-shadow:none;color:#999;font-size:100%;font-family:sans-serif;line-height:normal;border-radius:0}.chosen-container-multi .chosen-choices li.search-choice{position:relative;margin:3px 5px 3px 0;padding:3px 20px 3px 5px;border:1px solid #aaa;max-width:100%;border-radius:3px;background-color:#eee;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#f4f4f4),color-stop(50%,#f0f0f0),color-stop(52%,#e8e8e8),color-stop(100%,#eee));background-image:-webkit-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:-moz-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:-o-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-size:100% 19px;background-repeat:repeat-x;background-clip:padding-box;box-shadow:0 0 2px #fff inset,0 1px 0 rgba(0,0,0,.05);color:#333;line-height:13px;cursor:default}.chosen-container-multi .chosen-choices li.search-choice span{word-wrap:break-word}.chosen-container-multi .chosen-choices li.search-choice .search-choice-close{position:absolute;top:4px;right:3px;display:block;width:12px;height:12px;background:url(chosen-sprite.png) -42px 1px no-repeat;font-size:1px}.chosen-container-multi .chosen-choices li.search-choice .search-choice-close:hover{background-position:-42px -10px}.chosen-container-multi .chosen-choices li.search-choice-disabled{padding-right:5px;border:1px solid #ccc;background-color:#e4e4e4;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#f4f4f4),color-stop(50%,#f0f0f0),color-stop(52%,#e8e8e8),color-stop(100%,#eee));background-image:-webkit-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:-moz-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:-o-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);color:#666}.chosen-container-multi .chosen-choices li.search-choice-focus{background:#d4d4d4}.chosen-container-multi .chosen-choices li.search-choice-focus .search-choice-close{background-position:-42px -10px}.chosen-container-multi .chosen-results{margin:0;padding:0}.chosen-container-multi .chosen-drop .result-selected{display:list-item;color:#ccc;cursor:default}.chosen-container-active .chosen-single{border:1px solid #5897fb;box-shadow:0 0 5px rgba(0,0,0,.3)}.chosen-container-active.chosen-with-drop .chosen-single{border:1px solid #aaa;-moz-border-radius-bottomright:0;border-bottom-right-radius:0;-moz-border-radius-bottomleft:0;border-bottom-left-radius:0;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#eee),color-stop(80%,#fff));background-image:-webkit-linear-gradient(#eee 20%,#fff 80%);background-image:-moz-linear-gradient(#eee 20%,#fff 80%);background-image:-o-linear-gradient(#eee 20%,#fff 80%);background-image:linear-gradient(#eee 20%,#fff 80%);box-shadow:0 1px 0 #fff inset}.chosen-container-active.chosen-with-drop .chosen-single div{border-left:0;background:transparent}.chosen-container-active.chosen-with-drop .chosen-single div b{background-position:-18px 2px}.chosen-container-active .chosen-choices{border:1px solid #5897fb;box-shadow:0 0 5px rgba(0,0,0,.3)}.chosen-container-active .chosen-choices li.search-field input[type=text]{color:#222!important}.chosen-disabled{opacity:.5!important;cursor:default}.chosen-disabled .chosen-single{cursor:default}.chosen-disabled .chosen-choices .search-choice .search-choice-close{cursor:default}.chosen-rtl{text-align:right}.chosen-rtl .chosen-single{overflow:visible;padding:0 8px 0 0}.chosen-rtl .chosen-single span{margin-right:0;margin-left:26px;direction:rtl}.chosen-rtl .chosen-single-with-deselect span{margin-left:38px}.chosen-rtl .chosen-single div{right:auto;left:3px}.chosen-rtl .chosen-single abbr{right:auto;left:26px}.chosen-rtl .chosen-choices li{float:right}.chosen-rtl .chosen-choices li.search-field input[type=text]{direction:rtl}.chosen-rtl .chosen-choices li.search-choice{margin:3px 5px 3px 0;padding:3px 5px 3px 19px}.chosen-rtl .chosen-choices li.search-choice .search-choice-close{right:auto;left:4px}.chosen-rtl.chosen-container-single-nosearch .chosen-search,.chosen-rtl .chosen-drop{left:9999px}.chosen-rtl.chosen-container-single .chosen-results{margin:0 0 4px 4px;padding:0 4px 0 0}.chosen-rtl .chosen-results li.group-option{padding-right:15px;padding-left:0}.chosen-rtl.chosen-container-active.chosen-with-drop .chosen-single div{border-right:0}.chosen-rtl .chosen-search input[type=text]{padding:4px 5px 4px 20px;background:#fff url(chosen-sprite.png) no-repeat -30px -20px;background:url(chosen-sprite.png) no-repeat -30px -20px;direction:rtl}.chosen-rtl.chosen-container-single .chosen-single div b{background-position:6px 2px}.chosen-rtl.chosen-container-single.chosen-with-drop .chosen-single div b{background-position:-12px 2px}@media only screen and (-webkit-min-device-pixel-ratio:1.5),only screen and (min-resolution:144dpi),only screen and (min-resolution:1.5dppx){.chosen-rtl .chosen-search input[type=text],.chosen-container-single .chosen-single abbr,.chosen-container-single .chosen-single div b,.chosen-container-single .chosen-search input[type=text],.chosen-container-multi .chosen-choices .search-choice .search-choice-close,.chosen-container .chosen-results-scroll-down span,.chosen-container .chosen-results-scroll-up span{background-image:url(chosen-sprite@2x.png)!important;background-size:52px 37px!important;background-repeat:no-repeat!important}} \ No newline at end of file diff --git a/micropsi_server/static/js/chosen.jquery.min.js b/micropsi_server/static/js/chosen.jquery.min.js new file mode 100755 index 00000000..caf5ce7a --- /dev/null +++ b/micropsi_server/static/js/chosen.jquery.min.js @@ -0,0 +1,2 @@ +/* Chosen v1.5.0 | (c) 2011-2016 by Harvest | MIT License, https://github.com/harvesthq/chosen/blob/master/LICENSE.md */ +(function(){var a,AbstractChosen,Chosen,SelectParser,b,c={}.hasOwnProperty,d=function(a,b){function d(){this.constructor=a}for(var e in b)c.call(b,e)&&(a[e]=b[e]);return d.prototype=b.prototype,a.prototype=new d,a.__super__=b.prototype,a};SelectParser=function(){function SelectParser(){this.options_index=0,this.parsed=[]}return SelectParser.prototype.add_node=function(a){return"OPTGROUP"===a.nodeName.toUpperCase()?this.add_group(a):this.add_option(a)},SelectParser.prototype.add_group=function(a){var b,c,d,e,f,g;for(b=this.parsed.length,this.parsed.push({array_index:b,group:!0,label:this.escapeExpression(a.label),title:a.title?a.title:void 0,children:0,disabled:a.disabled,classes:a.className}),f=a.childNodes,g=[],d=0,e=f.length;e>d;d++)c=f[d],g.push(this.add_option(c,b,a.disabled));return g},SelectParser.prototype.add_option=function(a,b,c){return"OPTION"===a.nodeName.toUpperCase()?(""!==a.text?(null!=b&&(this.parsed[b].children+=1),this.parsed.push({array_index:this.parsed.length,options_index:this.options_index,value:a.value,text:a.text,html:a.innerHTML,title:a.title?a.title:void 0,selected:a.selected,disabled:c===!0?c:a.disabled,group_array_index:b,group_label:null!=b?this.parsed[b].label:null,classes:a.className,style:a.style.cssText})):this.parsed.push({array_index:this.parsed.length,options_index:this.options_index,empty:!0}),this.options_index+=1):void 0},SelectParser.prototype.escapeExpression=function(a){var b,c;return null==a||a===!1?"":/[\&\<\>\"\'\`]/.test(a)?(b={"<":"<",">":">",'"':""","'":"'","`":"`"},c=/&(?!\w+;)|[\<\>\"\'\`]/g,a.replace(c,function(a){return b[a]||"&"})):a},SelectParser}(),SelectParser.select_to_array=function(a){var b,c,d,e,f;for(c=new SelectParser,f=a.childNodes,d=0,e=f.length;e>d;d++)b=f[d],c.add_node(b);return c.parsed},AbstractChosen=function(){function AbstractChosen(a,b){this.form_field=a,this.options=null!=b?b:{},AbstractChosen.browser_is_supported()&&(this.is_multiple=this.form_field.multiple,this.set_default_text(),this.set_default_values(),this.setup(),this.set_up_html(),this.register_observers(),this.on_ready())}return AbstractChosen.prototype.set_default_values=function(){var a=this;return this.click_test_action=function(b){return a.test_active_click(b)},this.activate_action=function(b){return a.activate_field(b)},this.active_field=!1,this.mouse_on_container=!1,this.results_showing=!1,this.result_highlighted=null,this.allow_single_deselect=null!=this.options.allow_single_deselect&&null!=this.form_field.options[0]&&""===this.form_field.options[0].text?this.options.allow_single_deselect:!1,this.disable_search_threshold=this.options.disable_search_threshold||0,this.disable_search=this.options.disable_search||!1,this.enable_split_word_search=null!=this.options.enable_split_word_search?this.options.enable_split_word_search:!0,this.group_search=null!=this.options.group_search?this.options.group_search:!0,this.search_contains=this.options.search_contains||!1,this.single_backstroke_delete=null!=this.options.single_backstroke_delete?this.options.single_backstroke_delete:!0,this.max_selected_options=this.options.max_selected_options||1/0,this.inherit_select_classes=this.options.inherit_select_classes||!1,this.display_selected_options=null!=this.options.display_selected_options?this.options.display_selected_options:!0,this.display_disabled_options=null!=this.options.display_disabled_options?this.options.display_disabled_options:!0,this.include_group_label_in_selected=this.options.include_group_label_in_selected||!1,this.max_shown_results=this.options.max_shown_results||Number.POSITIVE_INFINITY},AbstractChosen.prototype.set_default_text=function(){return this.form_field.getAttribute("data-placeholder")?this.default_text=this.form_field.getAttribute("data-placeholder"):this.is_multiple?this.default_text=this.options.placeholder_text_multiple||this.options.placeholder_text||AbstractChosen.default_multiple_text:this.default_text=this.options.placeholder_text_single||this.options.placeholder_text||AbstractChosen.default_single_text,this.results_none_found=this.form_field.getAttribute("data-no_results_text")||this.options.no_results_text||AbstractChosen.default_no_result_text},AbstractChosen.prototype.choice_label=function(a){return this.include_group_label_in_selected&&null!=a.group_label?""+a.group_label+""+a.html:a.html},AbstractChosen.prototype.mouse_enter=function(){return this.mouse_on_container=!0},AbstractChosen.prototype.mouse_leave=function(){return this.mouse_on_container=!1},AbstractChosen.prototype.input_focus=function(a){var b=this;if(this.is_multiple){if(!this.active_field)return setTimeout(function(){return b.container_mousedown()},50)}else if(!this.active_field)return this.activate_field()},AbstractChosen.prototype.input_blur=function(a){var b=this;return this.mouse_on_container?void 0:(this.active_field=!1,setTimeout(function(){return b.blur_test()},100))},AbstractChosen.prototype.results_option_build=function(a){var b,c,d,e,f,g,h;for(b="",e=0,h=this.results_data,f=0,g=h.length;g>f&&(c=h[f],d="",d=c.group?this.result_add_group(c):this.result_add_option(c),""!==d&&(e++,b+=d),(null!=a?a.first:void 0)&&(c.selected&&this.is_multiple?this.choice_build(c):c.selected&&!this.is_multiple&&this.single_set_selected_text(this.choice_label(c))),!(e>=this.max_shown_results));f++);return b},AbstractChosen.prototype.result_add_option=function(a){var b,c;return a.search_match&&this.include_option_in_results(a)?(b=[],a.disabled||a.selected&&this.is_multiple||b.push("active-result"),!a.disabled||a.selected&&this.is_multiple||b.push("disabled-result"),a.selected&&b.push("result-selected"),null!=a.group_array_index&&b.push("group-option"),""!==a.classes&&b.push(a.classes),c=document.createElement("li"),c.className=b.join(" "),c.style.cssText=a.style,c.setAttribute("data-option-array-index",a.array_index),c.innerHTML=a.search_text,a.title&&(c.title=a.title),this.outerHTML(c)):""},AbstractChosen.prototype.result_add_group=function(a){var b,c;return(a.search_match||a.group_match)&&a.active_options>0?(b=[],b.push("group-result"),a.classes&&b.push(a.classes),c=document.createElement("li"),c.className=b.join(" "),c.innerHTML=a.search_text,a.title&&(c.title=a.title),this.outerHTML(c)):""},AbstractChosen.prototype.results_update_field=function(){return this.set_default_text(),this.is_multiple||this.results_reset_cleanup(),this.result_clear_highlight(),this.results_build(),this.results_showing?this.winnow_results():void 0},AbstractChosen.prototype.reset_single_select_options=function(){var a,b,c,d,e;for(d=this.results_data,e=[],b=0,c=d.length;c>b;b++)a=d[b],a.selected?e.push(a.selected=!1):e.push(void 0);return e},AbstractChosen.prototype.results_toggle=function(){return this.results_showing?this.results_hide():this.results_show()},AbstractChosen.prototype.results_search=function(a){return this.results_showing?this.winnow_results():this.results_show()},AbstractChosen.prototype.winnow_results=function(){var a,b,c,d,e,f,g,h,i,j,k,l;for(this.no_results_clear(),d=0,f=this.get_search_text(),a=f.replace(/[-[\]{}()*+?.,\\^$|#\s]/g,"\\$&"),i=new RegExp(a,"i"),c=this.get_search_regex(a),l=this.results_data,j=0,k=l.length;k>j;j++)b=l[j],b.search_match=!1,e=null,this.include_option_in_results(b)&&(b.group&&(b.group_match=!1,b.active_options=0),null!=b.group_array_index&&this.results_data[b.group_array_index]&&(e=this.results_data[b.group_array_index],0===e.active_options&&e.search_match&&(d+=1),e.active_options+=1),b.search_text=b.group?b.label:b.html,(!b.group||this.group_search)&&(b.search_match=this.search_string_match(b.search_text,c),b.search_match&&!b.group&&(d+=1),b.search_match?(f.length&&(g=b.search_text.search(i),h=b.search_text.substr(0,g+f.length)+""+b.search_text.substr(g+f.length),b.search_text=h.substr(0,g)+""+h.substr(g)),null!=e&&(e.group_match=!0)):null!=b.group_array_index&&this.results_data[b.group_array_index].search_match&&(b.search_match=!0)));return this.result_clear_highlight(),1>d&&f.length?(this.update_results_content(""),this.no_results(f)):(this.update_results_content(this.results_option_build()),this.winnow_results_set_highlight())},AbstractChosen.prototype.get_search_regex=function(a){var b;return b=this.search_contains?"":"^",new RegExp(b+a,"i")},AbstractChosen.prototype.search_string_match=function(a,b){var c,d,e,f;if(b.test(a))return!0;if(this.enable_split_word_search&&(a.indexOf(" ")>=0||0===a.indexOf("["))&&(d=a.replace(/\[|\]/g,"").split(" "),d.length))for(e=0,f=d.length;f>e;e++)if(c=d[e],b.test(c))return!0},AbstractChosen.prototype.choices_count=function(){var a,b,c,d;if(null!=this.selected_option_count)return this.selected_option_count;for(this.selected_option_count=0,d=this.form_field.options,b=0,c=d.length;c>b;b++)a=d[b],a.selected&&(this.selected_option_count+=1);return this.selected_option_count},AbstractChosen.prototype.choices_click=function(a){return a.preventDefault(),this.results_showing||this.is_disabled?void 0:this.results_show()},AbstractChosen.prototype.keyup_checker=function(a){var b,c;switch(b=null!=(c=a.which)?c:a.keyCode,this.search_field_scale(),b){case 8:if(this.is_multiple&&this.backstroke_length<1&&this.choices_count()>0)return this.keydown_backstroke();if(!this.pending_backstroke)return this.result_clear_highlight(),this.results_search();break;case 13:if(a.preventDefault(),this.results_showing)return this.result_select(a);break;case 27:return this.results_showing&&this.results_hide(),!0;case 9:case 38:case 40:case 16:case 91:case 17:case 18:break;default:return this.results_search()}},AbstractChosen.prototype.clipboard_event_checker=function(a){var b=this;return setTimeout(function(){return b.results_search()},50)},AbstractChosen.prototype.container_width=function(){return null!=this.options.width?this.options.width:""+this.form_field.offsetWidth+"px"},AbstractChosen.prototype.include_option_in_results=function(a){return this.is_multiple&&!this.display_selected_options&&a.selected?!1:!this.display_disabled_options&&a.disabled?!1:a.empty?!1:!0},AbstractChosen.prototype.search_results_touchstart=function(a){return this.touch_started=!0,this.search_results_mouseover(a)},AbstractChosen.prototype.search_results_touchmove=function(a){return this.touch_started=!1,this.search_results_mouseout(a)},AbstractChosen.prototype.search_results_touchend=function(a){return this.touch_started?this.search_results_mouseup(a):void 0},AbstractChosen.prototype.outerHTML=function(a){var b;return a.outerHTML?a.outerHTML:(b=document.createElement("div"),b.appendChild(a),b.innerHTML)},AbstractChosen.browser_is_supported=function(){return/iP(od|hone)/i.test(window.navigator.userAgent)?!1:/Android/i.test(window.navigator.userAgent)&&/Mobile/i.test(window.navigator.userAgent)?!1:/IEMobile/i.test(window.navigator.userAgent)?!1:/Windows Phone/i.test(window.navigator.userAgent)?!1:/BlackBerry/i.test(window.navigator.userAgent)?!1:/BB10/i.test(window.navigator.userAgent)?!1:"Microsoft Internet Explorer"===window.navigator.appName?document.documentMode>=8:!0},AbstractChosen.default_multiple_text="Select Some Options",AbstractChosen.default_single_text="Select an Option",AbstractChosen.default_no_result_text="No results match",AbstractChosen}(),a=jQuery,a.fn.extend({chosen:function(b){return AbstractChosen.browser_is_supported()?this.each(function(c){var d,e;return d=a(this),e=d.data("chosen"),"destroy"===b?void(e instanceof Chosen&&e.destroy()):void(e instanceof Chosen||d.data("chosen",new Chosen(this,b)))}):this}}),Chosen=function(c){function Chosen(){return b=Chosen.__super__.constructor.apply(this,arguments)}return d(Chosen,c),Chosen.prototype.setup=function(){return this.form_field_jq=a(this.form_field),this.current_selectedIndex=this.form_field.selectedIndex,this.is_rtl=this.form_field_jq.hasClass("chosen-rtl")},Chosen.prototype.set_up_html=function(){var b,c;return b=["chosen-container"],b.push("chosen-container-"+(this.is_multiple?"multi":"single")),this.inherit_select_classes&&this.form_field.className&&b.push(this.form_field.className),this.is_rtl&&b.push("chosen-rtl"),c={"class":b.join(" "),style:"width: "+this.container_width()+";",title:this.form_field.title},this.form_field.id.length&&(c.id=this.form_field.id.replace(/[^\w]/g,"_")+"_chosen"),this.container=a("
    ",c),this.is_multiple?this.container.html('
      '):this.container.html(''+this.default_text+'
        '),this.form_field_jq.hide().after(this.container),this.dropdown=this.container.find("div.chosen-drop").first(),this.search_field=this.container.find("input").first(),this.search_results=this.container.find("ul.chosen-results").first(),this.search_field_scale(),this.search_no_results=this.container.find("li.no-results").first(),this.is_multiple?(this.search_choices=this.container.find("ul.chosen-choices").first(),this.search_container=this.container.find("li.search-field").first()):(this.search_container=this.container.find("div.chosen-search").first(),this.selected_item=this.container.find(".chosen-single").first()),this.results_build(),this.set_tab_index(),this.set_label_behavior()},Chosen.prototype.on_ready=function(){return this.form_field_jq.trigger("chosen:ready",{chosen:this})},Chosen.prototype.register_observers=function(){var a=this;return this.container.bind("touchstart.chosen",function(b){return a.container_mousedown(b),b.preventDefault()}),this.container.bind("touchend.chosen",function(b){return a.container_mouseup(b),b.preventDefault()}),this.container.bind("mousedown.chosen",function(b){a.container_mousedown(b)}),this.container.bind("mouseup.chosen",function(b){a.container_mouseup(b)}),this.container.bind("mouseenter.chosen",function(b){a.mouse_enter(b)}),this.container.bind("mouseleave.chosen",function(b){a.mouse_leave(b)}),this.search_results.bind("mouseup.chosen",function(b){a.search_results_mouseup(b)}),this.search_results.bind("mouseover.chosen",function(b){a.search_results_mouseover(b)}),this.search_results.bind("mouseout.chosen",function(b){a.search_results_mouseout(b)}),this.search_results.bind("mousewheel.chosen DOMMouseScroll.chosen",function(b){a.search_results_mousewheel(b)}),this.search_results.bind("touchstart.chosen",function(b){a.search_results_touchstart(b)}),this.search_results.bind("touchmove.chosen",function(b){a.search_results_touchmove(b)}),this.search_results.bind("touchend.chosen",function(b){a.search_results_touchend(b)}),this.form_field_jq.bind("chosen:updated.chosen",function(b){a.results_update_field(b)}),this.form_field_jq.bind("chosen:activate.chosen",function(b){a.activate_field(b)}),this.form_field_jq.bind("chosen:open.chosen",function(b){a.container_mousedown(b)}),this.form_field_jq.bind("chosen:close.chosen",function(b){a.input_blur(b)}),this.search_field.bind("blur.chosen",function(b){a.input_blur(b)}),this.search_field.bind("keyup.chosen",function(b){a.keyup_checker(b)}),this.search_field.bind("keydown.chosen",function(b){a.keydown_checker(b)}),this.search_field.bind("focus.chosen",function(b){a.input_focus(b)}),this.search_field.bind("cut.chosen",function(b){a.clipboard_event_checker(b)}),this.search_field.bind("paste.chosen",function(b){a.clipboard_event_checker(b)}),this.is_multiple?this.search_choices.bind("click.chosen",function(b){a.choices_click(b)}):this.container.bind("click.chosen",function(a){a.preventDefault()})},Chosen.prototype.destroy=function(){return a(this.container[0].ownerDocument).unbind("click.chosen",this.click_test_action),this.search_field[0].tabIndex&&(this.form_field_jq[0].tabIndex=this.search_field[0].tabIndex),this.container.remove(),this.form_field_jq.removeData("chosen"),this.form_field_jq.show()},Chosen.prototype.search_field_disabled=function(){return this.is_disabled=this.form_field_jq[0].disabled,this.is_disabled?(this.container.addClass("chosen-disabled"),this.search_field[0].disabled=!0,this.is_multiple||this.selected_item.unbind("focus.chosen",this.activate_action),this.close_field()):(this.container.removeClass("chosen-disabled"),this.search_field[0].disabled=!1,this.is_multiple?void 0:this.selected_item.bind("focus.chosen",this.activate_action))},Chosen.prototype.container_mousedown=function(b){return this.is_disabled||(b&&"mousedown"===b.type&&!this.results_showing&&b.preventDefault(),null!=b&&a(b.target).hasClass("search-choice-close"))?void 0:(this.active_field?this.is_multiple||!b||a(b.target)[0]!==this.selected_item[0]&&!a(b.target).parents("a.chosen-single").length||(b.preventDefault(),this.results_toggle()):(this.is_multiple&&this.search_field.val(""),a(this.container[0].ownerDocument).bind("click.chosen",this.click_test_action),this.results_show()),this.activate_field())},Chosen.prototype.container_mouseup=function(a){return"ABBR"!==a.target.nodeName||this.is_disabled?void 0:this.results_reset(a)},Chosen.prototype.search_results_mousewheel=function(a){var b;return a.originalEvent&&(b=a.originalEvent.deltaY||-a.originalEvent.wheelDelta||a.originalEvent.detail),null!=b?(a.preventDefault(),"DOMMouseScroll"===a.type&&(b=40*b),this.search_results.scrollTop(b+this.search_results.scrollTop())):void 0},Chosen.prototype.blur_test=function(a){return!this.active_field&&this.container.hasClass("chosen-container-active")?this.close_field():void 0},Chosen.prototype.close_field=function(){return a(this.container[0].ownerDocument).unbind("click.chosen",this.click_test_action),this.active_field=!1,this.results_hide(),this.container.removeClass("chosen-container-active"),this.clear_backstroke(),this.show_search_field_default(),this.search_field_scale()},Chosen.prototype.activate_field=function(){return this.container.addClass("chosen-container-active"),this.active_field=!0,this.search_field.val(this.search_field.val()),this.search_field.focus()},Chosen.prototype.test_active_click=function(b){var c;return c=a(b.target).closest(".chosen-container"),c.length&&this.container[0]===c[0]?this.active_field=!0:this.close_field()},Chosen.prototype.results_build=function(){return this.parsing=!0,this.selected_option_count=null,this.results_data=SelectParser.select_to_array(this.form_field),this.is_multiple?this.search_choices.find("li.search-choice").remove():this.is_multiple||(this.single_set_selected_text(),this.disable_search||this.form_field.options.length<=this.disable_search_threshold?(this.search_field[0].readOnly=!0,this.container.addClass("chosen-container-single-nosearch")):(this.search_field[0].readOnly=!1,this.container.removeClass("chosen-container-single-nosearch"))),this.update_results_content(this.results_option_build({first:!0})),this.search_field_disabled(),this.show_search_field_default(),this.search_field_scale(),this.parsing=!1},Chosen.prototype.result_do_highlight=function(a){var b,c,d,e,f;if(a.length){if(this.result_clear_highlight(),this.result_highlight=a,this.result_highlight.addClass("highlighted"),d=parseInt(this.search_results.css("maxHeight"),10),f=this.search_results.scrollTop(),e=d+f,c=this.result_highlight.position().top+this.search_results.scrollTop(),b=c+this.result_highlight.outerHeight(),b>=e)return this.search_results.scrollTop(b-d>0?b-d:0);if(f>c)return this.search_results.scrollTop(c)}},Chosen.prototype.result_clear_highlight=function(){return this.result_highlight&&this.result_highlight.removeClass("highlighted"),this.result_highlight=null},Chosen.prototype.results_show=function(){return this.is_multiple&&this.max_selected_options<=this.choices_count()?(this.form_field_jq.trigger("chosen:maxselected",{chosen:this}),!1):(this.container.addClass("chosen-with-drop"),this.results_showing=!0,this.search_field.focus(),this.search_field.val(this.search_field.val()),this.winnow_results(),this.form_field_jq.trigger("chosen:showing_dropdown",{chosen:this}))},Chosen.prototype.update_results_content=function(a){return this.search_results.html(a)},Chosen.prototype.results_hide=function(){return this.results_showing&&(this.result_clear_highlight(),this.container.removeClass("chosen-with-drop"),this.form_field_jq.trigger("chosen:hiding_dropdown",{chosen:this})),this.results_showing=!1},Chosen.prototype.set_tab_index=function(a){var b;return this.form_field.tabIndex?(b=this.form_field.tabIndex,this.form_field.tabIndex=-1,this.search_field[0].tabIndex=b):void 0},Chosen.prototype.set_label_behavior=function(){var b=this;return this.form_field_label=this.form_field_jq.parents("label"),!this.form_field_label.length&&this.form_field.id.length&&(this.form_field_label=a("label[for='"+this.form_field.id+"']")),this.form_field_label.length>0?this.form_field_label.bind("click.chosen",function(a){return b.is_multiple?b.container_mousedown(a):b.activate_field()}):void 0},Chosen.prototype.show_search_field_default=function(){return this.is_multiple&&this.choices_count()<1&&!this.active_field?(this.search_field.val(this.default_text),this.search_field.addClass("default")):(this.search_field.val(""),this.search_field.removeClass("default"))},Chosen.prototype.search_results_mouseup=function(b){var c;return c=a(b.target).hasClass("active-result")?a(b.target):a(b.target).parents(".active-result").first(),c.length?(this.result_highlight=c,this.result_select(b),this.search_field.focus()):void 0},Chosen.prototype.search_results_mouseover=function(b){var c;return c=a(b.target).hasClass("active-result")?a(b.target):a(b.target).parents(".active-result").first(),c?this.result_do_highlight(c):void 0},Chosen.prototype.search_results_mouseout=function(b){return a(b.target).hasClass("active-result")?this.result_clear_highlight():void 0},Chosen.prototype.choice_build=function(b){var c,d,e=this;return c=a("
      • ",{"class":"search-choice"}).html(""+this.choice_label(b)+""),b.disabled?c.addClass("search-choice-disabled"):(d=a("",{"class":"search-choice-close","data-option-array-index":b.array_index}),d.bind("click.chosen",function(a){return e.choice_destroy_link_click(a)}),c.append(d)),this.search_container.before(c)},Chosen.prototype.choice_destroy_link_click=function(b){return b.preventDefault(),b.stopPropagation(),this.is_disabled?void 0:this.choice_destroy(a(b.target))},Chosen.prototype.choice_destroy=function(a){return this.result_deselect(a[0].getAttribute("data-option-array-index"))?(this.show_search_field_default(),this.is_multiple&&this.choices_count()>0&&this.search_field.val().length<1&&this.results_hide(),a.parents("li").first().remove(),this.search_field_scale()):void 0},Chosen.prototype.results_reset=function(){return this.reset_single_select_options(),this.form_field.options[0].selected=!0,this.single_set_selected_text(),this.show_search_field_default(),this.results_reset_cleanup(),this.form_field_jq.trigger("change"),this.active_field?this.results_hide():void 0},Chosen.prototype.results_reset_cleanup=function(){return this.current_selectedIndex=this.form_field.selectedIndex,this.selected_item.find("abbr").remove()},Chosen.prototype.result_select=function(a){var b,c;return this.result_highlight?(b=this.result_highlight,this.result_clear_highlight(),this.is_multiple&&this.max_selected_options<=this.choices_count()?(this.form_field_jq.trigger("chosen:maxselected",{chosen:this}),!1):(this.is_multiple?b.removeClass("active-result"):this.reset_single_select_options(),b.addClass("result-selected"),c=this.results_data[b[0].getAttribute("data-option-array-index")],c.selected=!0,this.form_field.options[c.options_index].selected=!0,this.selected_option_count=null,this.is_multiple?this.choice_build(c):this.single_set_selected_text(this.choice_label(c)),(a.metaKey||a.ctrlKey)&&this.is_multiple||this.results_hide(),this.show_search_field_default(),(this.is_multiple||this.form_field.selectedIndex!==this.current_selectedIndex)&&this.form_field_jq.trigger("change",{selected:this.form_field.options[c.options_index].value}),this.current_selectedIndex=this.form_field.selectedIndex,a.preventDefault(),this.search_field_scale())):void 0},Chosen.prototype.single_set_selected_text=function(a){return null==a&&(a=this.default_text),a===this.default_text?this.selected_item.addClass("chosen-default"):(this.single_deselect_control_build(),this.selected_item.removeClass("chosen-default")),this.selected_item.find("span").html(a)},Chosen.prototype.result_deselect=function(a){var b;return b=this.results_data[a],this.form_field.options[b.options_index].disabled?!1:(b.selected=!1,this.form_field.options[b.options_index].selected=!1,this.selected_option_count=null,this.result_clear_highlight(),this.results_showing&&this.winnow_results(),this.form_field_jq.trigger("change",{deselected:this.form_field.options[b.options_index].value}),this.search_field_scale(),!0)},Chosen.prototype.single_deselect_control_build=function(){return this.allow_single_deselect?(this.selected_item.find("abbr").length||this.selected_item.find("span").first().after(''),this.selected_item.addClass("chosen-single-with-deselect")):void 0},Chosen.prototype.get_search_text=function(){return a("
        ").text(a.trim(this.search_field.val())).html()},Chosen.prototype.winnow_results_set_highlight=function(){var a,b;return b=this.is_multiple?[]:this.search_results.find(".result-selected.active-result"),a=b.length?b.first():this.search_results.find(".active-result").first(),null!=a?this.result_do_highlight(a):void 0},Chosen.prototype.no_results=function(b){var c;return c=a('
      • '+this.results_none_found+' ""
        • '),c.find("span").first().html(b),this.search_results.append(c),this.form_field_jq.trigger("chosen:no_results",{chosen:this})},Chosen.prototype.no_results_clear=function(){return this.search_results.find(".no-results").remove()},Chosen.prototype.keydown_arrow=function(){var a;return this.results_showing&&this.result_highlight?(a=this.result_highlight.nextAll("li.active-result").first())?this.result_do_highlight(a):void 0:this.results_show()},Chosen.prototype.keyup_arrow=function(){var a;return this.results_showing||this.is_multiple?this.result_highlight?(a=this.result_highlight.prevAll("li.active-result"),a.length?this.result_do_highlight(a.first()):(this.choices_count()>0&&this.results_hide(),this.result_clear_highlight())):void 0:this.results_show()},Chosen.prototype.keydown_backstroke=function(){var a;return this.pending_backstroke?(this.choice_destroy(this.pending_backstroke.find("a").first()),this.clear_backstroke()):(a=this.search_container.siblings("li.search-choice").last(),a.length&&!a.hasClass("search-choice-disabled")?(this.pending_backstroke=a,this.single_backstroke_delete?this.keydown_backstroke():this.pending_backstroke.addClass("search-choice-focus")):void 0)},Chosen.prototype.clear_backstroke=function(){return this.pending_backstroke&&this.pending_backstroke.removeClass("search-choice-focus"),this.pending_backstroke=null},Chosen.prototype.keydown_checker=function(a){var b,c;switch(b=null!=(c=a.which)?c:a.keyCode,this.search_field_scale(),8!==b&&this.pending_backstroke&&this.clear_backstroke(),b){case 8:this.backstroke_length=this.search_field.val().length;break;case 9:this.results_showing&&!this.is_multiple&&this.result_select(a),this.mouse_on_container=!1;break;case 13:this.results_showing&&a.preventDefault();break;case 32:this.disable_search&&a.preventDefault();break;case 38:a.preventDefault(),this.keyup_arrow();break;case 40:a.preventDefault(),this.keydown_arrow()}},Chosen.prototype.search_field_scale=function(){var b,c,d,e,f,g,h,i,j;if(this.is_multiple){for(d=0,h=0,f="position:absolute; left: -1000px; top: -1000px; display:none;",g=["font-size","font-style","font-weight","font-family","line-height","text-transform","letter-spacing"],i=0,j=g.length;j>i;i++)e=g[i],f+=e+":"+this.search_field.css(e)+";";return b=a("
        ",{style:f}),b.text(this.search_field.val()),a("body").append(b),h=b.width()+25,b.remove(),c=this.container.outerWidth(),h>c-10&&(h=c-10),this.search_field.css({width:h+"px"})}},Chosen}(AbstractChosen)}).call(this); \ No newline at end of file diff --git a/micropsi_server/static/js/chosen.min.css b/micropsi_server/static/js/chosen.min.css new file mode 100755 index 00000000..9484b570 --- /dev/null +++ b/micropsi_server/static/js/chosen.min.css @@ -0,0 +1,3 @@ +/* Chosen v1.5.0 | (c) 2011-2016 by Harvest | MIT License, https://github.com/harvesthq/chosen/blob/master/LICENSE.md */ + +.chosen-container{position:relative;display:inline-block;vertical-align:middle;font-size:13px;-webkit-user-select:none;-moz-user-select:none;user-select:none}.chosen-container *{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.chosen-container .chosen-drop{position:absolute;top:100%;left:-9999px;z-index:1010;width:100%;border:1px solid #aaa;border-top:0;background:#fff;box-shadow:0 4px 5px rgba(0,0,0,.15)}.chosen-container.chosen-with-drop .chosen-drop{left:0}.chosen-container a{cursor:pointer}.chosen-container .search-choice .group-name,.chosen-container .chosen-single .group-name{margin-right:4px;overflow:hidden;white-space:nowrap;text-overflow:ellipsis;font-weight:400;color:#999}.chosen-container .search-choice .group-name:after,.chosen-container .chosen-single .group-name:after{content:":";padding-left:2px;vertical-align:top}.chosen-container-single .chosen-single{position:relative;display:block;overflow:hidden;padding:0 0 0 8px;height:25px;border:1px solid #aaa;border-radius:5px;background-color:#fff;background:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#fff),color-stop(50%,#f6f6f6),color-stop(52%,#eee),color-stop(100%,#f4f4f4));background:-webkit-linear-gradient(#fff 20%,#f6f6f6 50%,#eee 52%,#f4f4f4 100%);background:-moz-linear-gradient(#fff 20%,#f6f6f6 50%,#eee 52%,#f4f4f4 100%);background:-o-linear-gradient(#fff 20%,#f6f6f6 50%,#eee 52%,#f4f4f4 100%);background:linear-gradient(#fff 20%,#f6f6f6 50%,#eee 52%,#f4f4f4 100%);background-clip:padding-box;box-shadow:0 0 3px #fff inset,0 1px 1px rgba(0,0,0,.1);color:#444;text-decoration:none;white-space:nowrap;line-height:24px}.chosen-container-single .chosen-default{color:#999}.chosen-container-single .chosen-single span{display:block;overflow:hidden;margin-right:26px;text-overflow:ellipsis;white-space:nowrap}.chosen-container-single .chosen-single-with-deselect span{margin-right:38px}.chosen-container-single .chosen-single abbr{position:absolute;top:6px;right:26px;display:block;width:12px;height:12px;background:url(chosen-sprite.png) -42px 1px no-repeat;font-size:1px}.chosen-container-single .chosen-single abbr:hover{background-position:-42px -10px}.chosen-container-single.chosen-disabled .chosen-single abbr:hover{background-position:-42px -10px}.chosen-container-single .chosen-single div{position:absolute;top:0;right:0;display:block;width:18px;height:100%}.chosen-container-single .chosen-single div b{display:block;width:100%;height:100%;background:url(chosen-sprite.png) no-repeat 0 2px}.chosen-container-single .chosen-search{position:relative;z-index:1010;margin:0;padding:3px 4px;white-space:nowrap}.chosen-container-single .chosen-search input[type=text]{margin:1px 0;padding:4px 20px 4px 5px;width:100%;height:auto;outline:0;border:1px solid #aaa;background:#fff url(chosen-sprite.png) no-repeat 100% -20px;background:url(chosen-sprite.png) no-repeat 100% -20px;font-size:1em;font-family:sans-serif;line-height:normal;border-radius:0}.chosen-container-single .chosen-drop{margin-top:-1px;border-radius:0 0 4px 4px;background-clip:padding-box}.chosen-container-single.chosen-container-single-nosearch .chosen-search{position:absolute;left:-9999px}.chosen-container .chosen-results{color:#444;position:relative;overflow-x:hidden;overflow-y:auto;margin:0 4px 4px 0;padding:0 0 0 4px;max-height:240px;-webkit-overflow-scrolling:touch}.chosen-container .chosen-results li{display:none;margin:0;padding:5px 6px;list-style:none;line-height:15px;word-wrap:break-word;-webkit-touch-callout:none}.chosen-container .chosen-results li.active-result{display:list-item;cursor:pointer}.chosen-container .chosen-results li.disabled-result{display:list-item;color:#ccc;cursor:default}.chosen-container .chosen-results li.highlighted{background-color:#3875d7;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#3875d7),color-stop(90%,#2a62bc));background-image:-webkit-linear-gradient(#3875d7 20%,#2a62bc 90%);background-image:-moz-linear-gradient(#3875d7 20%,#2a62bc 90%);background-image:-o-linear-gradient(#3875d7 20%,#2a62bc 90%);background-image:linear-gradient(#3875d7 20%,#2a62bc 90%);color:#fff}.chosen-container .chosen-results li.no-results{color:#777;display:list-item;background:#f4f4f4}.chosen-container .chosen-results li.group-result{display:list-item;font-weight:700;cursor:default}.chosen-container .chosen-results li.group-option{padding-left:15px}.chosen-container .chosen-results li em{font-style:normal;text-decoration:underline}.chosen-container-multi .chosen-choices{position:relative;overflow:hidden;margin:0;padding:0 5px;width:100%;height:auto!important;height:1%;border:1px solid #aaa;background-color:#fff;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(1%,#eee),color-stop(15%,#fff));background-image:-webkit-linear-gradient(#eee 1%,#fff 15%);background-image:-moz-linear-gradient(#eee 1%,#fff 15%);background-image:-o-linear-gradient(#eee 1%,#fff 15%);background-image:linear-gradient(#eee 1%,#fff 15%);cursor:text}.chosen-container-multi .chosen-choices li{float:left;list-style:none}.chosen-container-multi .chosen-choices li.search-field{margin:0;padding:0;white-space:nowrap}.chosen-container-multi .chosen-choices li.search-field input[type=text]{margin:1px 0;padding:0;height:25px;outline:0;border:0!important;background:transparent!important;box-shadow:none;color:#999;font-size:100%;font-family:sans-serif;line-height:normal;border-radius:0}.chosen-container-multi .chosen-choices li.search-choice{position:relative;margin:3px 5px 3px 0;padding:3px 20px 3px 5px;border:1px solid #aaa;max-width:100%;border-radius:3px;background-color:#eee;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#f4f4f4),color-stop(50%,#f0f0f0),color-stop(52%,#e8e8e8),color-stop(100%,#eee));background-image:-webkit-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:-moz-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:-o-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-size:100% 19px;background-repeat:repeat-x;background-clip:padding-box;box-shadow:0 0 2px #fff inset,0 1px 0 rgba(0,0,0,.05);color:#333;line-height:13px;cursor:default}.chosen-container-multi .chosen-choices li.search-choice span{word-wrap:break-word}.chosen-container-multi .chosen-choices li.search-choice .search-choice-close{position:absolute;top:4px;right:3px;display:block;width:12px;height:12px;background:url(chosen-sprite.png) -42px 1px no-repeat;font-size:1px}.chosen-container-multi .chosen-choices li.search-choice .search-choice-close:hover{background-position:-42px -10px}.chosen-container-multi .chosen-choices li.search-choice-disabled{padding-right:5px;border:1px solid #ccc;background-color:#e4e4e4;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#f4f4f4),color-stop(50%,#f0f0f0),color-stop(52%,#e8e8e8),color-stop(100%,#eee));background-image:-webkit-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:-moz-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:-o-linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);background-image:linear-gradient(#f4f4f4 20%,#f0f0f0 50%,#e8e8e8 52%,#eee 100%);color:#666}.chosen-container-multi .chosen-choices li.search-choice-focus{background:#d4d4d4}.chosen-container-multi .chosen-choices li.search-choice-focus .search-choice-close{background-position:-42px -10px}.chosen-container-multi .chosen-results{margin:0;padding:0}.chosen-container-multi .chosen-drop .result-selected{display:list-item;color:#ccc;cursor:default}.chosen-container-active .chosen-single{border:1px solid #5897fb;box-shadow:0 0 5px rgba(0,0,0,.3)}.chosen-container-active.chosen-with-drop .chosen-single{border:1px solid #aaa;-moz-border-radius-bottomright:0;border-bottom-right-radius:0;-moz-border-radius-bottomleft:0;border-bottom-left-radius:0;background-image:-webkit-gradient(linear,50% 0,50% 100%,color-stop(20%,#eee),color-stop(80%,#fff));background-image:-webkit-linear-gradient(#eee 20%,#fff 80%);background-image:-moz-linear-gradient(#eee 20%,#fff 80%);background-image:-o-linear-gradient(#eee 20%,#fff 80%);background-image:linear-gradient(#eee 20%,#fff 80%);box-shadow:0 1px 0 #fff inset}.chosen-container-active.chosen-with-drop .chosen-single div{border-left:0;background:transparent}.chosen-container-active.chosen-with-drop .chosen-single div b{background-position:-18px 2px}.chosen-container-active .chosen-choices{border:1px solid #5897fb;box-shadow:0 0 5px rgba(0,0,0,.3)}.chosen-container-active .chosen-choices li.search-field input[type=text]{color:#222!important}.chosen-disabled{opacity:.5!important;cursor:default}.chosen-disabled .chosen-single{cursor:default}.chosen-disabled .chosen-choices .search-choice .search-choice-close{cursor:default}.chosen-rtl{text-align:right}.chosen-rtl .chosen-single{overflow:visible;padding:0 8px 0 0}.chosen-rtl .chosen-single span{margin-right:0;margin-left:26px;direction:rtl}.chosen-rtl .chosen-single-with-deselect span{margin-left:38px}.chosen-rtl .chosen-single div{right:auto;left:3px}.chosen-rtl .chosen-single abbr{right:auto;left:26px}.chosen-rtl .chosen-choices li{float:right}.chosen-rtl .chosen-choices li.search-field input[type=text]{direction:rtl}.chosen-rtl .chosen-choices li.search-choice{margin:3px 5px 3px 0;padding:3px 5px 3px 19px}.chosen-rtl .chosen-choices li.search-choice .search-choice-close{right:auto;left:4px}.chosen-rtl.chosen-container-single-nosearch .chosen-search,.chosen-rtl .chosen-drop{left:9999px}.chosen-rtl.chosen-container-single .chosen-results{margin:0 0 4px 4px;padding:0 4px 0 0}.chosen-rtl .chosen-results li.group-option{padding-right:15px;padding-left:0}.chosen-rtl.chosen-container-active.chosen-with-drop .chosen-single div{border-right:0}.chosen-rtl .chosen-search input[type=text]{padding:4px 5px 4px 20px;background:#fff url(chosen-sprite.png) no-repeat -30px -20px;background:url(chosen-sprite.png) no-repeat -30px -20px;direction:rtl}.chosen-rtl.chosen-container-single .chosen-single div b{background-position:6px 2px}.chosen-rtl.chosen-container-single.chosen-with-drop .chosen-single div b{background-position:-12px 2px}@media only screen and (-webkit-min-device-pixel-ratio:1.5),only screen and (min-resolution:144dpi),only screen and (min-resolution:1.5dppx){.chosen-rtl .chosen-search input[type=text],.chosen-container-single .chosen-single abbr,.chosen-container-single .chosen-single div b,.chosen-container-single .chosen-search input[type=text],.chosen-container-multi .chosen-choices .search-choice .search-choice-close,.chosen-container .chosen-results-scroll-down span,.chosen-container .chosen-results-scroll-up span{background-image:url(chosen-sprite@2x.png)!important;background-size:52px 37px!important;background-repeat:no-repeat!important}} \ No newline at end of file diff --git a/micropsi_server/static/js/dialogs.js b/micropsi_server/static/js/dialogs.js index 8d159303..8d835cd5 100644 --- a/micropsi_server/static/js/dialogs.js +++ b/micropsi_server/static/js/dialogs.js @@ -541,8 +541,8 @@ $(function() { $('#recipe_modal .docstring').hide(); $('#recipe_modal .btn-primary').hide(); } + var html = ''; if(name in recipes){ - var html = ''; for(var i in recipes[name].parameters){ var param = recipes[name].parameters[i]; html += '' + @@ -553,8 +553,8 @@ $(function() { '
        '+ '
        '; } - $('.recipe_param_container').html(html); } + $('.recipe_param_container').html(html); }; var run_recipe = function(event){ @@ -631,15 +631,30 @@ $(function() { $('#recipe_modal button').prop('disabled', false); api.call('get_available_recipes', {}, function(data){ recipes = data; - var options = []; - for(var key in data){ - options.push(data[key].name); + var categories = {}; + for(var key in recipes){ + if(!categories[recipes[key].category]){ + categories[recipes[key].category] = []; + } + categories[recipes[key].category].push(recipes[key]); + } + var sorted = Object.keys(categories); + sorted.sort(); + recipe_name_input.chosen('destroy'); + var html = ''; + var cat; + for(var i in sorted){ + cat = sorted[i] + html += ''; + categories[cat].sort(sortByName); + for(var i in categories[cat]){ + html += ''; + } + html += '' } - recipe_name_input.typeahead({'source': options, highlighter: function(item){ - var search = recipe_name_input.val(); - var text = item.replace(search, ""+search+""); - return ''+text+' ('+recipes[item].category+')'; - }}); + recipe_name_input.html(html); + recipe_name_input.val(''); + recipe_name_input.chosen({'search_contains': true}); recipe_name_input.focus(); update_parameters_for_recipe(); }); diff --git a/micropsi_server/view/boilerplate.tpl b/micropsi_server/view/boilerplate.tpl index f38cdf50..fa05cffb 100644 --- a/micropsi_server/view/boilerplate.tpl +++ b/micropsi_server/view/boilerplate.tpl @@ -23,6 +23,7 @@ + +
        +
        From 0c8c38c1c4f686c92babcfe4287a1f6e90f73e37 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 17 Feb 2016 22:56:35 +0100 Subject: [PATCH 069/306] some pep8 and a typo --- micropsi_core/_runtime_api_world.py | 2 +- micropsi_core/nodenet/theano_engine/theano_node.py | 2 +- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 1 - micropsi_core/world/world.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index c8a9ed1d..e18fd03c 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -183,7 +183,7 @@ def export_world(world_uid): def import_world(worlddata, owner=None): """Imports a JSON string with world data. May not overwrite an existing world.""" data = json.loads(worlddata) - if not 'uid' in data: + if 'uid' not in data: data['uid'] = tools.generate_uid() else: if data['uid'] in micropsi_core.runtime.worlds: diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index 958eba02..1c01250b 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -275,7 +275,7 @@ def set_parameter(self, parameter, value): self._partition.actuator_indices[old_datatarget_index] = 0 if value not in self._nodenet.worldadapter_instance.get_available_datatargets(): - self.logger.warn("Datatarget %s not known in world adapter %s, whill not be assigned." % (value, self._nodenet.worldadapter)) + self.logger.warn("Datatarget %s not known in world adapter %s, will not be assigned." % (value, self._nodenet.worldadapter)) return datatarget_index = self._nodenet.worldadapter_instance.get_available_datatargets().index(value) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 5cf70cbc..10b0aaaf 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -349,7 +349,6 @@ def load(self, filename): with self.netlock: initfrom = {} - datafile = None if os.path.isfile(filename): try: self.logger.info("Loading nodenet %s metadata from file %s", self.name, filename) diff --git a/micropsi_core/world/world.py b/micropsi_core/world/world.py index 9cc9019f..85cdc2a7 100644 --- a/micropsi_core/world/world.py +++ b/micropsi_core/world/world.py @@ -120,7 +120,7 @@ def __init__(self, filename, world_type="", name="", owner="", uid=None, engine= self.agents = {} self.objects = {} - #self.the_image = None + # self.the_image = None self.load() From b1b8700cf3f6da2d802035736d1a58a8ce2be9be Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 18 Feb 2016 12:36:37 +0100 Subject: [PATCH 070/306] include modulators in sensor/actor-indices --- .../nodenet/theano_engine/theano_node.py | 20 +++--- .../nodenet/theano_engine/theano_nodenet.py | 61 ++++++++++++------- .../nodenet/theano_engine/theano_partition.py | 4 +- 3 files changed, 53 insertions(+), 32 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index 1c01250b..577c9926 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -247,16 +247,17 @@ def set_parameter(self, parameter, value): else: value = None if self.type == "Sensor" and parameter == "datasource": - if value is not None and value != "" and self._nodenet.worldadapter_instance is not None: + if value is not None and value != "": + datasources = self._nodenet.get_datasources() sensor_element = self._partition.allocated_node_offsets[self._id] + GEN old_datasource_index = np.where(self._partition.sensor_indices == sensor_element)[0] self._partition.sensor_indices[old_datasource_index] = 0 - if value not in self._nodenet.worldadapter_instance.get_available_datasources(): - self.logger.warn("Datasource %s not known in world adapter %s, will not be assigned." % (value, self._nodenet.worldadapter)) + if value not in datasources: + self.logger.warn("Datasource %s not known, will not be assigned." % value) return - datasource_index = self._nodenet.worldadapter_instance.get_available_datasources().index(value) + datasource_index = datasources.index(value) if self._partition.sensor_indices[datasource_index] != sensor_element and \ self._partition.sensor_indices[datasource_index] > 0: @@ -269,16 +270,17 @@ def set_parameter(self, parameter, value): self._nodenet.sensormap[value] = self.uid self._partition.sensor_indices[datasource_index] = sensor_element elif self.type == "Actor" and parameter == "datatarget": - if value is not None and value != "" and self._nodenet.worldadapter_instance is not None: + if value is not None and value != "": + datatargets = self._nodenet.get_datatargets() actuator_element = self._partition.allocated_node_offsets[self._id] + GEN old_datatarget_index = np.where(self._partition.actuator_indices == actuator_element)[0] - self._partition.actuator_indices[old_datatarget_index] = 0 - if value not in self._nodenet.worldadapter_instance.get_available_datatargets(): - self.logger.warn("Datatarget %s not known in world adapter %s, will not be assigned." % (value, self._nodenet.worldadapter)) + if value not in datatargets: + import pdb; pdb.set_trace() + self.logger.warn("Datatarget %s not known, will not be assigned." % value) return - datatarget_index = self._nodenet.worldadapter_instance.get_available_datatargets().index(value) + datatarget_index = datatargets.index(value) if self._partition.actuator_indices[datatarget_index] != actuator_element and \ self._partition.actuator_indices[datatarget_index] > 0: diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 10b0aaaf..7a50ca67 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -173,27 +173,7 @@ def worldadapter_instance(self): @worldadapter_instance.setter def worldadapter_instance(self, _worldadapter_instance): self._worldadapter_instance = _worldadapter_instance - - for partition in self.partitions.values(): - if _worldadapter_instance is None: - partition.actuator_indices = np.zeros(0, np.int32) - partition.sensor_indices = np.zeros(0, np.int32) - continue - - partition.actuator_indices = np.zeros(len(_worldadapter_instance.get_available_datatargets()), np.int32) - partition.sensor_indices = np.zeros(len(_worldadapter_instance.get_available_datasources()), np.int32) - - for datatarget, node_id in self.actuatormap.items(): - if not isinstance(node_id, str): - node_id = node_id[0] - if self.get_partition(node_id) == partition: - self.get_node(node_id).set_parameter("datatarget", datatarget) - - for datasource, node_id in self.sensormap.items(): - if not isinstance(node_id, str): - node_id = node_id[0] - if self.get_partition(node_id) == partition: - self.get_node(node_id).set_parameter("datasource", datasource) + self._rebuild_sensor_actor_indices() @property def current_step(self): @@ -1317,6 +1297,45 @@ def read_actuators(self): return actuator_values_to_write + def _rebuild_sensor_actor_indices(self): + """ + Rebuilds the actor and sensor indices for all partitions + """ + for partition in self.partitions.values(): + partition.sensor_indices = np.zeros(len(self.get_datasources()), np.int32) + partition.actuator_indices = np.zeros(len(self.get_datatargets()), np.int32) + for datatarget, node_id in self.actuatormap.items(): + if not isinstance(node_id, str): + node_id = node_id[0] + if self.get_partition(node_id) == partition: + self.get_node(node_id).set_parameter("datatarget", datatarget) + + for datasource, node_id in self.sensormap.items(): + if not isinstance(node_id, str): + node_id = node_id[0] + if self.get_partition(node_id) == partition: + self.get_node(node_id).set_parameter("datasource", datasource) + + def get_datasources(self): + """ Returns a sorted list of available datasources, including worldadapter datasources + and readable modulators""" + modulators = sorted(self._modulators.keys()) + datasources = self.worldadapter_instance.get_available_datasources() if self.worldadapter_instance else [] + for item in modulators: + if item in DoernerianEmotionalModulators.readable_modulators: + datasources.append(item) + return datasources + + def get_datatargets(self): + """ Returns a sorted list of available datatargets, including worldadapter datatargets + and writeable modulators""" + modulators = sorted(self._modulators.keys()) + datatargets = self.worldadapter_instance.get_available_datatargets() if self.worldadapter_instance else [] + for item in modulators: + if item not in DoernerianEmotionalModulators.readable_modulators: + datatargets.append(item) + return datatargets + def group_nodes_by_names(self, nodespace_uid, node_name_prefix=None, gatetype="gen", sortby='id', group_name=None): if nodespace_uid is None: nodespace_uid = self.get_nodespace(None).uid diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index 1bad7554..2bc76f5c 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -237,8 +237,8 @@ def __init__(self, nodenet, pid, sparse=True, initial_number_of_nodes=2000, aver self.allocated_elements_to_activators = np.zeros(self.NoE, dtype=np.int32) - self.sensor_indices = np.zeros(0, dtype=np.int32) - self.actuator_indices = np.zeros(0, dtype=np.int32) + self.sensor_indices = np.zeros(0, dtype=np.int32) # index := datasource, value:=node_id + self.actuator_indices = np.zeros(0, dtype=np.int32) # index := datatarget, value:=node_id self.inlinks = {} From add4b174aacc58002037efa994ff0b2461f4e505 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 18 Feb 2016 12:37:14 +0100 Subject: [PATCH 071/306] fix sensor/actor deletion --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 7a50ca67..8846dffc 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -709,11 +709,11 @@ def delete_node(self, uid): # remove sensor association if there should be one if uid in self.sensormap.values(): - self.sensormap = {k:v for k, v in self.sensormap.items() if v is not uid} + self.sensormap = {k: v for k, v in self.sensormap.items() if v != uid} # remove actuator association if there should be one if uid in self.actuatormap.values(): - self.actuatormap = {k:v for k, v in self.actuatormap.items() if v is not uid} + self.actuatormap = {k: v for k, v in self.actuatormap.items() if v != uid} self.clear_supplements(uid) From 01f457d2c4a64aaf70a88884af9cf719298a352a Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 18 Feb 2016 17:50:32 +0100 Subject: [PATCH 072/306] fix ambiguous por-ret-decay modulator --- micropsi_core/nodenet/netapi.py | 8 ++++---- micropsi_core/nodenet/nodenet.py | 1 - micropsi_core/nodenet/stepoperators.py | 3 ++- micropsi_core/nodenet/theano_engine/theano_netapi.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/micropsi_core/nodenet/netapi.py b/micropsi_core/nodenet/netapi.py index 3eec25c7..57a2bda8 100644 --- a/micropsi_core/nodenet/netapi.py +++ b/micropsi_core/nodenet/netapi.py @@ -539,15 +539,15 @@ def set_dashboard_value(self, name, value): def decay_por_links(self, nodespace_uid): """ Decayes all por-links in the given nodespace """ - decay_factor = self.__nodenet.get_modulator('base_porret_decay_factor') + porretdecay = self.__nodenet.get_modulator('base_porret_decay_factor') nodes = self.get_nodes(nodespace=nodespace_uid, nodetype="Pipe") - pordecay = (1 - self.__nodenet.get_modulator('por_ret_decay')) - if decay_factor and pordecay is not None and pordecay > 0: + decay_factor = (1 - porretdecay) + if porretdecay != 0: for node in nodes: porgate = node.get_gate('por') for link in porgate.get_links(): if link.weight > 0: - link._set_weight(max(link.weight * pordecay, 0)) + link._set_weight(max(link.weight * decay_factor, 0)) def announce_nodes(self, nodespace_uid, numer_of_nodes, average_element_per_node): pass diff --git a/micropsi_core/nodenet/nodenet.py b/micropsi_core/nodenet/nodenet.py index 766b4a79..8e8e73ec 100644 --- a/micropsi_core/nodenet/nodenet.py +++ b/micropsi_core/nodenet/nodenet.py @@ -170,7 +170,6 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No self._modulators = {} if use_modulators: - self._modulators['por_ret_decay'] = 0. from micropsi_core.nodenet.stepoperators import DoernerianEmotionalModulators as emo for modulator in emo.writeable_modulators + emo.readable_modulators: self._modulators[modulator] = 1 diff --git a/micropsi_core/nodenet/stepoperators.py b/micropsi_core/nodenet/stepoperators.py index 9b1f9fb5..c04eb663 100644 --- a/micropsi_core/nodenet/stepoperators.py +++ b/micropsi_core/nodenet/stepoperators.py @@ -98,7 +98,8 @@ class DoernerianEmotionalModulators(StepOperator): 'base_number_of_expected_events', 'base_number_of_unexpected_events', 'base_urge_change', - 'base_age_influence_on_competence'] + 'base_age_influence_on_competence', + 'base_porret_decay_factor'] readable_modulators = [ 'emo_pleasure', 'emo_activation', diff --git a/micropsi_core/nodenet/theano_engine/theano_netapi.py b/micropsi_core/nodenet/theano_engine/theano_netapi.py index ca5aa5e1..521f0468 100644 --- a/micropsi_core/nodenet/theano_engine/theano_netapi.py +++ b/micropsi_core/nodenet/theano_engine/theano_netapi.py @@ -54,7 +54,7 @@ def decay_por_links(self, nodespace_uid): # self.decay = theano.function([por_cols, por_rows], None, updates={nodenet.w: new_w}, accept_inplace=True) import numpy as np from .theano_definitions import node_from_id, PIPE, POR - porretdecay = self.__nodenet.get_modulator('por_ret_decay') + porretdecay = self.__nodenet.get_modulator('base_porret_decay_factor') ns = self.get_nodespace(nodespace_uid) partition = ns._partition if partition.has_pipes and porretdecay != 0: From 5af2c33564161fce98ef8ac7a2c085b8f9557ce9 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 18 Feb 2016 17:51:40 +0100 Subject: [PATCH 073/306] squash --- micropsi_core/nodenet/theano_engine/theano_node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index 577c9926..fe1f6331 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -276,7 +276,6 @@ def set_parameter(self, parameter, value): old_datatarget_index = np.where(self._partition.actuator_indices == actuator_element)[0] self._partition.actuator_indices[old_datatarget_index] = 0 if value not in datatargets: - import pdb; pdb.set_trace() self.logger.warn("Datatarget %s not known, will not be assigned." % value) return From de490bae4629d006b299f6b2339079ca7fac0528 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 18 Feb 2016 17:52:02 +0100 Subject: [PATCH 074/306] warn is deprecated --- micropsi_core/nodenet/theano_engine/theano_node.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index fe1f6331..2965dda8 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -254,7 +254,7 @@ def set_parameter(self, parameter, value): self._partition.sensor_indices[old_datasource_index] = 0 if value not in datasources: - self.logger.warn("Datasource %s not known, will not be assigned." % value) + self.logger.warning("Datasource %s not known, will not be assigned." % value) return datasource_index = datasources.index(value) @@ -265,7 +265,7 @@ def set_parameter(self, parameter, value): other_sensor_element = self._partition.sensor_indices[datasource_index] other_sensor_id = node_to_id(self._partition.allocated_elements_to_nodes[other_sensor_element], self._partition.pid) - self.logger.warn("Datasource %s had already been assigned to sensor %s, which will now be unassigned." % (value, other_sensor_id)) + self.logger.warning("Datasource %s had already been assigned to sensor %s, which will now be unassigned." % (value, other_sensor_id)) self._nodenet.sensormap[value] = self.uid self._partition.sensor_indices[datasource_index] = sensor_element @@ -276,7 +276,7 @@ def set_parameter(self, parameter, value): old_datatarget_index = np.where(self._partition.actuator_indices == actuator_element)[0] self._partition.actuator_indices[old_datatarget_index] = 0 if value not in datatargets: - self.logger.warn("Datatarget %s not known, will not be assigned." % value) + self.logger.warning("Datatarget %s not known, will not be assigned." % value) return datatarget_index = datatargets.index(value) @@ -287,7 +287,7 @@ def set_parameter(self, parameter, value): other_actuator_element = self._partition.actuator_indices[datatarget_index] other_actuator_id = node_to_id(self._partition.allocated_elements_to_nodes[other_actuator_element], self._partition.pid) - self.logger.warn("Datatarget %s had already been assigned to actuator %s, which will now be unassigned." % (value, other_actuator_id)) + self.logger.warning("Datatarget %s had already been assigned to actuator %s, which will now be unassigned." % (value, other_actuator_id)) self._nodenet.actuatormap[value] = self.uid self._partition.actuator_indices[datatarget_index] = actuator_element From 82f842e13d901b3a7fb10962822b565b475f3b40 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 18 Feb 2016 17:52:18 +0100 Subject: [PATCH 075/306] document test_fixtures, use empty nodenet if the test does not use the test-content --- conftest.py | 9 ++++++ micropsi_core/tests/conftest.py | 17 ++++++++++ micropsi_core/tests/test_node_logic.py | 44 +++++++++++++------------- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/conftest.py b/conftest.py index ec3e6998..732a800b 100644 --- a/conftest.py +++ b/conftest.py @@ -125,6 +125,9 @@ def recipes_def(): @pytest.fixture(scope="function") def test_world(request): + """ + Fixture: A test world of type Island + """ global world_uid worlds = micropsi.get_available_worlds("Pytest User") if world_uid not in worlds: @@ -141,6 +144,9 @@ def fin(): @pytest.fixture(scope="function") def test_nodenet(request, test_world, engine): + """ + Fixture: A completely empty nodenet without a worldadapter + """ global nn_uid nodenets = micropsi.get_available_nodenets("Pytest User") or {} if nn_uid not in nodenets: @@ -153,6 +159,9 @@ def test_nodenet(request, test_world, engine): @pytest.fixture(scope="function") def node(request, test_nodenet): + """ + Fixture: A Pipe node with a genloop + """ res, uid = micropsi.add_node(test_nodenet, 'Pipe', [10, 10, 10], name='N1') micropsi.add_link(test_nodenet, uid, 'gen', uid, 'gen') return uid diff --git a/micropsi_core/tests/conftest.py b/micropsi_core/tests/conftest.py index 6a5629cb..1d142fb9 100644 --- a/micropsi_core/tests/conftest.py +++ b/micropsi_core/tests/conftest.py @@ -12,6 +12,23 @@ @pytest.fixture(scope="function") def fixed_nodenet(request, test_world, engine): + """ + A test nodenet filled with some example data (nodenet_data.py) + Structure: + + -> A1 -> A2 + / + S ACTA + \ + -> B1 -> B2 + + S: Sensor, brightness_l + A1: Pipe + A2: Pipe + B1: Pipe + B2: Pipe + ACTA: Activator, por + """ from micropsi_core.tests.nodenet_data import fixed_nodenet_data if engine == "theano_engine": fixed_nodenet_data = fixed_nodenet_data.replace('Root', 's0001') diff --git a/micropsi_core/tests/test_node_logic.py b/micropsi_core/tests/test_node_logic.py index 06706b6e..84036caa 100644 --- a/micropsi_core/tests/test_node_logic.py +++ b/micropsi_core/tests/test_node_logic.py @@ -32,8 +32,8 @@ def update_data_sources_and_targets(self): self.world.test_target_value = self.datatargets['test_target'] -def prepare(fixed_nodenet): - nodenet = micropsi.get_nodenet(fixed_nodenet) +def prepare(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) netapi = nodenet.netapi source = netapi.create_node("Register", None, "Source") netapi.link(source, "gen", source, "gen") @@ -42,8 +42,8 @@ def prepare(fixed_nodenet): return nodenet, netapi, source -def add_dummyworld(fixed_nodenet): - nodenet = micropsi.get_nodenet(fixed_nodenet) +def add_dummyworld(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) if nodenet.world: micropsi.worlds[nodenet.world].unregister_nodenet(nodenet) @@ -55,9 +55,9 @@ def add_dummyworld(fixed_nodenet): return micropsi.worlds[worlduid] -def test_node_logic_loop(fixed_nodenet): +def test_node_logic_loop(test_nodenet): # test gen looping behaviour - net, netapi, source = prepare(fixed_nodenet) + net, netapi, source = prepare(test_nodenet) net.step() assert source.get_gate("gen").activation == 1 net.step() @@ -67,18 +67,18 @@ def test_node_logic_loop(fixed_nodenet): assert source.get_gate("gen").activation == 0.5 -def test_node_logic_die(fixed_nodenet): +def test_node_logic_die(test_nodenet): # without the link, activation ought to drop to 0 - net, netapi, source = prepare(fixed_nodenet) + net, netapi, source = prepare(test_nodenet) netapi.unlink(source, "gen", source, "gen") net.step() assert source.get_gate("gen").activation == 0 -def test_node_logic_sum(fixed_nodenet): +def test_node_logic_sum(test_nodenet): # propagate positive activation, expect sum - net, netapi, source = prepare(fixed_nodenet) + net, netapi, source = prepare(test_nodenet) reg_a = netapi.create_node("Register", None, "RegA") reg_b = netapi.create_node("Register", None, "RegB") @@ -94,9 +94,9 @@ def test_node_logic_sum(fixed_nodenet): assert reg_result.get_gate("gen").activation == 1 -def test_node_logic_cancel(fixed_nodenet): +def test_node_logic_cancel(test_nodenet): # propagate positive and negative activation, expect cancellation - net, netapi, source = prepare(fixed_nodenet) + net, netapi, source = prepare(test_nodenet) reg_a = netapi.create_node("Register", None, "RegA") reg_b = netapi.create_node("Register", None, "RegB") @@ -113,9 +113,9 @@ def test_node_logic_cancel(fixed_nodenet): assert reg_result.get_gate("gen").activation == 0 -def test_node_logic_store_and_forward(fixed_nodenet): +def test_node_logic_store_and_forward(test_nodenet): # collect activation in one node, go forward only if both dependencies are met - net, netapi, source = prepare(fixed_nodenet) + net, netapi, source = prepare(test_nodenet) reg_a = netapi.create_node("Register", None, "RegA") reg_b = netapi.create_node("Register", None, "RegB") @@ -134,8 +134,8 @@ def test_node_logic_store_and_forward(fixed_nodenet): assert reg_result.get_gate("gen").activation == 1 -def test_node_logic_activators(fixed_nodenet): - net, netapi, source = prepare(fixed_nodenet) +def test_node_logic_activators(test_nodenet): + net, netapi, source = prepare(test_nodenet) activator = netapi.create_node('Activator', None) activator.set_parameter('type', 'sub') activator.activation = 1 @@ -147,10 +147,10 @@ def test_node_logic_activators(fixed_nodenet): assert testpipe.get_gate("sub").activation == 0 -def test_node_logic_sensor(fixed_nodenet): +def test_node_logic_sensor(test_nodenet): # read a sensor value from the dummy world adapter - net, netapi, source = prepare(fixed_nodenet) - world = add_dummyworld(fixed_nodenet) + net, netapi, source = prepare(test_nodenet) + world = add_dummyworld(test_nodenet) register = netapi.create_node("Register", None) netapi.link_sensor(register, "test_source", "gen") @@ -160,10 +160,10 @@ def test_node_logic_sensor(fixed_nodenet): assert round(register.get_gate("gen").activation, 1) == 0.7 -def test_node_logic_actor(fixed_nodenet): +def test_node_logic_actor(test_nodenet): # write a value to the dummy world adapter - net, netapi, source = prepare(fixed_nodenet) - world = add_dummyworld(fixed_nodenet) + net, netapi, source = prepare(test_nodenet) + world = add_dummyworld(test_nodenet) register = netapi.create_node("Register", None) netapi.link_actor(source, "test_target", 0.5, 1, "gen", "gen") From 774636ee9805a3f28ec206a0bee0dba82d15bb04 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 18 Feb 2016 17:52:57 +0100 Subject: [PATCH 076/306] Do not unset the worldadapter when unloading nodenets If we unset the worldadapter, the nodenet's sensors all will complain that their datasource is gone. Since we're unloading or even deleting the nodenet anyway, there's no harm in not removing it, the garbage collector will sooner or later delete the whole thing from memory. --- micropsi_core/world/world.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/micropsi_core/world/world.py b/micropsi_core/world/world.py index 85cdc2a7..60d504c8 100644 --- a/micropsi_core/world/world.py +++ b/micropsi_core/world/world.py @@ -261,9 +261,8 @@ def unregister_nodenet(self, nodenet): if nodenet.uid in self.agents: # stop corresponding nodenet micropsi_core.runtime.stop_nodenetrunner(nodenet.uid) - # remove agent - nodenet.worldadapter_instance = None + # nodenet.worldadapter_instance = None del self.agents[nodenet.uid] if nodenet.uid in self.data['agents']: del self.data['agents'][nodenet.uid] @@ -282,7 +281,6 @@ def spawn_agent(self, worldadapter_name, nodenet, **options): self.logger.error("World %s does not support Worldadapter %s" % (self.name, worldadapter_name)) return False, "World %s does not support Worldadapter %s" % (self.name, worldadapter_name) - def set_object_properties(self, uid, position=None, orientation=None, name=None, parameters=None): """set attributes of the world object 'uid'; only supplied attributes will be changed. From 333fdf313f3add94b8205a429a6fb572e40fb454 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 18 Feb 2016 18:03:46 +0100 Subject: [PATCH 077/306] use complete datasources and targets in clone_parameters() --- micropsi_core/nodenet/theano_engine/theano_node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index 2965dda8..640ac48a 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -324,14 +324,14 @@ def clone_parameters(self): if len(datasource_index) == 0: parameters['datasource'] = None else: - parameters['datasource'] = self._nodenet.worldadapter_instance.get_available_datasources()[datasource_index] + parameters['datasource'] = self._nodenet.get_datasources()[datasource_index] elif self.type == "Actor": actuator_element = self._partition.allocated_node_offsets[self._id] + GEN datatarget_index = np.where(self._partition.actuator_indices == actuator_element)[0] if len(datatarget_index) == 0: parameters['datatarget'] = None else: - parameters['datatarget'] = self._nodenet.worldadapter_instance.get_available_datatargets()[datatarget_index] + parameters['datatarget'] = self._nodenet.get_datatargets()[datatarget_index] elif self.type == "Activator": activator_type = None if self._id in self._partition.allocated_nodespaces_por_activators: From bc1160c318257fab983849e6c74b0d9b6b47150d Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 18 Feb 2016 18:04:22 +0100 Subject: [PATCH 078/306] this seems to be unused now --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 8846dffc..9799200a 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -360,10 +360,6 @@ def load(self, filename): # was saved). self.reload_native_modules(self.native_module_definitions) - for actuator, id_list in self.actuatormap.items(): - for id in id_list: - self.inverted_actuator_map[id] = actuator - # re-initialize step operators for theano recompile to new shared variables self.initialize_stepoperators() From 1fa640be29f5d1b461e8629390d103a5fc8aba1f Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Fri, 19 Feb 2016 17:33:26 +0100 Subject: [PATCH 079/306] Avoiding ugly exceptions on missing data file --- micropsi_core/world/timeseries/timeseries.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 44259362..497787ff 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -40,6 +40,9 @@ def __init__(self, filename, world_type="Island", name="", owner="", engine=None self.enddate = f['enddate'] except IOError as error: self.logger.error("Could not load data file %s, error was: %s" % (path, str(error))) + self.ids = [0] + self.timeseries[[0, 0, 0]] + self.len_ts = 1 return # todo use the new configurable world options. From 9e5dfb3e70ad94cb8b11f71414624248510ac9b7 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 22 Feb 2016 19:04:52 +0100 Subject: [PATCH 080/306] initialize runtime explicitly --- micropsi_core/config.py | 3 +- micropsi_core/runtime.py | 87 ++++++++++--------- .../tests/test_configurationManager.py | 3 +- micropsi_server/micropsi_app.py | 2 + 4 files changed, 52 insertions(+), 43 deletions(-) diff --git a/micropsi_core/config.py b/micropsi_core/config.py index 0facde86..3ce9571e 100644 --- a/micropsi_core/config.py +++ b/micropsi_core/config.py @@ -51,7 +51,8 @@ def __init__(self, config_path="config-data.json", auto_save=True): # check if we already have a configuration manager with this resource file absolute_path = os.path.abspath(config_path) if absolute_path in ConfigurationManager.instances: - raise RuntimeError("A configuration manager with this resource path already exists!") + logging.getLogger("system").warning("A configuration manager with this resource path already exists!") + # raise RuntimeError("A configuration manager with this resource path already exists!") ConfigurationManager.instances.append(absolute_path) self.key = absolute_path diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index f9e14c9c..edd56a47 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -35,33 +35,23 @@ NODENET_DIRECTORY = "nodenets" WORLD_DIRECTORY = "worlds" -RESOURCE_PATH = cfg['paths']['resource_path'] -sys.path.append(RESOURCE_PATH) +signal_handler_registry = [] +runner = {'timestep': 1000, 'runner': None, 'factor': 1} + +nodenet_lock = threading.Lock() + +# global variables set by intialize() +RESOURCE_PATH = None -configs = config.ConfigurationManager(cfg['paths']['server_settings_path']) +configs = None +logger = None worlds = {} nodenets = {} native_modules = {} custom_recipes = {} -runner = {'timestep': 1000, 'runner': None, 'factor': 1} - -signal_handler_registry = [] - -logger = MicropsiLogger({ - 'system': cfg['logging']['level_system'], - 'world': cfg['logging']['level_world'] -}, cfg['logging'].get('logfile')) - -nodenet_lock = threading.Lock() - -if cfg['micropsi2'].get('profile_runner'): - import cProfile - import pstats - import io - def add_signal_handler(handler): signal_handler_registry.append(handler) @@ -84,6 +74,7 @@ class MicropsiRunner(threading.Thread): def __init__(self): threading.Thread.__init__(self) if cfg['micropsi2'].get('profile_runner'): + import cProfile self.profiler = cProfile.Profile() else: self.profiler = None @@ -144,6 +135,8 @@ def run(self): average_duration = self.sum_of_durations / self.number_of_samples if self.total_steps % self.granularity == 0: if self.profiler: + import pstats + import io s = io.StringIO() sortby = 'cumtime' ps = pstats.Stats(self.profiler, stream=s).sort_stats(sortby) @@ -1403,30 +1396,44 @@ def reload_native_modules(): else: return False, errors -native_modules = {} -custom_recipes = {} -load_definitions() -init_worlds(world_data) -result, errors = reload_native_modules() -for e in errors: - logging.getLogger("system").error(e) +def initialize(): + global RESOURCE_PATH, configs, logger, runner + + RESOURCE_PATH = cfg['path']['resource_path'] + + sys.path.append(resource_path) + + configs = config.ConfigurationManager(cfg['paths']['server_settings_path']) + + if logger is None: + logger = MicropsiLogger({ + 'system': cfg['logging']['level_system'], + 'world': cfg['logging']['level_world'] + }, cfg['logging'].get('logfile')) + + load_definitions() + init_worlds(world_data) + result, errors = reload_native_modules() + for e in errors: + logging.getLogger("system").error(e) -# initialize runners -# Initialize the threads for the continuous simulation of nodenets and worlds -if 'runner_timestep' not in configs: - configs['runner_timestep'] = 200 - configs.save_configs() -if 'runner_factor' not in configs: - configs['runner_factor'] = 2 - configs.save_configs() + # initialize runners + # Initialize the threads for the continuous simulation of nodenets and worlds + if 'runner_timestep' not in configs: + configs['runner_timestep'] = 200 + configs.save_configs() + if 'runner_factor' not in configs: + configs['runner_factor'] = 2 + configs.save_configs() -set_runner_properties(configs['runner_timestep'], configs['runner_factor']) + set_runner_properties(configs['runner_timestep'], configs['runner_factor']) -runner['running'] = True -runner['runner'] = MicropsiRunner() + runner['running'] = True + runner['runner'] = MicropsiRunner() -add_signal_handler(kill_runners) + if kill_runners not in signal_handler_registry: + add_signal_handler(kill_runners) -signal.signal(signal.SIGINT, signal_handler) -signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) diff --git a/micropsi_core/tests/test_configurationManager.py b/micropsi_core/tests/test_configurationManager.py index 5a4cd824..f6a84827 100644 --- a/micropsi_core/tests/test_configurationManager.py +++ b/micropsi_core/tests/test_configurationManager.py @@ -64,5 +64,4 @@ def test_create_configs(path, path2): assert conf_mgr["record"]["i"] == 12 assert conf2["color"] == "blue" - with pytest.raises(RuntimeError): - ConfigurationManager(path) # we cannot have more than one config manager at a single path + ConfigurationManager(path) # we can now have more than one config manager at a single path diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index 75f5aaea..ea81c37c 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -28,6 +28,8 @@ from micropsi_server import minidoc import logging +runtime.initialize() + from configuration import config as cfg VERSION = cfg['micropsi2']['version'] From 007a44ced0b86d4ea1a3ab97167230eb60814e70 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 22 Feb 2016 19:09:32 +0100 Subject: [PATCH 081/306] separate persistency and ressource paths if need be --- configuration.py | 2 +- micropsi_core/_runtime_api_world.py | 6 +++--- micropsi_core/nodenet/node.py | 1 - micropsi_core/runtime.py | 30 ++++++++++++++++++----------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/configuration.py b/configuration.py index aa01aa8a..9648565d 100644 --- a/configuration.py +++ b/configuration.py @@ -43,6 +43,6 @@ config['logging'][level] = 'WARNING' config.add_section('paths') -config['paths']['resource_path'] = os.path.join(os.path.dirname(__file__), data_path) +config['paths']['data_directory'] = os.path.join(os.path.dirname(__file__), data_path) config['paths']['usermanager_path'] = os.path.join(os.path.dirname(__file__), 'resources', 'user-db.json') config['paths']['server_settings_path'] = os.path.join(os.path.dirname(__file__), 'resources', 'server-config.json') diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index e18fd03c..f008d8f7 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -108,7 +108,7 @@ def new_world(world_name, world_type, owner="", uid=None, config={}): if micropsi_core.runtime.worlds[uid].__class__.__name__.startswith('Minecraft'): raise RuntimeError("Only one instance of a minecraft world is supported right now") - filename = os.path.join(micropsi_core.runtime.RESOURCE_PATH, micropsi_core.runtime.WORLD_DIRECTORY, uid + ".json") + filename = os.path.join(micropsi_core.runtime.PERSISTENCY_PATH, micropsi_core.runtime.WORLD_DIRECTORY, uid + ".json") micropsi_core.runtime.world_data[uid] = Bunch(uid=uid, name=world_name, world_type=world_type, filename=filename, version=1, owner=owner, config=config) @@ -169,7 +169,7 @@ def revert_world(world_uid): def save_world(world_uid): """Stores the world state on the server.""" - with open(os.path.join(micropsi_core.runtime.RESOURCE_PATH, micropsi_core.runtime.WORLD_DIRECTORY, + with open(os.path.join(micropsi_core.runtime.PERSISTENCY_PATH, micropsi_core.runtime.WORLD_DIRECTORY, world_uid) + '.json', 'w+') as fp: fp.write(json.dumps(micropsi_core.runtime.worlds[world_uid].data, sort_keys=True, indent=4)) return True @@ -190,7 +190,7 @@ def import_world(worlddata, owner=None): raise RuntimeError("A world with this ID already exists.") if owner is not None: data['owner'] = owner - filename = os.path.join(micropsi_core.runtime.RESOURCE_PATH, micropsi_core.runtime.WORLD_DIRECTORY, data['uid'] + '.json') + filename = os.path.join(micropsi_core.runtime.PERSISTENCY_PATH, micropsi_core.runtime.WORLD_DIRECTORY, data['uid'] + '.json') data['filename'] = filename with open(filename, 'w+') as fp: fp.write(json.dumps(data)) diff --git a/micropsi_core/nodenet/node.py b/micropsi_core/nodenet/node.py index 9d6fb475..84f01a0e 100644 --- a/micropsi_core/nodenet/node.py +++ b/micropsi_core/nodenet/node.py @@ -595,7 +595,6 @@ def nodefunction_name(self): def nodefunction_name(self, nodefunction_name): import os from importlib.machinery import SourceFileLoader - from micropsi_core.runtime import RESOURCE_PATH self._nodefunction_name = nodefunction_name try: if self.path: diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index edd56a47..dff60e48 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -43,6 +43,7 @@ # global variables set by intialize() RESOURCE_PATH = None +PERSISTENCY_PATH = None configs = None logger = None @@ -302,7 +303,7 @@ def load_nodenet(nodenet_uid): nodenet_lock.release() return False, "Nodenet %s requires unknown engine %s" % (nodenet_uid, engine) - nodenets[nodenet_uid].load(os.path.join(RESOURCE_PATH, NODENET_DIRECTORY, nodenet_uid + ".json")) + nodenets[nodenet_uid].load(os.path.join(PERSISTENCY_PATH, NODENET_DIRECTORY, nodenet_uid + ".json")) if "settings" in data: nodenets[nodenet_uid].settings = data["settings"].copy() @@ -316,7 +317,7 @@ def load_nodenet(nodenet_uid): nodenet_lock.release() return True, nodenet_uid - return False, "Nodenet %s not found in %s" % (nodenet_uid, RESOURCE_PATH) + return False, "Nodenet %s not found in %s" % (nodenet_uid, PERSISTENCY_PATH) def get_nodenet_data(nodenet_uid, nodespace, step=0, include_links=True): @@ -437,7 +438,7 @@ def new_nodenet(nodenet_name, engine="dict_engine", worldadapter=None, template= engine=engine, use_modulators=use_modulators) - filename = os.path.join(RESOURCE_PATH, NODENET_DIRECTORY, data['uid'] + ".json") + filename = os.path.join(PERSISTENCY_PATH, NODENET_DIRECTORY, data['uid'] + ".json") nodenet_data[data['uid']] = Bunch(**data) load_nodenet(data['uid']) if template is not None and template in nodenet_data: @@ -456,7 +457,7 @@ def delete_nodenet(nodenet_uid): Simple unloading is maintained automatically when a nodenet is suspended and another one is accessed. """ - filename = os.path.join(RESOURCE_PATH, NODENET_DIRECTORY, nodenet_uid + '.json') + filename = os.path.join(PERSISTENCY_PATH, NODENET_DIRECTORY, nodenet_uid + '.json') nodenet = get_nodenet(nodenet_uid) nodenet.remove(filename) unload_nodenet(nodenet_uid) @@ -575,7 +576,7 @@ def revert_nodenet(nodenet_uid, also_revert_world=False): def save_nodenet(nodenet_uid): """Stores the nodenet on the server (but keeps it open).""" nodenet = nodenets[nodenet_uid] - nodenet.save(os.path.join(RESOURCE_PATH, NODENET_DIRECTORY, nodenet_uid + '.json')) + nodenet.save(os.path.join(PERSISTENCY_PATH, NODENET_DIRECTORY, nodenet_uid + '.json')) nodenet_data[nodenet_uid] = Bunch(**nodenet.metadata) return True @@ -605,7 +606,7 @@ def import_nodenet(string, owner=None): if 'owner': import_data['owner'] = owner # assert import_data['world'] in worlds - filename = os.path.join(RESOURCE_PATH, NODENET_DIRECTORY, import_data['uid'] + '.json') + filename = os.path.join(PERSISTENCY_PATH, NODENET_DIRECTORY, import_data['uid'] + '.json') with open(filename, 'w+') as fp: fp.write(json.dumps(import_data)) nodenet_data[import_data['uid']] = parse_definition(import_data, filename) @@ -1273,7 +1274,7 @@ def init_worlds(world_data): return worlds -def load_user_files(path=RESOURCE_PATH, reload_nodefunctions=False, errors=[]): +def load_user_files(path, reload_nodefunctions=False, errors=[]): global native_modules, custom_recipes for f in os.listdir(path): if not f.startswith('.') and f != '__pycache__': @@ -1384,7 +1385,7 @@ def reload_native_modules(): if nodenets[uid].is_active: runners[uid] = True nodenets[uid].is_active = False - errors = load_user_files(reload_nodefunctions=True, errors=[]) + errors = load_user_files(RESOURCE_PATH, reload_nodefunctions=True, errors=[]) for nodenet_uid in nodenets: nodenets[nodenet_uid].reload_native_modules(filter_native_modules(nodenets[nodenet_uid].engine)) # restart previously active nodenets @@ -1397,10 +1398,17 @@ def reload_native_modules(): return False, errors -def initialize(): - global RESOURCE_PATH, configs, logger, runner +def initialize(persistency_path=None, resource_path=None): + global PERSISTENCY_PATH, RESOURCE_PATH, configs, logger, runner - RESOURCE_PATH = cfg['path']['resource_path'] + if persistency_path is None: + persistency_path = cfg['paths']['data_directory'] + + if resource_path is None: + resource_path = persistency_path + + PERSISTENCY_PATH = persistency_path + RESOURCE_PATH = resource_path sys.path.append(resource_path) From 7b57b7ea716caebb2f854a8edfc3245922aa7dc2 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 22 Feb 2016 19:14:33 +0100 Subject: [PATCH 082/306] adjust conftest to new path handling --- conftest.py | 37 ++++++++++++++++++------------- micropsi_server/tests/conftest.py | 2 +- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/conftest.py b/conftest.py index 96d95b6b..31dfba4e 100644 --- a/conftest.py +++ b/conftest.py @@ -13,16 +13,20 @@ pass # override config -from configuration import config -config['paths']['resource_path'] = testpath -config['paths']['server_settings_path'] = os.path.join(config['paths']['resource_path'], 'server_config.json') -config['paths']['usermanager_path'] = os.path.join(config['paths']['resource_path'], 'user-db.json') -config['micropsi2']['single_agent_mode'] = '' -if 'theano' in config: - config['theano']['initial_number_of_nodes'] = '50' - from micropsi_core import runtime as micropsi +from micropsi_core.runtime import cfg +original_ini_data_directory = cfg['paths']['data_directory'] + +cfg['paths']['data_directory'] = testpath +cfg['paths']['server_settings_path'] = os.path.join(testpath, 'server_cfg.json') +cfg['paths']['usermanager_path'] = os.path.join(testpath, 'user-db.json') +cfg['micropsi2']['single_agent_mode'] = '' +if 'theano' in cfg: + cfg['theano']['initial_number_of_nodes'] = '50' + +micropsi.initialize() + # create testuser from micropsi_server.micropsi_app import usermanager @@ -82,12 +86,13 @@ def pytest_runtest_setup(item): micropsi.delete_nodenet(uid) for uid in list(micropsi.worlds.keys()): micropsi.delete_world(uid) - shutil.rmtree(config['paths']['resource_path']) - os.mkdir(config['paths']['resource_path']) - os.mkdir(os.path.join(config['paths']['resource_path'], 'nodenets')) - os.mkdir(os.path.join(config['paths']['resource_path'], 'worlds')) + shutil.rmtree(testpath) + os.mkdir(testpath) + os.mkdir(os.path.join(testpath, 'nodenets')) + os.mkdir(os.path.join(testpath, 'worlds')) # default native module container - os.mkdir(os.path.join(config['paths']['resource_path'], 'Test')) + os.mkdir(os.path.join(testpath, 'Test')) + open(os.path.join(testpath, 'Test', '__init__.py'), 'w').close() micropsi.reload_native_modules() micropsi.logger.clear_logs() set_logging_levels() @@ -95,7 +100,7 @@ def pytest_runtest_setup(item): @pytest.fixture(scope="session") def resourcepath(): - return config['paths']['resource_path'] + return cfg['paths']['data_directory'] @pytest.fixture(scope="function") @@ -131,9 +136,9 @@ def node(request, test_nodenet): def pytest_internalerror(excrepr, excinfo): """ called for internal errors. """ - shutil.rmtree(config['paths']['resource_path']) + shutil.rmtree(testpath) def pytest_keyboard_interrupt(excinfo): """ called for keyboard interrupt. """ - shutil.rmtree(config['paths']['resource_path']) + shutil.rmtree(testpath) diff --git a/micropsi_server/tests/conftest.py b/micropsi_server/tests/conftest.py index a80fe771..39ef6d45 100644 --- a/micropsi_server/tests/conftest.py +++ b/micropsi_server/tests/conftest.py @@ -9,7 +9,7 @@ from conftest import user_token from micropsi_server import usermanagement -test_path = os.path.join(config['paths']['resource_path'], 'user-test-db.json') +test_path = os.path.join(config['paths']['data_directory'], 'user-test-db.json') class MicropsiTestApp(TestApp): From f9a120934103f542240317c7717b60a3b8783cd3 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 22 Feb 2016 19:16:33 +0100 Subject: [PATCH 083/306] offer option to test agent repository --- conftest.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/conftest.py b/conftest.py index 31dfba4e..6d597a31 100644 --- a/conftest.py +++ b/conftest.py @@ -25,7 +25,6 @@ if 'theano' in cfg: cfg['theano']['initial_number_of_nodes'] = '50' -micropsi.initialize() # create testuser from micropsi_server.micropsi_app import usermanager @@ -33,10 +32,6 @@ usermanager.create_user('Pytest User', 'test', 'Administrator', uid='Pytest User') user_token = usermanager.start_session('Pytest User', 'test', True) -# reset logging levels -logging.getLogger('system').setLevel(logging.WARNING) -logging.getLogger('world').setLevel(logging.WARNING) - world_uid = 'WorldOfPain' nn_uid = 'Testnet' @@ -57,6 +52,19 @@ def set_logging_levels(): def pytest_addoption(parser): parser.addoption("--engine", action="store", default=engine_defaults, help="The engine that should be used for this testrun.") + parser.addoption("--agents", action="store_true", + help="Only test agents-code from the data_directory") + + +def pytest_cmdline_main(config): + """ called for performing the main command line action. The default + implementation will invoke the configure hooks and runtest_mainloop. """ + if config.getoption('agents'): + config.args = [original_ini_data_directory] + micropsi.initialize(persistency_path=testpath, resource_path=original_ini_data_directory) + else: + micropsi.initialize(persistency_path=testpath) + set_logging_levels() def pytest_generate_tests(metafunc): From 9b53cfc646e9c2c86f7a2fa2fd297f586eef6ee3 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 22 Feb 2016 19:17:48 +0100 Subject: [PATCH 084/306] offer fixture for runtime --- conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conftest.py b/conftest.py index 6d597a31..0912484e 100644 --- a/conftest.py +++ b/conftest.py @@ -106,6 +106,11 @@ def pytest_runtest_setup(item): set_logging_levels() +@pytest.fixture(scope="session") +def runtime(): + return micropsi + + @pytest.fixture(scope="session") def resourcepath(): return cfg['paths']['data_directory'] From b0788cf8c9ac7ba078155d15c1fc1b8d7c03b842 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 22 Feb 2016 19:18:12 +0100 Subject: [PATCH 085/306] shut down any threads if session finished --- conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conftest.py b/conftest.py index 0912484e..333e0154 100644 --- a/conftest.py +++ b/conftest.py @@ -106,6 +106,11 @@ def pytest_runtest_setup(item): set_logging_levels() +def pytest_sessionfinish(session, exitstatus): + shutil.rmtree(testpath) + micropsi.kill_runners() + + @pytest.fixture(scope="session") def runtime(): return micropsi From d22e266c0ac4f632df9851c65c15c075a6d35839 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 22 Feb 2016 19:22:56 +0100 Subject: [PATCH 086/306] load nodenets from persistency_path --- micropsi_core/runtime.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index dff60e48..12e66778 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1244,12 +1244,12 @@ def parse_definition(json, filename=None): # Set up the MicroPsi runtime def load_definitions(): global nodenet_data, world_data - nodenet_data = crawl_definition_files(path=os.path.join(RESOURCE_PATH, NODENET_DIRECTORY), type="nodenet") - world_data = crawl_definition_files(path=os.path.join(RESOURCE_PATH, WORLD_DIRECTORY), type="world") + nodenet_data = crawl_definition_files(path=os.path.join(PERSISTENCY_PATH, NODENET_DIRECTORY), type="nodenet") + world_data = crawl_definition_files(path=os.path.join(PERSISTENCY_PATH, WORLD_DIRECTORY), type="world") if not world_data: # create a default world for convenience. uid = tools.generate_uid() - filename = os.path.join(RESOURCE_PATH, WORLD_DIRECTORY, uid + '.json') + filename = os.path.join(PERSISTENCY_PATH, WORLD_DIRECTORY, uid + '.json') world_data[uid] = Bunch(uid=uid, name="default", version=1, filename=filename) with open(filename, 'w+') as fp: fp.write(json.dumps(world_data[uid], sort_keys=True, indent=4)) From da7dcea4aedc7698c3e022284860c859e5ed0e2c Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 23 Feb 2016 12:32:41 +0100 Subject: [PATCH 087/306] load recipe file explicitly from source. --- micropsi_core/runtime.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 12e66778..06ff3e1d 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1320,8 +1320,10 @@ def parse_recipe_file(path, reload=False): pyname = relpath.replace(os.path.sep, '.')[:-3] try: - recipes = __import__(pyname, fromlist=['recipes']) - importlib.reload(sys.modules[pyname]) + loader = importlib.machinery.SourceFileLoader('recipes', path) + recipes = loader.load_module() + # recipes = __import__(pyname, fromlist=['recipes']) + # importlib.reload(sys.modules[pyname]) except SyntaxError as e: return "%s in recipe file %s, line %d" % (e.__class__.__name__, relpath, e.lineno) From bb437bc0a2e14a646aead82130fb0296c1b74c1f Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 23 Feb 2016 18:26:28 +0100 Subject: [PATCH 088/306] functions with the same id just got discovered twice for some reason this happens when calling getmembers --- micropsi_core/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 06ff3e1d..ee3db48c 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1347,7 +1347,7 @@ def parse_recipe_file(path, reload=False): 'name': arg, 'default': default }) - if name in custom_recipes: + if name in custom_recipes and id(func) != id(custom_recipes[name]['function']): logging.getLogger("system").warning("Recipe function names must be unique. %s is not." % name) custom_recipes[name] = { 'name': name, From e6d758d941e2cdf9502811e208fc0242857fb7bd Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 23 Feb 2016 18:32:44 +0100 Subject: [PATCH 089/306] don't pollute the path unnecessarily --- micropsi_core/runtime.py | 1 - 1 file changed, 1 deletion(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index ee3db48c..21066626 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1307,7 +1307,6 @@ def parse_native_module_file(path): if key in native_modules: logging.getLogger("system").warning("Native module names must be unique. %s is not." % key) native_modules[key] = modules[key] - sys.path.append(path) def parse_recipe_file(path, reload=False): From 54f57e55a1de5f9e6935d14741e4cde5ecab906e Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 23 Feb 2016 18:33:16 +0100 Subject: [PATCH 090/306] makefile target for convenience --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1239b23a..adc8aa2d 100644 --- a/Makefile +++ b/Makefile @@ -22,4 +22,8 @@ tests: test-coverage: bin/py.test --cov micropsi_core --cov micropsi_server --cov-report html -.PHONY: run \ No newline at end of file +agent-tests: + bin/py.test --agents + + +.PHONY: run From 5a27e54bc1bb1ec0e91380201acc6a3a11a4ea09 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 24 Feb 2016 16:57:53 +0100 Subject: [PATCH 091/306] Timeseries world adapter initialization fixes --- micropsi_core/world/timeseries/timeseries.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 497787ff..b851f6fd 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -46,11 +46,11 @@ def __init__(self, filename, world_type="Island", name="", owner="", engine=None return # todo use the new configurable world options. - dummydata = config('dummydata') == "True" - z_transform = config('z_transform') == "True" - clip_and_scale = config('clip_and_scale') == "True" - sigmoid = config('sigmoid') == "True" - self.shuffle = config('shuffle') == "True" + dummydata = config['dummy_data'] == "True" + z_transform = config['z_transform'] == "True" + clip_and_scale = config['clip_and_scale'] == "True" + sigmoid = config['sigmoid'] == "True" + self.shuffle = config['shuffle'] == "True" if clip_and_scale and sigmoid: self.logger.warn("clip_and_scale and sigmoid cannot both be configured, choosing sigmoid") From 672fb687c5a26f8f335b335a6f0abdb672198646 Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 24 Feb 2016 18:34:25 +0100 Subject: [PATCH 092/306] Slightly less netapi calls to get activations --- micropsi_core/nodenet/native_modules.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/micropsi_core/nodenet/native_modules.py b/micropsi_core/nodenet/native_modules.py index 2782093e..7231ba0f 100644 --- a/micropsi_core/nodenet/native_modules.py +++ b/micropsi_core/nodenet/native_modules.py @@ -994,10 +994,8 @@ def gradient_descent(netapi, node=None, **params): node.initialized = True - # get activations from node net + # get input activations from node net a_i_array = netapi.get_activations(ns_input_uid, input_) - a_h_array = netapi.get_activations(ns_hidden_uid, hidden) - a_o_array = netapi.get_activations(ns_output_uid, output) # learn only if activation on the input layer has been persistent for as many steps as your neural net has layers # Note: since we're currently using denoising autoencoders, this means persistent up to Bernoulli noise @@ -1025,6 +1023,10 @@ def gradient_descent(netapi, node=None, **params): if node.get_parameter('ctr') < 3: return + # get other activations from node net + a_h_array = netapi.get_activations(ns_hidden_uid, hidden) + a_o_array = netapi.get_activations(ns_output_uid, output) + # define learning parameters param = node.get_parameter('weight_decay') if param is None: From 5bfb38cd24a6811e1f5837ad61fa3dead497b5dc Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 24 Feb 2016 18:59:38 +0100 Subject: [PATCH 093/306] Debug output every 1000 backprops --- micropsi_core/nodenet/native_modules.py | 26 ++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/micropsi_core/nodenet/native_modules.py b/micropsi_core/nodenet/native_modules.py index 7231ba0f..50904e94 100644 --- a/micropsi_core/nodenet/native_modules.py +++ b/micropsi_core/nodenet/native_modules.py @@ -797,6 +797,8 @@ def gradient_descent(netapi, node=None, **params): # initialization if not hasattr(node, 'initialized'): + node.set_state('cumulative_error', 0) + sparse = str(node.get_parameter('ae_type')) == "sparse" # denoising = str(node.get_parameter('ae_type')) == "denoising" tied_weights = str(node.get_parameter('tied_weights')) == "True" @@ -896,7 +898,7 @@ def gradient_descent(netapi, node=None, **params): cost = error_term + weight_constraint node.cost = theano.function([weight_decay, sparsity_value, sparsity_penalty], cost, on_unused_input='ignore') - node.error = theano.function([], error_term) + node.error = theano.function([], error_term / len(b_h_array)) # compute gradients sigmoid_deriv_a_o = a_o * (1. - a_o) @@ -1107,15 +1109,6 @@ def gradient_descent(netapi, node=None, **params): node.a_h.set_value(a_h_array, borrow=True) node.a_o.set_value(a_o_array, borrow=True) - # save current error as node parameter - node.set_parameter('error', float(node.error())) - # # print average reconstruction error - # node.set_parameter('error', node.get_parameter('error') + node.error()) - # if node.get_parameter('t') % 100 == 0: - # netapi.logger.debug("Number of backprop steps computed %d" % node.get_parameter('t')) - # netapi.logger.debug("Reconstruction error %f" % (node.get_parameter('error') / 100)) - # node.set_parameter('error', 0.0) - # update values in shared variables ( using backpropgation of the gradients ) node.get_updated_parameters(weight_decay, sparsity_value, sparsity_penalty, ada_rho, ada_eps) @@ -1125,9 +1118,20 @@ def gradient_descent(netapi, node=None, **params): netapi.set_thetas(ns_hidden_uid, hidden, node.b_h.get_value(borrow=True)) netapi.set_link_weights(ns_input_uid, input_, ns_hidden_uid, hidden, node.w_hi.get_value(borrow=True)) + error = float(node.error()) + # save current error as node parameter + node.set_parameter('error', error) + node.set_state('cumulative_error', node.get_state('cumulative_error') + error) + + t = int(node.get_parameter('t')) + if t % 1000 == 0: + netapi.logger.debug("Number of backprop steps computed %d" % t) + netapi.logger.debug("Average Error %.6f (Latest: 0=%.6f)" % ((node.get_state('cumulative_error') / 1000), error)) + node.set_state('cumulative_error', 0.0) + # reset counter after successful backprop step; cf. must wait for new sensory activation to reach output layer node.set_parameter('ctr', 0) - node.set_parameter('t', int(node.get_parameter('t')) + 1) + node.set_parameter('t', t + 1) def sigmoid(z): From 9945ba0b0628e429f28dc1a3b8d177a85e7165a0 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 24 Feb 2016 22:15:14 +0100 Subject: [PATCH 094/306] restructure & cleanup conftest, use field_fixtures instead of removing the whole data folder --- conftest.py | 114 +++++++++++++++--------------- micropsi_core/runtime.py | 3 +- micropsi_core/tests/conftest.py | 8 ++- micropsi_server/tests/conftest.py | 3 +- 4 files changed, 68 insertions(+), 60 deletions(-) diff --git a/conftest.py b/conftest.py index 333e0154..6c056cb3 100644 --- a/conftest.py +++ b/conftest.py @@ -4,17 +4,20 @@ import pytest import logging +try: + import theano + engine_defaults = "dict_engine,theano_engine" +except: + engine_defaults = "dict_engine" -testpath = os.path.join('.', 'test-data') +testpath = os.path.abspath(os.path.join('.', 'test-data')) try: shutil.rmtree(testpath) except OSError: pass -# override config from micropsi_core import runtime as micropsi - from micropsi_core.runtime import cfg original_ini_data_directory = cfg['paths']['data_directory'] @@ -26,30 +29,12 @@ cfg['theano']['initial_number_of_nodes'] = '50' -# create testuser -from micropsi_server.micropsi_app import usermanager - -usermanager.create_user('Pytest User', 'test', 'Administrator', uid='Pytest User') -user_token = usermanager.start_session('Pytest User', 'test', True) - world_uid = 'WorldOfPain' nn_uid = 'Testnet' -try: - import theano - engine_defaults = "dict_engine,theano_engine" -except: - engine_defaults = "dict_engine" - - -def set_logging_levels(): - logging.getLogger('system').setLevel(logging.WARNING) - logging.getLogger('world').setLevel(logging.WARNING) - micropsi.cfg['logging']['level_agent'] = 'WARNING' - - def pytest_addoption(parser): + """register argparse-style options and ini-style config values.""" parser.addoption("--engine", action="store", default=engine_defaults, help="The engine that should be used for this testrun.") parser.addoption("--agents", action="store_true", @@ -64,9 +49,20 @@ def pytest_cmdline_main(config): micropsi.initialize(persistency_path=testpath, resource_path=original_ini_data_directory) else: micropsi.initialize(persistency_path=testpath) + from micropsi_server.micropsi_app import usermanager + + usermanager.create_user('Pytest User', 'test', 'Administrator', uid='Pytest User') + usermanager.start_session('Pytest User', 'test', True) + set_logging_levels() +def pytest_configure(config): + # register an additional marker + config.addinivalue_line("markers", + "engine(name): mark test to run only on the specified engine") + + def pytest_generate_tests(metafunc): if 'engine' in metafunc.fixturenames: engines = [] @@ -78,27 +74,19 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("engine", engines, scope="session") -def pytest_configure(config): - # register an additional marker - config.addinivalue_line("markers", - "engine(name): mark test to run only on the specified engine") - - def pytest_runtest_setup(item): engine_marker = item.get_marker("engine") if engine_marker is not None: engine_marker = engine_marker.args[0] if engine_marker != item.callspec.params['engine']: pytest.skip("test requires engine %s" % engine_marker) - for uid in list(micropsi.nodenets.keys()): - micropsi.delete_nodenet(uid) - for uid in list(micropsi.worlds.keys()): - micropsi.delete_world(uid) - shutil.rmtree(testpath) - os.mkdir(testpath) - os.mkdir(os.path.join(testpath, 'nodenets')) - os.mkdir(os.path.join(testpath, 'worlds')) - # default native module container + for item in os.listdir(testpath): + if item != 'worlds' and item != 'nodenets': + path = os.path.join(testpath, item) + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) os.mkdir(os.path.join(testpath, 'Test')) open(os.path.join(testpath, 'Test', '__init__.py'), 'w').close() micropsi.reload_native_modules() @@ -106,32 +94,52 @@ def pytest_runtest_setup(item): set_logging_levels() -def pytest_sessionfinish(session, exitstatus): +def pytest_internalerror(excrepr, excinfo): + """ called for internal errors. """ + micropsi.kill_runners() shutil.rmtree(testpath) + + +def pytest_keyboard_interrupt(excinfo): + """ called for keyboard interrupt. """ micropsi.kill_runners() + shutil.rmtree(testpath) -@pytest.fixture(scope="session") -def runtime(): - return micropsi +def set_logging_levels(): + """ sets the logging levels of the default loggers back to WARNING """ + logging.getLogger('system').setLevel(logging.WARNING) + logging.getLogger('world').setLevel(logging.WARNING) + micropsi.cfg['logging']['level_agent'] = 'WARNING' @pytest.fixture(scope="session") def resourcepath(): - return cfg['paths']['data_directory'] + """ Fixture: the resource path """ + return micropsi.RESOURCE_PATH -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") +def runtime(): + """ Fixture: The micropsi runtime """ + return micropsi + + +@pytest.yield_fixture(scope="function") def test_world(request): """ Fixture: A test world of type Island """ global world_uid success, world_uid = micropsi.new_world("World of Pain", "Island", "Pytest User", uid=world_uid) - return world_uid + yield world_uid + try: + micropsi.delete_world(world_uid) + except: + pass -@pytest.fixture(scope="function") +@pytest.yield_fixture(scope="function") def test_nodenet(request, test_world, engine): """ Fixture: A completely empty nodenet without a worldadapter @@ -139,7 +147,11 @@ def test_nodenet(request, test_world, engine): global nn_uid success, nn_uid = micropsi.new_nodenet("Testnet", engine=engine, owner="Pytest User", uid='Testnet') micropsi.save_nodenet(nn_uid) - return nn_uid + yield nn_uid + try: + micropsi.delete_nodenet(nn_uid) + except: + pass @pytest.fixture(scope="function") @@ -150,13 +162,3 @@ def node(request, test_nodenet): res, uid = micropsi.add_node(test_nodenet, 'Pipe', [10, 10, 10], name='N1') micropsi.add_link(test_nodenet, uid, 'gen', uid, 'gen') return uid - - -def pytest_internalerror(excrepr, excinfo): - """ called for internal errors. """ - shutil.rmtree(testpath) - - -def pytest_keyboard_interrupt(excinfo): - """ called for keyboard interrupt. """ - shutil.rmtree(testpath) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 21066626..1ac0348c 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1439,7 +1439,8 @@ def initialize(persistency_path=None, resource_path=None): set_runner_properties(configs['runner_timestep'], configs['runner_factor']) runner['running'] = True - runner['runner'] = MicropsiRunner() + if runner.get('runner') is None: + runner['runner'] = MicropsiRunner() if kill_runners not in signal_handler_registry: add_signal_handler(kill_runners) diff --git a/micropsi_core/tests/conftest.py b/micropsi_core/tests/conftest.py index 1d142fb9..736095e4 100644 --- a/micropsi_core/tests/conftest.py +++ b/micropsi_core/tests/conftest.py @@ -10,7 +10,7 @@ nn_uid = 'Testnet' -@pytest.fixture(scope="function") +@pytest.yield_fixture(scope="function") def fixed_nodenet(request, test_world, engine): """ A test nodenet filled with some example data (nodenet_data.py) @@ -36,4 +36,8 @@ def fixed_nodenet(request, test_world, engine): micropsi.get_nodenet(uid) micropsi.merge_nodenet(uid, fixed_nodenet_data, keep_uids=True) micropsi.save_nodenet(uid) - return uid + yield uid + try: + micropsi.delete_nodenet(uid) + except: + pass diff --git a/micropsi_server/tests/conftest.py b/micropsi_server/tests/conftest.py index 39ef6d45..a2cfafe6 100644 --- a/micropsi_server/tests/conftest.py +++ b/micropsi_server/tests/conftest.py @@ -6,9 +6,10 @@ nn_uid = 'Testnet' from configuration import config -from conftest import user_token from micropsi_server import usermanagement +from micropsi_server.micropsi_app import usermanager +user_token = usermanager.users['Pytest User']['session_token'] test_path = os.path.join(config['paths']['data_directory'], 'user-test-db.json') From b91d6c2bcd2da3eb64a4378f4662e7d83881cc08 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 24 Feb 2016 22:16:13 +0100 Subject: [PATCH 095/306] fix missing datatarget_feedback --- .../world/minecraft/minecraft_histogram_vision.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/micropsi_core/world/minecraft/minecraft_histogram_vision.py b/micropsi_core/world/minecraft/minecraft_histogram_vision.py index 93ba2f0d..88df004f 100644 --- a/micropsi_core/world/minecraft/minecraft_histogram_vision.py +++ b/micropsi_core/world/minecraft/minecraft_histogram_vision.py @@ -50,11 +50,14 @@ def __init__(self, world, uid=None, **data): 'fov_hist__106': 0, }) - self.datatargets.update({ + targets = { 'orientation': 0, 'fov_x': 0, 'fov_y': 0 - }) + } + + self.datatargets.update(targets) + self.datatarget_feedback.update(targets) # add datasources for fovea for i in range(self.num_fov): From e9af6518f1e482054694388837c46fdf1c2a3feb Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 24 Feb 2016 22:16:53 +0100 Subject: [PATCH 096/306] don't init runtime on import, init when needed we can have multiple runtime instances now, but we don't want to. --- micropsi_server/micropsi_app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index ea81c37c..87701f89 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -28,8 +28,6 @@ from micropsi_server import minidoc import logging -runtime.initialize() - from configuration import config as cfg VERSION = cfg['micropsi2']['version'] @@ -1232,6 +1230,7 @@ def main(host=None, port=None): port = port or cfg['micropsi2']['port'] server = cfg['micropsi2']['server'] print("Starting App on Port " + str(port)) + runtime.initialize() run(micropsi_app, host=host, port=port, quiet=True, server=server) if __name__ == "__main__": From 024eb37cd64d54b123664cae7f28220f7dd05d5c Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 24 Feb 2016 22:18:37 +0100 Subject: [PATCH 097/306] no netlock for growing. we need to clarify our locking mechanisms --- .../nodenet/theano_engine/theano_partition.py | 198 +++++++++--------- 1 file changed, 98 insertions(+), 100 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index 2bc76f5c..e7de28f9 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -1077,143 +1077,141 @@ def grow_number_of_nodespaces(self, growby): new_NoNS = int(self.NoNS + growby) - with self.nodenet.netlock: - new_allocated_nodespaces = np.zeros(new_NoNS, dtype=np.int32) - new_allocated_nodespaces[0:self.NoNS] = self.allocated_nodespaces - self.allocated_nodespaces = new_allocated_nodespaces + new_allocated_nodespaces = np.zeros(new_NoNS, dtype=np.int32) + new_allocated_nodespaces[0:self.NoNS] = self.allocated_nodespaces + self.allocated_nodespaces = new_allocated_nodespaces - new_allocated_nodespaces_por_activators = np.zeros(new_NoNS, dtype=np.int32) - new_allocated_nodespaces_por_activators[0:self.NoNS] = self.allocated_nodespaces_por_activators - self.allocated_nodespaces_por_activators = new_allocated_nodespaces_por_activators + new_allocated_nodespaces_por_activators = np.zeros(new_NoNS, dtype=np.int32) + new_allocated_nodespaces_por_activators[0:self.NoNS] = self.allocated_nodespaces_por_activators + self.allocated_nodespaces_por_activators = new_allocated_nodespaces_por_activators - new_allocated_nodespaces_ret_activators = np.zeros(new_NoNS, dtype=np.int32) - new_allocated_nodespaces_ret_activators[0:self.NoNS] = self.allocated_nodespaces_ret_activators - self.allocated_nodespaces_ret_activators = new_allocated_nodespaces_ret_activators + new_allocated_nodespaces_ret_activators = np.zeros(new_NoNS, dtype=np.int32) + new_allocated_nodespaces_ret_activators[0:self.NoNS] = self.allocated_nodespaces_ret_activators + self.allocated_nodespaces_ret_activators = new_allocated_nodespaces_ret_activators - new_allocated_nodespaces_sub_activators = np.zeros(new_NoNS, dtype=np.int32) - new_allocated_nodespaces_sub_activators[0:self.NoNS] = self.allocated_nodespaces_sub_activators - self.allocated_nodespaces_sub_activators = new_allocated_nodespaces_sub_activators + new_allocated_nodespaces_sub_activators = np.zeros(new_NoNS, dtype=np.int32) + new_allocated_nodespaces_sub_activators[0:self.NoNS] = self.allocated_nodespaces_sub_activators + self.allocated_nodespaces_sub_activators = new_allocated_nodespaces_sub_activators - new_allocated_nodespaces_sur_activators = np.zeros(new_NoNS, dtype=np.int32) - new_allocated_nodespaces_sur_activators[0:self.NoNS] = self.allocated_nodespaces_sur_activators - self.allocated_nodespaces_sur_activators = new_allocated_nodespaces_sur_activators + new_allocated_nodespaces_sur_activators = np.zeros(new_NoNS, dtype=np.int32) + new_allocated_nodespaces_sur_activators[0:self.NoNS] = self.allocated_nodespaces_sur_activators + self.allocated_nodespaces_sur_activators = new_allocated_nodespaces_sur_activators - new_allocated_nodespaces_cat_activators = np.zeros(new_NoNS, dtype=np.int32) - new_allocated_nodespaces_cat_activators[0:self.NoNS] = self.allocated_nodespaces_cat_activators - self.allocated_nodespaces_cat_activators = new_allocated_nodespaces_cat_activators + new_allocated_nodespaces_cat_activators = np.zeros(new_NoNS, dtype=np.int32) + new_allocated_nodespaces_cat_activators[0:self.NoNS] = self.allocated_nodespaces_cat_activators + self.allocated_nodespaces_cat_activators = new_allocated_nodespaces_cat_activators - new_allocated_nodespaces_exp_activators = np.zeros(new_NoNS, dtype=np.int32) - new_allocated_nodespaces_exp_activators[0:self.NoNS] = self.allocated_nodespaces_exp_activators - self.allocated_nodespaces_exp_activators = new_allocated_nodespaces_exp_activators + new_allocated_nodespaces_exp_activators = np.zeros(new_NoNS, dtype=np.int32) + new_allocated_nodespaces_exp_activators[0:self.NoNS] = self.allocated_nodespaces_exp_activators + self.allocated_nodespaces_exp_activators = new_allocated_nodespaces_exp_activators - new_allocated_nodespaces_sampling_activators = np.zeros(new_NoNS, dtype=np.int32) - new_allocated_nodespaces_sampling_activators[0:self.NoNS] = self.allocated_nodespaces_sampling_activators - self.allocated_nodespaces_sampling_activators = new_allocated_nodespaces_sampling_activators + new_allocated_nodespaces_sampling_activators = np.zeros(new_NoNS, dtype=np.int32) + new_allocated_nodespaces_sampling_activators[0:self.NoNS] = self.allocated_nodespaces_sampling_activators + self.allocated_nodespaces_sampling_activators = new_allocated_nodespaces_sampling_activators - new_nodespaces_last_changed = np.zeros(new_NoNS, dtype=np.int32) - new_nodespaces_last_changed[0:self.NoNS] = self.nodespaces_last_changed - self.nodespaces_last_changed = new_nodespaces_last_changed + new_nodespaces_last_changed = np.zeros(new_NoNS, dtype=np.int32) + new_nodespaces_last_changed[0:self.NoNS] = self.nodespaces_last_changed + self.nodespaces_last_changed = new_nodespaces_last_changed - new_nodespaces_contents_last_changed = np.zeros(new_NoNS, dtype=np.int32) - new_nodespaces_contents_last_changed[0:self.NoNS] = self.nodespaces_contents_last_changed - self.nodespaces_contents_last_changed = new_nodespaces_contents_last_changed + new_nodespaces_contents_last_changed = np.zeros(new_NoNS, dtype=np.int32) + new_nodespaces_contents_last_changed[0:self.NoNS] = self.nodespaces_contents_last_changed + self.nodespaces_contents_last_changed = new_nodespaces_contents_last_changed - self.has_new_usages = True - self.NoNS = new_NoNS + self.has_new_usages = True + self.NoNS = new_NoNS def grow_number_of_elements(self, growby): new_NoE = int(self.NoE + growby) - with self.nodenet.netlock: - new_allocated_elements_to_nodes = np.zeros(new_NoE, dtype=np.int32) - new_allocated_elements_to_nodes[0:self.NoE] = self.allocated_elements_to_nodes - self.allocated_elements_to_nodes = new_allocated_elements_to_nodes + new_allocated_elements_to_nodes = np.zeros(new_NoE, dtype=np.int32) + new_allocated_elements_to_nodes[0:self.NoE] = self.allocated_elements_to_nodes + self.allocated_elements_to_nodes = new_allocated_elements_to_nodes - new_allocated_elements_to_activators = np.zeros(new_NoE, dtype=np.int32) - new_allocated_elements_to_activators[0:self.NoE] = self.allocated_elements_to_activators - self.allocated_elements_to_activators = new_allocated_elements_to_activators + new_allocated_elements_to_activators = np.zeros(new_NoE, dtype=np.int32) + new_allocated_elements_to_activators[0:self.NoE] = self.allocated_elements_to_activators + self.allocated_elements_to_activators = new_allocated_elements_to_activators - if self.sparse: - new_w = sp.csr_matrix((new_NoE, new_NoE), dtype=self.nodenet.scipyfloatX) - else: - new_w = np.zeros((new_NoE, new_NoE), dtype=self.nodenet.scipyfloatX) - new_w[0:self.NoE, 0:self.NoE] = self.w.get_value(borrow=True) - self.w.set_value(new_w, borrow=True) + if self.sparse: + new_w = sp.csr_matrix((new_NoE, new_NoE), dtype=self.nodenet.scipyfloatX) + else: + new_w = np.zeros((new_NoE, new_NoE), dtype=self.nodenet.scipyfloatX) + new_w[0:self.NoE, 0:self.NoE] = self.w.get_value(borrow=True) + self.w.set_value(new_w, borrow=True) - new_a = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) - new_a[0:self.NoE] = self.a.get_value(borrow=True) - self.a.set_value(new_a, borrow=True) + new_a = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) + new_a[0:self.NoE] = self.a.get_value(borrow=True) + self.a.set_value(new_a, borrow=True) - new_a_shifted = np.lib.stride_tricks.as_strided(new_a, shape=(new_NoE, 7), strides=(self.nodenet.byte_per_float, self.nodenet.byte_per_float)) - self.a_shifted.set_value(new_a_shifted, borrow=True) + new_a_shifted = np.lib.stride_tricks.as_strided(new_a, shape=(new_NoE, 7), strides=(self.nodenet.byte_per_float, self.nodenet.byte_per_float)) + self.a_shifted.set_value(new_a_shifted, borrow=True) - new_a_in = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) - new_a_in[0:self.NoE] = self.a_in.get_value(borrow=True) - self.a_in.set_value(new_a_in, borrow=True) + new_a_in = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) + new_a_in[0:self.NoE] = self.a_in.get_value(borrow=True) + self.a_in.set_value(new_a_in, borrow=True) - new_a_prev = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) - new_a_prev[0:self.NoE] = self.a_prev.get_value(borrow=True) - self.a_prev.set_value(new_a_prev, borrow=True) + new_a_prev = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) + new_a_prev[0:self.NoE] = self.a_prev.get_value(borrow=True) + self.a_prev.set_value(new_a_prev, borrow=True) - new_g_theta = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) - new_g_theta[0:self.NoE] = self.g_theta.get_value(borrow=True) - self.g_theta.set_value(new_g_theta, borrow=True) + new_g_theta = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) + new_g_theta[0:self.NoE] = self.g_theta.get_value(borrow=True) + self.g_theta.set_value(new_g_theta, borrow=True) - new_g_theta_shifted = np.lib.stride_tricks.as_strided(new_g_theta, shape=(self.NoE, 7), strides=(self.nodenet.byte_per_float, self.nodenet.byte_per_float)) - self.g_theta_shifted.set_value(new_g_theta_shifted, borrow=True) + new_g_theta_shifted = np.lib.stride_tricks.as_strided(new_g_theta, shape=(self.NoE, 7), strides=(self.nodenet.byte_per_float, self.nodenet.byte_per_float)) + self.g_theta_shifted.set_value(new_g_theta_shifted, borrow=True) - new_g_factor = np.ones(new_NoE, dtype=self.nodenet.numpyfloatX) - new_g_factor[0:self.NoE] = self.g_factor.get_value(borrow=True) - self.g_factor.set_value(new_g_factor, borrow=True) + new_g_factor = np.ones(new_NoE, dtype=self.nodenet.numpyfloatX) + new_g_factor[0:self.NoE] = self.g_factor.get_value(borrow=True) + self.g_factor.set_value(new_g_factor, borrow=True) - new_g_threshold = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) - new_g_threshold[0:self.NoE] = self.g_threshold.get_value(borrow=True) - self.g_threshold.set_value(new_g_threshold, borrow=True) + new_g_threshold = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) + new_g_threshold[0:self.NoE] = self.g_threshold.get_value(borrow=True) + self.g_threshold.set_value(new_g_threshold, borrow=True) - new_g_amplification = np.ones(new_NoE, dtype=self.nodenet.numpyfloatX) - new_g_amplification[0:self.NoE] = self.g_amplification.get_value(borrow=True) - self.g_amplification.set_value(new_g_amplification, borrow=True) + new_g_amplification = np.ones(new_NoE, dtype=self.nodenet.numpyfloatX) + new_g_amplification[0:self.NoE] = self.g_amplification.get_value(borrow=True) + self.g_amplification.set_value(new_g_amplification, borrow=True) - new_g_min = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) - new_g_min[0:self.NoE] = self.g_min.get_value(borrow=True) - self.g_min.set_value(new_g_min, borrow=True) + new_g_min = np.zeros(new_NoE, dtype=self.nodenet.numpyfloatX) + new_g_min[0:self.NoE] = self.g_min.get_value(borrow=True) + self.g_min.set_value(new_g_min, borrow=True) - new_g_max = np.ones(new_NoE, dtype=self.nodenet.numpyfloatX) - new_g_max[0:self.NoE] = self.g_max.get_value(borrow=True) - self.g_max.set_value(new_g_max, borrow=True) + new_g_max = np.ones(new_NoE, dtype=self.nodenet.numpyfloatX) + new_g_max[0:self.NoE] = self.g_max.get_value(borrow=True) + self.g_max.set_value(new_g_max, borrow=True) - new_g_function_selector = np.zeros(new_NoE, dtype=np.int8) - new_g_function_selector[0:self.NoE] = self.g_function_selector.get_value(borrow=True) - self.g_function_selector.set_value(new_g_function_selector, borrow=True) + new_g_function_selector = np.zeros(new_NoE, dtype=np.int8) + new_g_function_selector[0:self.NoE] = self.g_function_selector.get_value(borrow=True) + self.g_function_selector.set_value(new_g_function_selector, borrow=True) - new_g_expect = np.ones(new_NoE, dtype=self.nodenet.numpyfloatX) - new_g_expect[0:self.NoE] = self.g_expect.get_value(borrow=True) - self.g_expect.set_value(new_g_expect, borrow=True) + new_g_expect = np.ones(new_NoE, dtype=self.nodenet.numpyfloatX) + new_g_expect[0:self.NoE] = self.g_expect.get_value(borrow=True) + self.g_expect.set_value(new_g_expect, borrow=True) - new_g_countdown = np.zeros(new_NoE, dtype=np.int16) - new_g_countdown[0:self.NoE] = self.g_countdown.get_value(borrow=True) - self.g_countdown.set_value(new_g_countdown, borrow=True) + new_g_countdown = np.zeros(new_NoE, dtype=np.int16) + new_g_countdown[0:self.NoE] = self.g_countdown.get_value(borrow=True) + self.g_countdown.set_value(new_g_countdown, borrow=True) - new_g_wait = np.ones(new_NoE, dtype=np.int16) - new_g_wait[0:self.NoE] = self.g_wait.get_value(borrow=True) - self.g_wait.set_value(new_g_wait, borrow=True) + new_g_wait = np.ones(new_NoE, dtype=np.int16) + new_g_wait[0:self.NoE] = self.g_wait.get_value(borrow=True) + self.g_wait.set_value(new_g_wait, borrow=True) - new_n_function_selector = np.zeros(new_NoE, dtype=np.int8) - new_n_function_selector[0:self.NoE] = self.n_function_selector.get_value(borrow=True) - self.n_function_selector.set_value(new_n_function_selector, borrow=True) + new_n_function_selector = np.zeros(new_NoE, dtype=np.int8) + new_n_function_selector[0:self.NoE] = self.n_function_selector.get_value(borrow=True) + self.n_function_selector.set_value(new_n_function_selector, borrow=True) - new_n_node_porlinked = np.zeros(new_NoE, dtype=np.int8) - self.n_node_porlinked.set_value(new_n_node_porlinked, borrow=True) + new_n_node_porlinked = np.zeros(new_NoE, dtype=np.int8) + self.n_node_porlinked.set_value(new_n_node_porlinked, borrow=True) - new_n_node_retlinked = np.zeros(new_NoE, dtype=np.int8) - self.n_node_retlinked.set_value(new_n_node_retlinked, borrow=True) + new_n_node_retlinked = np.zeros(new_NoE, dtype=np.int8) + self.n_node_retlinked.set_value(new_n_node_retlinked, borrow=True) - self.NoE = new_NoE - self.has_new_usages = True + self.NoE = new_NoE + self.has_new_usages = True - if self.has_pipes: - self.por_ret_dirty = True + if self.has_pipes: + self.por_ret_dirty = True def announce_nodes(self, number_of_nodes, average_elements_per_node): From 8e48005e776b1769e5befca4e6e2b9a0394df678 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 25 Feb 2016 12:22:07 +0100 Subject: [PATCH 098/306] new makefile targets, make tests now tests both toolkit and agents --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index adc8aa2d..07fd9a8b 100644 --- a/Makefile +++ b/Makefile @@ -18,11 +18,15 @@ clean: tests: bin/py.test + bin/py.test --agents test-coverage: bin/py.test --cov micropsi_core --cov micropsi_server --cov-report html -agent-tests: +test-toolkit: + bin/py.test + +test-agents: bin/py.test --agents From cecf748210f80ec2911806d1544df6ea76b555ba Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 25 Feb 2016 14:49:49 +0100 Subject: [PATCH 099/306] run agent tests even if toolkit tests contain failures --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 07fd9a8b..ec457e7e 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,8 @@ clean: rm -rf include lib .Python bin tests: - bin/py.test - bin/py.test --agents + -bin/py.test + -bin/py.test --agents test-coverage: bin/py.test --cov micropsi_core --cov micropsi_server --cov-report html From f1d6933d9771eb047b25faed6a99dcfe9367bade Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 25 Feb 2016 14:50:09 +0100 Subject: [PATCH 100/306] exclude conftest files from coverage --- .coveragerc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index d5800712..a9a72c98 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,5 @@ [run] omit = */tests/* - micropsi_server/bottle.py \ No newline at end of file + micropsi_server/bottle.py + */conftest.py From 23ce9c30e3f6a120a42a3612ccbd3cba1d0e9619 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 26 Feb 2016 15:30:05 +0100 Subject: [PATCH 101/306] don't pin scipy version, 0.15 seems incompatible to python3.5 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 21b4107a..c3056526 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ py==1.4.26 pycrypto==2.6.1 pytest==2.6.4 pytest-cov==1.8.1 -scipy==0.15.1 +scipy six==1.8.0 -e git+https://github.com/micropsi-industries/spock.git#egg=spock-dev waitress==0.8.9 From 4b7a560897d6a50e8650e87fa1077cdbbf188705 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 26 Feb 2016 15:32:26 +0100 Subject: [PATCH 102/306] open recipe_name dropdown with dialog --- micropsi_server/static/js/dialogs.js | 1 + 1 file changed, 1 insertion(+) diff --git a/micropsi_server/static/js/dialogs.js b/micropsi_server/static/js/dialogs.js index 12b0f2a1..2d0c1cb7 100644 --- a/micropsi_server/static/js/dialogs.js +++ b/micropsi_server/static/js/dialogs.js @@ -663,6 +663,7 @@ $(function() { recipe_name_input.val(''); recipe_name_input.chosen({'search_contains': true}); recipe_name_input.focus(); + recipe_name_input.trigger('chosen:open'); update_parameters_for_recipe(); }); }); From 3761faf11bd0434fea716f3ab1140ad98d3ca8bc Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 26 Feb 2016 17:02:36 +0100 Subject: [PATCH 103/306] ignore recipes imported from another recipe file to avoid false duplicate-function-name alerts --- micropsi_core/runtime.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 1ac0348c..a1975dbd 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1332,6 +1332,11 @@ def parse_recipe_file(path, reload=False): all_functions = inspect.getmembers(recipes, inspect.isfunction) for name, func in all_functions: + filename = os.path.realpath(func.__code__.co_filename) + if filename != os.path.realpath(path) and os.path.basename(filename) == 'recipes.py': + # import from another recipes file. ignore, to avoid + # false duplicate-function-name alerts + continue argspec = inspect.getargspec(func) arguments = argspec.args[1:] defaults = argspec.defaults or [] From 42ad8164b6800ab3fa0aaeb17f1d4bf2df7d5e49 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 26 Feb 2016 17:51:44 +0100 Subject: [PATCH 104/306] new version pins for 3.5 compatibility --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c3056526..6712d738 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ CherryPy==3.6.0 Theano==0.7.0 WebOb==1.4 WebTest==2.0.16 -beautifulsoup4==4.3.2 +beautifulsoup4==4.4.1 cov-core==1.14.0 coverage==3.7.1 matplotlib==1.4.3 @@ -12,7 +12,7 @@ py==1.4.26 pycrypto==2.6.1 pytest==2.6.4 pytest-cov==1.8.1 -scipy +scipy==0.17.0 six==1.8.0 -e git+https://github.com/micropsi-industries/spock.git#egg=spock-dev waitress==0.8.9 From c975d62c73613175ff28186d099cf81e4055ef55 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 26 Feb 2016 17:52:47 +0100 Subject: [PATCH 105/306] fix for 3.5 --- micropsi_core/nodenet/node_alignment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/micropsi_core/nodenet/node_alignment.py b/micropsi_core/nodenet/node_alignment.py index c93739ad..f02f96fe 100644 --- a/micropsi_core/nodenet/node_alignment.py +++ b/micropsi_core/nodenet/node_alignment.py @@ -258,6 +258,7 @@ def _fix_link_inheritance(group, excluded_nodes): The function adds the links as .directions to the group and its sub-groups, and carries a set of excluded_nodes to remember which links should not be inherited upwards""" + from copy import deepcopy if hasattr(group, "uid"): excluded_nodes.add(group) else: @@ -274,7 +275,7 @@ def _fix_link_inheritance(group, excluded_nodes): # now delete all links to excluded nodes dirs_copy = group.directions.copy() for d in dirs_copy: - for node in dirs_copy[d]: + for node in deepcopy(dirs_copy[d]): if node in excluded_nodes: group.directions[d].remove(node) if not group.directions[d]: From 50532761e9fbf3e39da6060c7c819a7ffd8d1a43 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 26 Feb 2016 17:58:58 +0100 Subject: [PATCH 106/306] readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2efcf219..3ff73824 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ For more information visit [micropsi.com](http://www.micropsi.com), for instance Prerequisites ----- -* Python3 +* Python3 (tested with 3.4.3 and 3.5.1) * On Windows, we recommend downloading and installing [WinPython 3.4.3.7](http://winpython.github.io/) From 735acd6cb4b95ee0c85ec689b94e5f7b74247649 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 26 Feb 2016 18:22:44 +0100 Subject: [PATCH 107/306] use new py & py.test versions --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6712d738..fdc4a6b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,9 +8,9 @@ coverage==3.7.1 matplotlib==1.4.3 mock==1.0.1 numpy==1.9.2 -py==1.4.26 +py==1.4.31 pycrypto==2.6.1 -pytest==2.6.4 +pytest==2.8.7 pytest-cov==1.8.1 scipy==0.17.0 six==1.8.0 From 7d1922b7951af3736470539d479743afc152cd69 Mon Sep 17 00:00:00 2001 From: Doik Date: Sun, 28 Feb 2016 23:57:21 +0100 Subject: [PATCH 108/306] typo --- README_additional_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_additional_configuration.md b/README_additional_configuration.md index 400df4ec..2f4d9a87 100644 --- a/README_additional_configuration.md +++ b/README_additional_configuration.md @@ -37,7 +37,7 @@ Windows support for advanced features is experimental. * From the mingw Folder, add the `bin` Folder to your PATH environment variable * Install pycrypto for python3.4. Get one of the [pycrypto windows binaries](https://github.com/axper/python3-pycrypto-windows-installer), open the WinPython Control Panel, click "Add Packages", select the downloaded pycrypto installer, and click "Install packages" * now you can install our modified spock via -`pip install -e -e git+https://github.com/micropsi-industries/spock.git#egg=spock-dev` +`pip install -e git+https://github.com/micropsi-industries/spock.git#egg=spock-dev` * this should lead to a working MicroPsi with Theano and minecraft support. * install the optional packages with `pip install cherrypy pytest mock webtest` * run `python start_micropsi_server.py` From 2c0413d4f263c3b510f6f595793384ff8da699cd Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Sat, 27 Feb 2016 17:28:05 +0100 Subject: [PATCH 109/306] fix error if partition has no actuators --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 9799200a..df84fd0b 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -1288,8 +1288,9 @@ def read_actuators(self): actuator_values_to_write = np.zeros_like(self.rootpartition.actuator_indices) for partition in self.partitions.values(): - a_array = partition.a.get_value(borrow=True) - actuator_values_to_write = actuator_values_to_write + a_array[partition.actuator_indices] + if len(partition.actuator_indices): + a_array = partition.a.get_value(borrow=True) + actuator_values_to_write = actuator_values_to_write + a_array[partition.actuator_indices] return actuator_values_to_write From f486802d970c7874aa0295dd639854139b35175c Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Sat, 27 Feb 2016 17:57:14 +0100 Subject: [PATCH 110/306] fix setting nodes dirty in inlink_weights --- micropsi_core/nodenet/theano_engine/theano_partition.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index e7de28f9..989faa54 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -1794,6 +1794,7 @@ def set_link_weights(self, nodespace_from_uid, group_from, nodespace_to_uid, gro self.por_ret_dirty = self.has_pipes def set_inlink_weights(self, partition_from_spid, new_from_elements, new_to_elements, new_weights): + from_partition = self.nodenet.partitions[partition_from_spid] if partition_from_spid in self.inlinks: theano_from_elements = self.inlinks[partition_from_spid][0] theano_to_elements = self.inlinks[partition_from_spid][1] @@ -1814,8 +1815,6 @@ def set_inlink_weights(self, partition_from_spid, new_from_elements, new_to_elem theano_to_elements = theano.shared(value=old_to_elements, name=toname, borrow=True) theano_weights = theano.shared(value=old_weights.astype(T.config.floatX), name=weightsname, borrow=True) - from_partition = self.nodenet.partitions[partition_from_spid] - propagation_function = self.get_compiled_propagate_inlinks( from_partition, theano_from_elements, @@ -1840,9 +1839,9 @@ def set_inlink_weights(self, partition_from_spid, new_from_elements, new_to_elem theano_to_elements.set_value(to_elements, borrow=True) theano_weights.set_value(weights, borrow=True) - for id in self.allocated_elements_to_nodes[theano_from_elements.get_value()]: - self.nodes_last_changed[id] = self.nodenet.current_step - self.nodespaces_contents_last_changed[self.allocated_node_parents[id]] = self.nodenet.current_step + for id in from_partition.allocated_elements_to_nodes[theano_from_elements.get_value()]: + from_partition.nodes_last_changed[id] = self.nodenet.current_step + from_partition.nodespaces_contents_last_changed[from_partition.allocated_node_parents[id]] = self.nodenet.current_step for id in self.allocated_elements_to_nodes[theano_to_elements.get_value()]: self.nodes_last_changed[id] = self.nodenet.current_step self.nodespaces_contents_last_changed[self.allocated_node_parents[id]] = self.nodenet.current_step From d5084d08087b65b2dd8d581caaf0697cb7cfb616 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 29 Feb 2016 11:02:17 +0100 Subject: [PATCH 111/306] gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f22e294b..da83dc01 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ /*.sublime-project /*.sublime-workspace micropsi.log -/test-data/ \ No newline at end of file +/test-data/ +pip-selfcheck.json From 83665d309e78688645c835ebadff06703f1628b2 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 29 Feb 2016 16:44:51 +0100 Subject: [PATCH 112/306] fix for reading/writing modulators via sensors/actors --- .../nodenet/theano_engine/theano_nodenet.py | 41 ++++++++++++++----- .../tests/test_runtime_nodenet_basics.py | 36 ++++++++++++++++ 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index df84fd0b..b948be17 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -800,6 +800,7 @@ def create_partition(self, pid, parent_uid, sparse, initial_number_of_nodes, ave self.partitionmap[parent_uid] = [] self.partitionmap[parent_uid].append(partition) self.inverted_partitionmap[partition.spid] = parent_uid + self._rebuild_sensor_actor_indices(partition) return partition.spid def delete_partition(self, pid): @@ -1273,8 +1274,19 @@ def get_standard_nodetype_definitions(self): def set_sensors_and_actuator_feedback_to_values(self, sensor_values, actuator_feedback_values): """ - Sets the sensors for the given data sources to the given values + Sets the sensors for the given data sources or modulators to the given values """ + if self.use_modulators: + # include modulators + readables = [0 for _ in DoernerianEmotionalModulators.readable_modulators] + for idx, key in enumerate(sorted(DoernerianEmotionalModulators.readable_modulators)): + readables[idx] = self.get_modulator(key) + sensor_values = sensor_values + readables + writeables = [0 for _ in DoernerianEmotionalModulators.writeable_modulators] + for idx, key in enumerate(sorted(DoernerianEmotionalModulators.writeable_modulators)): + writeables[idx] = 1 + actuator_feedback_values = actuator_feedback_values + writeables + for partition in self.partitions.values(): a_array = partition.a.get_value(borrow=True) a_array[partition.sensor_indices] = sensor_values @@ -1283,22 +1295,31 @@ def set_sensors_and_actuator_feedback_to_values(self, sensor_values, actuator_fe def read_actuators(self): """ - Returns a list of datatarget values for writing back to the world adapter + Returns a list of datatarget values for writing back to the world adapter, + writes modulator-values from datatargets """ actuator_values_to_write = np.zeros_like(self.rootpartition.actuator_indices) - for partition in self.partitions.values(): - if len(partition.actuator_indices): - a_array = partition.a.get_value(borrow=True) - actuator_values_to_write = actuator_values_to_write + a_array[partition.actuator_indices] - + a_array = partition.a.get_value(borrow=True) + actuator_values_to_write = actuator_values_to_write + a_array[partition.actuator_indices] + if self.use_modulators: + writeables = sorted(DoernerianEmotionalModulators.writeable_modulators) + # remove modulators from actuator values + modulator_values = actuator_values_to_write[-len(writeables):] + actuator_values_to_write = actuator_values_to_write[:-len(writeables)] + for idx, key in enumerate(writeables): + self.set_modulator(key, modulator_values[idx]) return actuator_values_to_write - def _rebuild_sensor_actor_indices(self): + def _rebuild_sensor_actor_indices(self, partition=None): """ - Rebuilds the actor and sensor indices for all partitions + Rebuilds the actor and sensor indices of the given partition or all partitions if None """ - for partition in self.partitions.values(): + if partition is not None: + partitions = [partition] + else: + partitions = self.partitions.values() + for partition in partitions: partition.sensor_indices = np.zeros(len(self.get_datasources()), np.int32) partition.actuator_indices = np.zeros(len(self.get_datatargets()), np.int32) for datatarget, node_id in self.actuatormap.items(): diff --git a/micropsi_core/tests/test_runtime_nodenet_basics.py b/micropsi_core/tests/test_runtime_nodenet_basics.py index 3b163bc0..41cf84e5 100644 --- a/micropsi_core/tests/test_runtime_nodenet_basics.py +++ b/micropsi_core/tests/test_runtime_nodenet_basics.py @@ -257,6 +257,42 @@ def test_modulators(fixed_nodenet, engine): assert 'Emotional' not in item.__class__.__name__ +def test_modulators_sensor_actor_connection(test_nodenet, test_world): + nodenet = micropsi.get_nodenet(test_nodenet) + micropsi.set_nodenet_properties(test_nodenet, worldadapter="Braitenberg", world_uid=test_world) + res, s1_id = micropsi.add_node(test_nodenet, "Sensor", [10, 10], None, name="brightness_l", parameters={'datasource': 'brightness_l'}) + res, s2_id = micropsi.add_node(test_nodenet, "Sensor", [20, 20], None, name="emo_activation", parameters={'datasource': 'emo_activation'}) + res, a1_id = micropsi.add_node(test_nodenet, "Actor", [30, 30], None, name="engine_l", parameters={'datatarget': 'engine_l'}) + res, a2_id = micropsi.add_node(test_nodenet, "Actor", [40, 40], None, name="base_importance_of_intention", parameters={'datatarget': 'base_importance_of_intention'}) + res, r1_id = micropsi.add_node(test_nodenet, "Register", [10, 30], None, name="r1") + res, r2_id = micropsi.add_node(test_nodenet, "Register", [10, 30], None, name="r2") + s1 = nodenet.get_node(s1_id) + s2 = nodenet.get_node(s2_id) + r1 = nodenet.get_node(r1_id) + r2 = nodenet.get_node(r2_id) + s2.set_gate_parameter('gen', 'maximum', 999) + micropsi.add_link(test_nodenet, r1_id, 'gen', a1_id, 'gen') + micropsi.add_link(test_nodenet, r2_id, 'gen', a2_id, 'gen') + r1.activation = 0.3 + r2.activation = 0.7 + emo_val = nodenet.get_modulator("emo_activation") + + # patch reset method, to check if datatarget was written + def nothing(): + pass + nodenet.worldadapter_instance.reset_datatargets = nothing + + nodenet.step() + assert round(nodenet.worldadapter_instance.datatargets['engine_l'], 3) == 0.3 + assert s1.activation == nodenet.worldadapter_instance.get_datasource_value('brightness_l') + assert s2.activation == emo_val + assert round(nodenet.get_modulator('base_importance_of_intention'), 3) == 0.7 + assert round(nodenet.worldadapter_instance.datatargets['engine_l'], 3) == 0.3 + emo_val = nodenet.get_modulator("emo_activation") + nodenet.step() + assert s2.activation == emo_val + + def test_node_parameters(fixed_nodenet, resourcepath): import os nodetype_file = os.path.join(resourcepath, 'Test', 'nodetypes.json') From 04d751e9cc4604526f26d82bc4f2c9cc935844e8 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 29 Feb 2016 19:03:08 +0100 Subject: [PATCH 113/306] avoid warnings b/c using an array as listindex DeprecationWarning: converting an array with ndim > 0 to an index will result in an error in the future --- micropsi_core/nodenet/theano_engine/theano_node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_node.py b/micropsi_core/nodenet/theano_engine/theano_node.py index 640ac48a..785c9560 100644 --- a/micropsi_core/nodenet/theano_engine/theano_node.py +++ b/micropsi_core/nodenet/theano_engine/theano_node.py @@ -324,14 +324,14 @@ def clone_parameters(self): if len(datasource_index) == 0: parameters['datasource'] = None else: - parameters['datasource'] = self._nodenet.get_datasources()[datasource_index] + parameters['datasource'] = self._nodenet.get_datasources()[datasource_index[0]] elif self.type == "Actor": actuator_element = self._partition.allocated_node_offsets[self._id] + GEN datatarget_index = np.where(self._partition.actuator_indices == actuator_element)[0] if len(datatarget_index) == 0: parameters['datatarget'] = None else: - parameters['datatarget'] = self._nodenet.get_datatargets()[datatarget_index] + parameters['datatarget'] = self._nodenet.get_datatargets()[datatarget_index[0]] elif self.type == "Activator": activator_type = None if self._id in self._partition.allocated_nodespaces_por_activators: From 3b4ef5940d87a121c607ebfa029091b154568bdd Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 29 Feb 2016 19:04:56 +0100 Subject: [PATCH 114/306] rebuild sensor actor indices * for a new nodenet's rootpartition * after loading the nodenet data from a file --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index b948be17..dd8483ef 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -258,6 +258,7 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No self.rootpartition = rootpartition self.partitionmap = {} self.inverted_partitionmap = {} + self._rebuild_sensor_actor_indices(rootpartition) self._version = NODENET_VERSION # used to check compatibility of the node net data self._step = 0 @@ -363,6 +364,8 @@ def load(self, filename): # re-initialize step operators for theano recompile to new shared variables self.initialize_stepoperators() + self._rebuild_sensor_actor_indices() + return True def remove(self, filename): From 1c08652d2ea7543d4f89f1485247b9c8b7933bb8 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 29 Feb 2016 19:06:00 +0100 Subject: [PATCH 115/306] fix for frontend: actors/sensors for modulators w/o worldadapters --- micropsi_server/static/js/nodenet.js | 50 +++++++++++++++------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index f868fe21..ea714c73 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -2916,15 +2916,18 @@ function openLinkCreationDialog(nodeUid){ } function get_datasource_options(worldadapter, value){ - var sources = worldadapters[worldadapter].datasources; - html = ''; - for(var i in sources){ - html += ''; - } - if(value && sources.indexOf(value) < 0) { - html += ''; + var html = ''; + if(worldadapter){ + var sources = worldadapters[worldadapter].datasources; + html += ''; + for(var i in sources){ + html += ''; + } + if(value && sources.indexOf(value) < 0) { + html += ''; + } + html += ''; } - html += ''; html += ''; for(var i in globalDataSources){ html += ''; @@ -2934,15 +2937,18 @@ function get_datasource_options(worldadapter, value){ } function get_datatarget_options(worldadapter, value){ - var targets = worldadapters[worldadapter].datatargets; - html = ''; - for(var i in targets){ - html += ''; - } - if(value && targets.indexOf(value) < 0) { - html += ''; + var html = ''; + if(worldadapter){ + var targets = worldadapters[worldadapter].datatargets; + html += ''; + for(var i in targets){ + html += ''; + } + if(value && targets.indexOf(value) < 0) { + html += ''; + } + html += ''; } - html += ''; html += ''; for(var i in globalDataTargets){ html += ''; @@ -3871,16 +3877,12 @@ function getNodeParameterHTML(parameters, parameter_values){ var i; switch(name){ case "datatarget": - if(currentWorldadapter in worldadapters){ - var opts = get_datatarget_options(currentWorldadapter, value); - input = ""; - } + var opts = get_datatarget_options(currentWorldadapter, value); + input = ""; break; case "datasource": - if(currentWorldadapter in worldadapters){ - var opts = get_datasource_options(currentWorldadapter, value); - input = ""; - } + var opts = get_datasource_options(currentWorldadapter, value); + input = ""; break; default: if(parameter_values && parameter_values[name]){ From 8b9397f965fec5b8e28c8f0007d3da0336440972 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 29 Feb 2016 19:06:47 +0100 Subject: [PATCH 116/306] do not reset the base porret decay factor --- micropsi_core/nodenet/stepoperators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/nodenet/stepoperators.py b/micropsi_core/nodenet/stepoperators.py index c04eb663..5636c015 100644 --- a/micropsi_core/nodenet/stepoperators.py +++ b/micropsi_core/nodenet/stepoperators.py @@ -188,7 +188,7 @@ def execute(self, nodenet, nodes, netapi): nodenet.set_modulator("base_number_of_expected_events", 0) nodenet.set_modulator("base_number_of_unexpected_events", 0) nodenet.set_modulator("base_urge_change", 0) - nodenet.set_modulator("base_porret_decay_factor", 1) + # nodenet.set_modulator("base_porret_decay_factor", 1) # setting emotional parameters nodenet.set_modulator("emo_pleasure", emo_pleasure) From 3a31f71cea5c76ca58699ea7f2984aaaba63f487 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 1 Mar 2016 17:23:03 +0100 Subject: [PATCH 117/306] remove outdated theano_netapi methods --- .../nodenet/theano_engine/theano_netapi.py | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_netapi.py b/micropsi_core/nodenet/theano_engine/theano_netapi.py index 521f0468..9b16c290 100644 --- a/micropsi_core/nodenet/theano_engine/theano_netapi.py +++ b/micropsi_core/nodenet/theano_engine/theano_netapi.py @@ -20,32 +20,6 @@ def floatX(self): def announce_nodes(self, nodespace_uid, numer_of_nodes, average_element_per_node): self.__nodenet.announce_nodes(nodespace_uid, numer_of_nodes, average_element_per_node) - def get_selectors(self, group): - """ - Returns The indices for the elements for the given group, as an ndarray of ints. - These indices are valid in a, w, and theta. - """ - return self.__nodenet.nodegroups[group] - - def get_a(self): - """ - Returns the theano shared variable with the activation vector - """ - return self.__nodenet.a - - def get_w(self): - """ - Returns the theano shared variable with the link weights - Caution: Changing non-zero values to zero or zero-values to non-zero will lead to inconsistencies. - """ - return self.__nodenet.w - - def get_theta(self): - """ - Returns the theano shared variable with the "theta" parameter values - """ - return self.__nodenet.g_theta - def decay_por_links(self, nodespace_uid): """ Decays all por-links in the given nodespace """ # por_cols = T.lvector("por_cols") From b28abcfef3d5eb18b0758be0d6a2d700f7f556ec Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 1 Mar 2016 17:23:30 +0100 Subject: [PATCH 118/306] test missing netpai method plus, make theano-version work with None-nodespace-identifier --- .../nodenet/theano_engine/theano_netapi.py | 1 + micropsi_core/tests/test_node_netapi.py | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/micropsi_core/nodenet/theano_engine/theano_netapi.py b/micropsi_core/nodenet/theano_engine/theano_netapi.py index 9b16c290..2d200c41 100644 --- a/micropsi_core/nodenet/theano_engine/theano_netapi.py +++ b/micropsi_core/nodenet/theano_engine/theano_netapi.py @@ -28,6 +28,7 @@ def decay_por_links(self, nodespace_uid): # self.decay = theano.function([por_cols, por_rows], None, updates={nodenet.w: new_w}, accept_inplace=True) import numpy as np from .theano_definitions import node_from_id, PIPE, POR + nodespace_uid = self.get_nodespace(nodespace_uid).uid porretdecay = self.__nodenet.get_modulator('base_porret_decay_factor') ns = self.get_nodespace(nodespace_uid) partition = ns._partition diff --git a/micropsi_core/tests/test_node_netapi.py b/micropsi_core/tests/test_node_netapi.py index 434acc5b..9d929f27 100644 --- a/micropsi_core/tests/test_node_netapi.py +++ b/micropsi_core/tests/test_node_netapi.py @@ -1074,3 +1074,26 @@ def test_set_dashboard_value(test_nodenet, node): netapi = nodenet.netapi netapi.set_dashboard_value('foo', 'bar') assert nodenet.dashboard_values['foo'] == 'bar' + + +def test_decay_porret_links(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + pipes = [] + netapi.set_modulator('base_porret_decay_factor', 0.1) + for i in range(10): + node = netapi.create_node("Pipe", None, "P%d" % i) + pipes.append(node) + if i > 0: + netapi.link_with_reciprocal(pipes[i - 1], node, 'porret', weight=0.1 * i) + + netapi.link_with_reciprocal(pipes[0], pipes[1], 'subsur', weight=0.5) + reg = netapi.create_node("Register", None, "source") + netapi.link(reg, 'gen', pipes[0], 'gen', 0.4) + netapi.decay_por_links(None) + for i in range(9): + assert round(pipes[i].get_gate('por').get_links()[0].weight, 3) == round(0.1 * (i + 1) * 0.9, 3) + # sub/sur/ret/gen links unchanged + assert round(reg.get_gate('gen').get_links()[0].weight, 3) == 0.4 + assert round(pipes[0].get_gate('sub').get_links()[0].weight, 3) == 0.5 + assert round(pipes[7].get_gate('ret').get_links()[0].weight, 3) == 0.7 From 3b3213938383697ff6d33607cdaa3e74a525b38e Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 1 Mar 2016 17:24:16 +0100 Subject: [PATCH 119/306] exclude the unsupported-warnings from test-coverage --- .../nodenet/theano_engine/theano_nodenet.py | 18 +++--- .../nodenet/theano_engine/theano_partition.py | 62 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index dd8483ef..38e30c11 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -206,7 +206,7 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No self.scipyfloatX = scipy.float64 self.numpyfloatX = np.float64 self.byte_per_float = 8 - else: + else: # pragma: no cover self.logger.warn("Unsupported precision value from configuration: %s, falling back to float64", precision) T.config.floatX = "float64" self.scipyfloatX = scipy.float64 @@ -230,14 +230,14 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No try: average_elements_per_node_assumption = int(configured_elements_per_node_assumption) except: - self.logger.warn("Unsupported elements_per_node_assumption value from configuration: %s, falling back to 4", configured_elements_per_node_assumption) + self.logger.warn("Unsupported elements_per_node_assumption value from configuration: %s, falling back to 4", configured_elements_per_node_assumption) # pragma: no cover initial_number_of_nodes = 2000 configured_initial_number_of_nodes = settings['theano']['initial_number_of_nodes'] try: initial_number_of_nodes = int(configured_initial_number_of_nodes) except: - self.logger.warn("Unsupported initial_number_of_nodes value from configuration: %s, falling back to 2000", configured_initial_number_of_nodes) + self.logger.warn("Unsupported initial_number_of_nodes value from configuration: %s, falling back to 2000", configured_initial_number_of_nodes) # pragma: no cover sparse = True configuredsparse = settings['theano']['sparse_weight_matrix'] @@ -246,7 +246,7 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No elif configuredsparse == "False": sparse = False else: - self.logger.warn("Unsupported sparse_weight_matrix value from configuration: %s, falling back to True", configuredsparse) + self.logger.warn("Unsupported sparse_weight_matrix value from configuration: %s, falling back to True", configuredsparse) # pragma: no cover sparse = True rootpartition = TheanoPartition(self, @@ -335,10 +335,10 @@ def load(self, filename): self.logger.info("Loading nodenet %s metadata from file %s", self.name, filename) with open(filename) as file: initfrom.update(json.load(file)) - except ValueError: + except ValueError: # pragma: no cover self.logger.warn("Could not read nodenet metadata from file %s", filename) return False - except IOError: + except IOError: # pragma: no cover self.logger.warn("Could not open nodenet metadata file %s", filename) return False @@ -855,7 +855,7 @@ def create_nodespace(self, parent_uid, position, name="", uid=None, options=None try: average_elements_per_node_assumption = int(configured_elements_per_node_assumption) except: - self.logger.warn("Unsupported elements_per_node_assumption value from configuration: %s, falling back to 4", configured_elements_per_node_assumption) + self.logger.warn("Unsupported elements_per_node_assumption value from configuration: %s, falling back to 4", configured_elements_per_node_assumption) # pragma: no cover initial_number_of_nodes = 2000 if "initial_number_of_nodes" in options: @@ -865,7 +865,7 @@ def create_nodespace(self, parent_uid, position, name="", uid=None, options=None try: initial_number_of_nodes = int(configured_initial_number_of_nodes) except: - self.logger.warn("Unsupported initial_number_of_nodes value from configuration: %s, falling back to 2000", configured_initial_number_of_nodes) + self.logger.warn("Unsupported initial_number_of_nodes value from configuration: %s, falling back to 2000", configured_initial_number_of_nodes) # pragma: no cover sparse = True if "sparse" in options: @@ -877,7 +877,7 @@ def create_nodespace(self, parent_uid, position, name="", uid=None, options=None elif configuredsparse == "False": sparse = False else: - self.logger.warn("Unsupported sparse_weight_matrix value from configuration: %s, falling back to True", configuredsparse) + self.logger.warn("Unsupported sparse_weight_matrix value from configuration: %s, falling back to True", configuredsparse) # pragma: no cover sparse = True self.last_allocated_partition += 1 diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index 989faa54..c52623dc 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -811,10 +811,10 @@ def load_data(self, datafilename, nodes_data): try: self.logger.info("Loading nodenet %s partition %i bulk data from file %s" % (self.nodenet.name, self.pid, datafilename)) datafile = np.load(datafilename) - except ValueError: + except ValueError: # pragma: no cover self.logger.warn("Could not read nodenet data from file %s" % datafile) return False - except IOError: + except IOError: # pragma: no cover self.logger.warn("Could not open nodenet file %s" % datafile) return False @@ -835,73 +835,73 @@ def load_data(self, datafilename, nodes_data): self.a_prev = theano.shared(value=a_prev_array.astype(T.config.floatX), name="a_prev", borrow=True) else: - self.logger.warn("no sizeinformation in file, falling back to defaults") + self.logger.warn("no sizeinformation in file, falling back to defaults") # pragma: no cover # the load bulk data into numpy arrays if 'allocated_nodes' in datafile: self.allocated_nodes = datafile['allocated_nodes'] else: - self.logger.warn("no allocated_nodes in file, falling back to defaults") + self.logger.warn("no allocated_nodes in file, falling back to defaults") # pragma: no cover if 'allocated_node_offsets' in datafile: self.allocated_node_offsets = datafile['allocated_node_offsets'] else: - self.logger.warn("no allocated_node_offsets in file, falling back to defaults") + self.logger.warn("no allocated_node_offsets in file, falling back to defaults") # pragma: no cover if 'allocated_elements_to_nodes' in datafile: self.allocated_elements_to_nodes = datafile['allocated_elements_to_nodes'] else: - self.logger.warn("no allocated_elements_to_nodes in file, falling back to defaults") + self.logger.warn("no allocated_elements_to_nodes in file, falling back to defaults") # pragma: no cover if 'allocated_nodespaces' in datafile: self.allocated_nodespaces = datafile['allocated_nodespaces'] else: - self.logger.warn("no allocated_nodespaces in file, falling back to defaults") + self.logger.warn("no allocated_nodespaces in file, falling back to defaults") # pragma: no cover if 'allocated_node_parents' in datafile: self.allocated_node_parents = datafile['allocated_node_parents'] else: - self.logger.warn("no allocated_node_parents in file, falling back to defaults") + self.logger.warn("no allocated_node_parents in file, falling back to defaults") # pragma: no cover if 'allocated_elements_to_activators' in datafile: self.allocated_elements_to_activators = datafile['allocated_elements_to_activators'] else: - self.logger.warn("no allocated_elements_to_activators in file, falling back to defaults") + self.logger.warn("no allocated_elements_to_activators in file, falling back to defaults") # pragma: no cover if 'allocated_nodespaces_por_activators' in datafile: self.allocated_nodespaces_por_activators = datafile['allocated_nodespaces_por_activators'] else: - self.logger.warn("no allocated_nodespaces_por_activators in file, falling back to defaults") + self.logger.warn("no allocated_nodespaces_por_activators in file, falling back to defaults") # pragma: no cover if 'allocated_nodespaces_ret_activators' in datafile: self.allocated_nodespaces_ret_activators = datafile['allocated_nodespaces_ret_activators'] else: - self.logger.warn("no allocated_nodespaces_ret_activators in file, falling back to defaults") + self.logger.warn("no allocated_nodespaces_ret_activators in file, falling back to defaults") # pragma: no cover if 'allocated_nodespaces_sub_activators' in datafile: self.allocated_nodespaces_sub_activators = datafile['allocated_nodespaces_sub_activators'] else: - self.logger.warn("no allocated_nodespaces_sub_activators in file, falling back to defaults") + self.logger.warn("no allocated_nodespaces_sub_activators in file, falling back to defaults") # pragma: no cover if 'allocated_nodespaces_sur_activators' in datafile: self.allocated_nodespaces_sur_activators = datafile['allocated_nodespaces_sur_activators'] else: - self.logger.warn("no allocated_nodespaces_sur_activators in file, falling back to defaults") + self.logger.warn("no allocated_nodespaces_sur_activators in file, falling back to defaults") # pragma: no cover if 'allocated_nodespaces_cat_activators' in datafile: self.allocated_nodespaces_cat_activators = datafile['allocated_nodespaces_cat_activators'] else: - self.logger.warn("no allocated_nodespaces_cat_activators in file, falling back to defaults") + self.logger.warn("no allocated_nodespaces_cat_activators in file, falling back to defaults") # pragma: no cover if 'allocated_nodespaces_exp_activators' in datafile: self.allocated_nodespaces_exp_activators = datafile['allocated_nodespaces_exp_activators'] else: - self.logger.warn("no allocated_nodespaces_exp_activators in file, falling back to defaults") + self.logger.warn("no allocated_nodespaces_exp_activators in file, falling back to defaults") # pragma: no cover if 'allocated_nodespaces_sampling_activators' in datafile: self.allocated_nodespaces_sampling_activators = datafile['allocated_nodespaces_sampling_activators'] else: - self.logger.warn("no allocated_nodespaces_por_activators in file, falling back to defaults") + self.logger.warn("no allocated_nodespaces_por_activators in file, falling back to defaults") # pragma: no cover if 'w_data' in datafile and 'w_indices' in datafile and 'w_indptr' in datafile: w = sp.csr_matrix((datafile['w_data'], datafile['w_indices'], datafile['w_indptr']), shape = (self.NoE, self.NoE)) @@ -912,62 +912,62 @@ def load_data(self, datafilename, nodes_data): self.a = theano.shared(value=datafile['a'].astype(T.config.floatX), name="a", borrow=False) self.a_in = theano.shared(value=np.zeros_like(datafile['a']).astype(T.config.floatX), name="a_in", borrow=False) else: - self.logger.warn("no w_data, w_indices or w_indptr in file, falling back to defaults") + self.logger.warn("no w_data, w_indices or w_indptr in file, falling back to defaults") # pragma: no cover if 'g_theta' in datafile: self.g_theta = theano.shared(value=datafile['g_theta'].astype(T.config.floatX), name="theta", borrow=False) else: - self.logger.warn("no g_theta in file, falling back to defaults") + self.logger.warn("no g_theta in file, falling back to defaults") # pragma: no cover if 'g_factor' in datafile: self.g_factor = theano.shared(value=datafile['g_factor'].astype(T.config.floatX), name="g_factor", borrow=False) else: - self.logger.warn("no g_factor in file, falling back to defaults") + self.logger.warn("no g_factor in file, falling back to defaults") # pragma: no cover if 'g_threshold' in datafile: self.g_threshold = theano.shared(value=datafile['g_threshold'].astype(T.config.floatX), name="g_threshold", borrow=False) else: - self.logger.warn("no g_threshold in file, falling back to defaults") + self.logger.warn("no g_threshold in file, falling back to defaults") # pragma: no cover if 'g_amplification' in datafile: self.g_amplification = theano.shared(value=datafile['g_amplification'].astype(T.config.floatX), name="g_amplification", borrow=False) else: - self.logger.warn("no g_amplification in file, falling back to defaults") + self.logger.warn("no g_amplification in file, falling back to defaults") # pragma: no cover if 'g_min' in datafile: self.g_min = theano.shared(value=datafile['g_min'].astype(T.config.floatX), name="g_min", borrow=False) else: - self.logger.warn("no g_min in file, falling back to defaults") + self.logger.warn("no g_min in file, falling back to defaults") # pragma: no cover if 'g_max' in datafile: self.g_max = theano.shared(value=datafile['g_max'].astype(T.config.floatX), name="g_max", borrow=False) else: - self.logger.warn("no g_max in file, falling back to defaults") + self.logger.warn("no g_max in file, falling back to defaults") # pragma: no cover if 'g_function_selector' in datafile: self.g_function_selector = theano.shared(value=datafile['g_function_selector'], name="gatefunction", borrow=False) else: - self.logger.warn("no g_function_selector in file, falling back to defaults") + self.logger.warn("no g_function_selector in file, falling back to defaults") # pragma: no cover if 'g_expect' in datafile: self.g_expect = theano.shared(value=datafile['g_expect'], name="expectation", borrow=False) else: - self.logger.warn("no g_expect in file, falling back to defaults") + self.logger.warn("no g_expect in file, falling back to defaults") # pragma: no cover if 'g_countdown' in datafile: self.g_countdown = theano.shared(value=datafile['g_countdown'], name="countdown", borrow=False) else: - self.logger.warn("no g_countdown in file, falling back to defaults") + self.logger.warn("no g_countdown in file, falling back to defaults") # pragma: no cover if 'g_wait' in datafile: self.g_wait = theano.shared(value=datafile['g_wait'], name="wait", borrow=False) else: - self.logger.warn("no g_wait in file, falling back to defaults") + self.logger.warn("no g_wait in file, falling back to defaults") # pragma: no cover if 'n_function_selector' in datafile: self.n_function_selector = theano.shared(value=datafile['n_function_selector'], name="nodefunction_per_gate", borrow=False) else: - self.logger.warn("no n_function_selector in file, falling back to defaults") + self.logger.warn("no n_function_selector in file, falling back to defaults") # pragma: no cover # reconstruct other states self.por_ret_dirty = True @@ -1029,10 +1029,10 @@ def load_inlinks(self, datafilename): if os.path.isfile(datafilename): try: datafile = np.load(datafilename) - except ValueError: + except ValueError: # pragma: no cover self.logger.warn("Could not read nodenet data from file %s" % datafile) return False - except IOError: + except IOError: # pragma: no cover self.logger.warn("Could not open nodenet file %s" % datafile) return False @@ -1071,7 +1071,7 @@ def load_inlinks(self, datafilename): inlink_from_offset += inlink_from_lengths[i] inlink_to_offset += inlink_to_lengths[i] else: - self.logger.warn("no or incomplete inlink information in file, no inter-partition links will be loaded") + self.logger.warn("no or incomplete inlink information in file, no inter-partition links will be loaded") # pragma: no cover def grow_number_of_nodespaces(self, growby): From 02b16372549bee17c01cbb76427b15c41126bb7c Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 1 Mar 2016 17:25:26 +0100 Subject: [PATCH 120/306] some basic tests for partitions: * creation, * x-partition links * deletion * growing --- .../tests/test_nodenet_partitions.py | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 micropsi_core/tests/test_nodenet_partitions.py diff --git a/micropsi_core/tests/test_nodenet_partitions.py b/micropsi_core/tests/test_nodenet_partitions.py new file mode 100644 index 00000000..d4bc6304 --- /dev/null +++ b/micropsi_core/tests/test_nodenet_partitions.py @@ -0,0 +1,134 @@ + +import pytest +from micropsi_core import runtime as micropsi + + +def prepare(netapi, partition_options={}): + partition_options.update({'new_partition': True}) + nodespace = netapi.create_nodespace(None, name="partition", options=partition_options) + source = netapi.create_node('Register', None, "Source") + register = netapi.create_node('Register', nodespace.uid, "Register") + netapi.link(source, 'gen', register, 'gen') + netapi.link(source, 'gen', source, 'gen') + source.activation = 1 + return nodespace, source, register + + +@pytest.mark.engine("theano_engine") +def test_partition_creation(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + netapi.create_nodespace(None, name="partition", options={'new_partition': True}) + assert len(nodenet.partitions.keys()) == 2 + + +@pytest.mark.engine("theano_engine") +def test_cross_partition_links(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + nodespace, source, register = prepare(netapi) + nodenet.step() + assert register.activation == 1 + # change link weight + netapi.link(source, 'gen', register, 'gen', weight=0.7) + link = register.get_slot('gen').get_links()[0] + assert round(link.weight, 3) == 0.7 + nodenet.step() + assert round(register.activation, 3) == 0.7 + netapi.unlink(source, 'gen', register, 'gen', ) + assert netapi.get_node(register.uid).get_gate('gen').get_links() == [] + nodenet.step() + assert register.activation == 0 + + +@pytest.mark.engine("theano_engine") +def test_partition_persistence(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + nodespace, source, register = prepare(netapi) + micropsi.save_nodenet(test_nodenet) + micropsi.revert_nodenet(test_nodenet) + nodenet.step() + assert register.activation == 1 + + +@pytest.mark.engine("theano_engine") +def test_delete_node(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + nodespace, source, register = prepare(netapi) + netapi.delete_node(register) + links = netapi.get_node(source.uid).get_gate('gen').get_links() + assert len(links) == 1 + assert links[0].target_node.uid == source.uid + + +@pytest.mark.engine("theano_engine") +def test_grow_partitions(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + nodespace = netapi.create_nodespace(None, name="partition", options={ + "new_partition": True, + "initial_number_of_nodes": 2, + "average_elements_per_node_assumption": 4, + "initial_number_of_nodespaces": 1 + }) + + for i in range(20): + netapi.create_node("Pipe", nodespace.uid, "N %d" % i) + + partition = None + for p in nodenet.partitions.values(): + if p != nodenet.rootpartition: + partition = p + # growby (NoN // 2): 2,3,4,6,9,13,19,28 + assert len(partition.allocated_nodes) == 28 + assert partition.NoE > 28 * 4 + + for i in range(2): + netapi.create_nodespace(nodespace.uid, name="NS %d" % i) + + assert len(partition.allocated_nodespaces) == 4 + + +@pytest.mark.engine("theano_engine") +def test_announce_nodes(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + nodespace = netapi.create_nodespace(None, name="partition", options={ + "new_partition": True, + "initial_number_of_nodes": 2, + "average_elements_per_node_assumption": 4, + "initial_number_of_nodespaces": 1 + }) + + # announce 20 pipe nodes + netapi.announce_nodes(nodespace.uid, 20, 8) + + partition = None + for p in nodenet.partitions.values(): + if p != nodenet.rootpartition: + partition = p + + # 18 nodes needed + assert partition.NoN == 26 # growby: 18 + 18//3 + # 152 elements needed + assert partition.NoE == 210 # growby: 152 + 152//3 + + for i in range(20): + netapi.create_node("Pipe", nodespace.uid, "N %d" % i) + + # assert that we did not grow again + assert partition.NoN == 26 + assert partition.NoE == 210 + + +@pytest.mark.engine("theano_engine") +def test_delete_partition(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + nodespace, source, register = prepare(netapi) + netapi.delete_nodespace(nodespace) + links = source.get_gate('gen').get_links() + assert len(links) == 1 + assert links[0].target_node == source From 9bcac802da1f11c07bb172d83d588e1075773159 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 1 Mar 2016 17:34:01 +0100 Subject: [PATCH 121/306] fix no covers --- .../nodenet/theano_engine/theano_nodenet.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 38e30c11..b5d1a160 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -229,15 +229,15 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No configured_elements_per_node_assumption = settings['theano']['elements_per_node_assumption'] try: average_elements_per_node_assumption = int(configured_elements_per_node_assumption) - except: - self.logger.warn("Unsupported elements_per_node_assumption value from configuration: %s, falling back to 4", configured_elements_per_node_assumption) # pragma: no cover + except: # pragma: no cover + self.logger.warn("Unsupported elements_per_node_assumption value from configuration: %s, falling back to 4", configured_elements_per_node_assumption) initial_number_of_nodes = 2000 configured_initial_number_of_nodes = settings['theano']['initial_number_of_nodes'] try: initial_number_of_nodes = int(configured_initial_number_of_nodes) - except: - self.logger.warn("Unsupported initial_number_of_nodes value from configuration: %s, falling back to 2000", configured_initial_number_of_nodes) # pragma: no cover + except: # pragma: no cover + self.logger.warn("Unsupported initial_number_of_nodes value from configuration: %s, falling back to 2000", configured_initial_number_of_nodes) sparse = True configuredsparse = settings['theano']['sparse_weight_matrix'] @@ -245,8 +245,8 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No sparse = True elif configuredsparse == "False": sparse = False - else: - self.logger.warn("Unsupported sparse_weight_matrix value from configuration: %s, falling back to True", configuredsparse) # pragma: no cover + else: # pragma: no cover + self.logger.warn("Unsupported sparse_weight_matrix value from configuration: %s, falling back to True", configuredsparse) sparse = True rootpartition = TheanoPartition(self, From db4f8d2e76827d91d5e783d4cc97a58b84eb8f7f Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 1 Mar 2016 18:03:48 +0100 Subject: [PATCH 122/306] also test node.get_associated_node_uids --- micropsi_core/tests/test_nodenet_partitions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/micropsi_core/tests/test_nodenet_partitions.py b/micropsi_core/tests/test_nodenet_partitions.py index d4bc6304..890b8ca7 100644 --- a/micropsi_core/tests/test_nodenet_partitions.py +++ b/micropsi_core/tests/test_nodenet_partitions.py @@ -31,12 +31,18 @@ def test_cross_partition_links(test_nodenet): assert register.activation == 1 # change link weight netapi.link(source, 'gen', register, 'gen', weight=0.7) + + assert register.uid in netapi.get_node(source.uid).get_associated_node_uids() + assert source.uid in netapi.get_node(register.uid).get_associated_node_uids() + link = register.get_slot('gen').get_links()[0] assert round(link.weight, 3) == 0.7 nodenet.step() assert round(register.activation, 3) == 0.7 - netapi.unlink(source, 'gen', register, 'gen', ) - assert netapi.get_node(register.uid).get_gate('gen').get_links() == [] + netapi.unlink(source, 'gen', register, 'gen') + assert len(source.get_gate('gen').get_links()) == 1 + assert netapi.get_node(register.uid).get_gate('gen').empty + assert netapi.get_node(register.uid).get_slot('gen').empty nodenet.step() assert register.activation == 0 From ea7db12102829bfe1f7670a0dea8e299c1564036 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 1 Mar 2016 18:30:35 +0100 Subject: [PATCH 123/306] also make assertions about partition.inlinks and, also test node_deletion with remaining inlinks --- .../nodenet/theano_engine/theano_nodespace.py | 2 +- .../nodenet/theano_engine/theano_partition.py | 1 - .../tests/test_nodenet_partitions.py | 40 ++++++++++++++----- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodespace.py b/micropsi_core/nodenet/theano_engine/theano_nodespace.py index 6cfa461e..5330bd70 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodespace.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodespace.py @@ -77,7 +77,7 @@ def get_known_ids(self, entitytype=None): for partition in self._nodenet.partitionmap[self.uid]: uids.append(partition.rootnodespace_uid) return uids - elif entitytype == None: + elif entitytype is None: ids = self.get_known_ids('nodes') ids.extend(self.get_known_ids('nodespaces')) return ids diff --git a/micropsi_core/nodenet/theano_engine/theano_partition.py b/micropsi_core/nodenet/theano_engine/theano_partition.py index c52623dc..46f12f4d 100644 --- a/micropsi_core/nodenet/theano_engine/theano_partition.py +++ b/micropsi_core/nodenet/theano_engine/theano_partition.py @@ -671,7 +671,6 @@ def rebuild_ret_linked(self): self.n_node_retlinked.set_value(n_node_retlinked_array) def grow_number_of_nodes(self, growby): - new_NoN = int(self.NoN + growby) new_allocated_nodes = np.zeros(new_NoN, dtype=np.int32) diff --git a/micropsi_core/tests/test_nodenet_partitions.py b/micropsi_core/tests/test_nodenet_partitions.py index 890b8ca7..88756b65 100644 --- a/micropsi_core/tests/test_nodenet_partitions.py +++ b/micropsi_core/tests/test_nodenet_partitions.py @@ -59,14 +59,41 @@ def test_partition_persistence(test_nodenet): @pytest.mark.engine("theano_engine") -def test_delete_node(test_nodenet): +def test_delete_node_deletes_inlinks(test_nodenet): nodenet = micropsi.get_nodenet(test_nodenet) netapi = nodenet.netapi nodespace, source, register = prepare(netapi) + target = netapi.create_node("Register", None, "target") + netapi.link(register, 'gen', target, 'gen') netapi.delete_node(register) links = netapi.get_node(source.uid).get_gate('gen').get_links() assert len(links) == 1 assert links[0].target_node.uid == source.uid + assert target.get_slot('gen').empty + assert nodespace.partition.inlinks == {} + assert len(nodenet.rootpartition.inlinks[nodespace.partition.spid][1].get_value()) == 0 + + +@pytest.mark.engine("theano_engine") +def test_delete_node_modifies_inlinks(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + nodespace, source, register = prepare(netapi) + target = netapi.create_node("Register", None, "target") + + register2 = netapi.create_node("Register", nodespace.uid, "reg2") + netapi.link(register, 'gen', target, 'gen') + netapi.link(register2, 'gen', target, 'gen') + netapi.link(source, 'gen', register2, 'gen') + + netapi.delete_node(register) + assert len(source.get_gate('gen').get_links()) == 2 + assert len(target.get_slot('gen').get_links()) == 1 + + assert list(nodespace.partition.inlinks.keys()) == [nodenet.rootpartition.spid] + assert list(nodenet.rootpartition.inlinks.keys()) == [nodespace.partition.spid] + assert len(nodespace.partition.inlinks[nodenet.rootpartition.spid][1].get_value()) == 1 + assert len(nodenet.rootpartition.inlinks[nodespace.partition.spid][1].get_value()) == 1 @pytest.mark.engine("theano_engine") @@ -83,10 +110,8 @@ def test_grow_partitions(test_nodenet): for i in range(20): netapi.create_node("Pipe", nodespace.uid, "N %d" % i) - partition = None - for p in nodenet.partitions.values(): - if p != nodenet.rootpartition: - partition = p + partition = nodespace.partition + # growby (NoN // 2): 2,3,4,6,9,13,19,28 assert len(partition.allocated_nodes) == 28 assert partition.NoE > 28 * 4 @@ -111,10 +136,7 @@ def test_announce_nodes(test_nodenet): # announce 20 pipe nodes netapi.announce_nodes(nodespace.uid, 20, 8) - partition = None - for p in nodenet.partitions.values(): - if p != nodenet.rootpartition: - partition = p + partition = nodespace.partition # 18 nodes needed assert partition.NoN == 26 # growby: 18 + 18//3 From ed0f1b80a1c84aabcef1beecce7a2b4c34e67465 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 2 Mar 2016 11:58:36 +0100 Subject: [PATCH 124/306] reset runner-config for each testrun --- conftest.py | 1 + micropsi_core/tests/test_runtime.py | 2 -- micropsi_core/tests/test_runtime_world_basics.py | 1 - micropsi_core/tests/test_vizapi.py | 2 -- 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/conftest.py b/conftest.py index 6c056cb3..f265cd77 100644 --- a/conftest.py +++ b/conftest.py @@ -91,6 +91,7 @@ def pytest_runtest_setup(item): open(os.path.join(testpath, 'Test', '__init__.py'), 'w').close() micropsi.reload_native_modules() micropsi.logger.clear_logs() + micropsi.set_runner_properties(1, 1) set_logging_levels() diff --git a/micropsi_core/tests/test_runtime.py b/micropsi_core/tests/test_runtime.py index 855d1e29..50583ea7 100644 --- a/micropsi_core/tests/test_runtime.py +++ b/micropsi_core/tests/test_runtime.py @@ -85,7 +85,6 @@ def test_get_multiple_logger_messages_are_sorted(): def test_register_runner_condition_step(test_nodenet): import time - micropsi.set_runner_properties(1, 1) success, data = micropsi.set_runner_condition(test_nodenet, steps=7) assert data['step'] == 7 assert data['step_amount'] == 7 @@ -104,7 +103,6 @@ def test_register_runner_condition_step(test_nodenet): def test_register_runner_condition_monitor(test_nodenet): import time - micropsi.set_runner_properties(1, 1) nn = micropsi.nodenets[test_nodenet] node = nn.netapi.create_node('Register', None) nn.netapi.link(node, 'gen', node, 'gen', weight=2) diff --git a/micropsi_core/tests/test_runtime_world_basics.py b/micropsi_core/tests/test_runtime_world_basics.py index 1997bcfc..d0f0bb0a 100644 --- a/micropsi_core/tests/test_runtime_world_basics.py +++ b/micropsi_core/tests/test_runtime_world_basics.py @@ -172,7 +172,6 @@ def test_worldadapter_update_calls_reset_datatargets(test_world, test_nodenet): runtime.load_nodenet(test_nodenet) nodenet.world = test_world runtime.set_nodenet_properties(nodenet.uid, worldadapter='Braitenberg', world_uid=world.uid) - runtime.set_runner_properties(1, 1) world.agents[test_nodenet].reset_datatargets = mock.MagicMock(name='reset') runtime.step_nodenet(test_nodenet) world.agents[test_nodenet].reset_datatargets.assert_called_once_with() diff --git a/micropsi_core/tests/test_vizapi.py b/micropsi_core/tests/test_vizapi.py index 55ae1e1f..95aea62e 100644 --- a/micropsi_core/tests/test_vizapi.py +++ b/micropsi_core/tests/test_vizapi.py @@ -85,10 +85,8 @@ def plotfunc(netapi, node=None, **params): micropsi.reload_native_modules() node = nodenet.netapi.create_node("Plotter", None, name="Plotter") node.set_parameter("plotpath", resourcepath) - micropsi.set_runner_properties(1000, 1) micropsi.start_nodenetrunner(test_nodenet) sleep(2) micropsi.stop_nodenetrunner(test_nodenet) - print(micropsi.MicropsiRunner.last_nodenet_exception) assert micropsi.MicropsiRunner.last_nodenet_exception == {} assert os.path.isfile(os.path.join(resourcepath, "plot.png")) From 8253c462653416cf75effaab228f3d730ebfa61f Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 2 Mar 2016 12:06:03 +0100 Subject: [PATCH 125/306] fix the demo-worldadapter --- micropsi_core/world/worldadapter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/micropsi_core/world/worldadapter.py b/micropsi_core/world/worldadapter.py index 4a0b5758..3ff2d810 100644 --- a/micropsi_core/world/worldadapter.py +++ b/micropsi_core/world/worldadapter.py @@ -107,11 +107,13 @@ def is_alive(self): class Default(WorldAdapter): """ - The default world adapter + A default Worldadapter, that provides example-datasources and -targets """ def __init__(self, world, uid=None, **data): + super().__init__(world, uid=uid, **data) self.datasources = dict((s, 0) for s in ['static_on', 'random', 'static_off']) self.datatargets = {'echo': 0} + self.datatarget_feedback = {'echo': 0} def update_data_sources_and_targets(self): import random From d7d98497590beae0183c87da5f819c0d76d873c2 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 2 Mar 2016 12:06:23 +0100 Subject: [PATCH 126/306] test actor/sensor indices --- .../tests/test_nodenet_partitions.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/micropsi_core/tests/test_nodenet_partitions.py b/micropsi_core/tests/test_nodenet_partitions.py index 88756b65..927d445f 100644 --- a/micropsi_core/tests/test_nodenet_partitions.py +++ b/micropsi_core/tests/test_nodenet_partitions.py @@ -160,3 +160,29 @@ def test_delete_partition(test_nodenet): links = source.get_gate('gen').get_links() assert len(links) == 1 assert links[0].target_node == source + + +@pytest.mark.engine("theano_engine") +def test_sensor_actuator_indices(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + result, world_uid = micropsi.new_world('default', 'World') + micropsi.set_nodenet_properties(test_nodenet, worldadapter='Default', world_uid=world_uid) + sensor = netapi.create_node("Sensor", None, "static_sensor") + sensor.set_parameter("datasource", "static_on") + actor = netapi.create_node("Actor", None, "echo_actor") + actor.set_parameter("datatarget", "echo") + register = netapi.create_node("Register", None, "source") + register.activation = 0.8 + netapi.link(register, 'gen', register, 'gen', weight=0.5) + netapi.link(register, 'gen', actor, 'gen') + micropsi.step_nodenet(test_nodenet) + assert sensor.activation == 0 + assert actor.get_gate('gen').activation == 0 + micropsi.step_nodenet(test_nodenet) + assert sensor.activation == 1 + assert round(actor.get_gate('gen').activation, 3) == 0.8 + netapi.delete_node(sensor) + netapi.delete_node(actor) + assert set(nodenet.rootpartition.actuator_indices) == {0} + assert set(nodenet.rootpartition.sensor_indices) == {0} From bbe4d009a8f63ae91248941f2772d095ea5ef6f9 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 2 Mar 2016 15:53:07 +0100 Subject: [PATCH 127/306] replace tools.mkdir with builtin, pep8ify tools --- micropsi_core/config.py | 2 +- micropsi_core/runtime.py | 2 +- micropsi_core/tools.py | 48 ++++++++++++++----------------- micropsi_server/usermanagement.py | 2 +- 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/micropsi_core/config.py b/micropsi_core/config.py index 3ce9571e..e5d62559 100644 --- a/micropsi_core/config.py +++ b/micropsi_core/config.py @@ -58,7 +58,7 @@ def __init__(self, config_path="config-data.json", auto_save=True): self.key = absolute_path # set up persistence - micropsi_core.tools.mkdir(os.path.dirname(config_path)) + os.makedirs(os.path.dirname(config_path), exist_ok=True) self.config_file_name = config_path self.auto_save = auto_save diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index a1975dbd..6ce599e1 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1201,7 +1201,7 @@ def crawl_definition_files(path, type="definition"): """ result = {} - tools.mkdir(path) + os.makedirs(path, exist_ok=True) for user_directory_name, user_directory_names, file_names in os.walk(path): for definition_file_name in file_names: diff --git a/micropsi_core/tools.py b/micropsi_core/tools.py index 2349a401..981febf8 100644 --- a/micropsi_core/tools.py +++ b/micropsi_core/tools.py @@ -9,27 +9,14 @@ __date__ = '29.06.12' import uuid -import os + def generate_uid(): """produce a unique identifier, restricted to an ASCII string""" return uuid.uuid1().hex -def mkdir(new_directory_name): - """if the directory does not exist, create it; otherwise, exit quietly""" - - if os.path.isdir(new_directory_name): - pass - elif os.path.isfile(new_directory_name): - raise OSError("a file with the same name as the desired directory, '%s', already exists." % new_directory_name) - else: - head, tail = os.path.split(new_directory_name) - if head and not os.path.isdir(head): - mkdir(head) - if tail: - os.mkdir(new_directory_name) -def check_for_url_proof_id(id, existing_ids = None, min_id_length = 1, max_id_length = 21): +def check_for_url_proof_id(id, existing_ids=None, min_id_length=1, max_id_length=21): """Returns (True, id) if id is permissible, and (False, error message) otherwise. Since we strip the id, you should use the returned one, not the original one""" @@ -38,9 +25,10 @@ def check_for_url_proof_id(id, existing_ids = None, min_id_length = 1, max_id_le # maybe this is too restrictive, but I want to use the id directly in urls for c in id: if not c.lower() in "0123456789abcdefghijklmnopqrstuvwxyz@._-": - return False, "The character '%s' is not allowed" %c + return False, "The character '%s' is not allowed" % c - if existing_ids and id.lower() in existing_ids: return False, "ID already exists" + if existing_ids and id.lower() in existing_ids: + return False, "ID already exists" if len(id) < min_id_length: return False, "Must at least have %s characters" % min_id_length if len(id) > max_id_length: @@ -49,7 +37,6 @@ def check_for_url_proof_id(id, existing_ids = None, min_id_length = 1, max_id_le return True, id - # Global parameters for all created functions # symbols that are included by default in the generated function's environment @@ -61,12 +48,15 @@ def check_for_url_proof_id(id, existing_ids = None, min_id_length = 1, max_id_le # add standard exceptions __bi = __builtins__ -if type(__bi) is not dict: __bi = __bi.__dict__ +if type(__bi) is not dict: + __bi = __bi.__dict__ for k in __bi: - if k.endswith("Error") or k.endswith("Warning"): SAFE_SYMBOLS.append(k) + if k.endswith("Error") or k.endswith("Warning"): + SAFE_SYMBOLS.append(k) del __bi -def create_function(source_string, parameters = "", additional_symbols = None): + +def create_function(source_string, parameters="", additional_symbols=None): """Create a python function from the given source code. Arguments: @@ -86,7 +76,7 @@ def create_function(source_string, parameters = "", additional_symbols = None): get_user = create_function(my_function_source, parameters="index = 0", additional_symbols = {'usermanager': usermanager}) print get_user("klaus") - + (This function is inspired by a recipe by David Decotigny.) """ @@ -99,12 +89,12 @@ def create_function(source_string, parameters = "", additional_symbols = None): # Setup the local and global dictionaries of the execution # environment for __my_function__ - bis = dict() # builtins + bis = dict() # builtins globs = dict() - locs = dict() + locs = dict() # Setup a standard-compatible python environment - bis["locals"] = lambda: locs + bis["locals"] = lambda: locs bis["globals"] = lambda: globs globs["__builtins__"] = bis globs["__name__"] = "SUBENV" @@ -159,8 +149,10 @@ def __init__(self, **kwargs): for i in kwargs: self[i] = kwargs[i] + import collections + class OrderedSet(collections.OrderedDict, collections.MutableSet): def update(self, *args, **kwargs): @@ -205,6 +197,7 @@ def __str__(self): symmetric_difference_update = property(lambda self: self.__ixor__) union = property(lambda self: self.__or__) + def itersubclasses(cls, folder=None, _seen=None): """ Generator over all subclasses of a given class, in depth first order. @@ -216,10 +209,11 @@ def itersubclasses(cls, folder=None, _seen=None): if not isinstance(cls, type): raise TypeError('itersubclasses must be called with new-style classes, not %.100r' % cls) - if _seen is None: _seen = set() + if _seen is None: + _seen = set() try: subs = cls.__subclasses__() - except TypeError: # fails only when cls is type + except TypeError: # fails only when cls is type subs = cls.__subclasses__(cls) for sub in subs: if sub not in _seen: diff --git a/micropsi_server/usermanagement.py b/micropsi_server/usermanagement.py index 116483d0..a83bd286 100644 --- a/micropsi_server/usermanagement.py +++ b/micropsi_server/usermanagement.py @@ -89,7 +89,7 @@ def __init__(self, userfile_path=None): # set up persistence if userfile_path is None: userfile_path = cfg['paths']['usermanager_path'] - micropsi_core.tools.mkdir(os.path.dirname(userfile_path)) + os.makedirs(os.path.dirname(userfile_path), exist_ok=True) self.user_file_name = userfile_path # todo: make this work without a file system try: From 83f453c118175622633b348ee4f77241a1e912fd Mon Sep 17 00:00:00 2001 From: Ronnie Vuine Date: Wed, 2 Mar 2016 19:40:20 +0100 Subject: [PATCH 128/306] Improvement for partition growing test --- micropsi_core/tests/test_nodenet_partitions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/micropsi_core/tests/test_nodenet_partitions.py b/micropsi_core/tests/test_nodenet_partitions.py index 927d445f..412ab938 100644 --- a/micropsi_core/tests/test_nodenet_partitions.py +++ b/micropsi_core/tests/test_nodenet_partitions.py @@ -121,6 +121,11 @@ def test_grow_partitions(test_nodenet): assert len(partition.allocated_nodespaces) == 4 + # step, save, and load the net to make sure all data structures have been grown properly + micropsi.step_nodenet(test_nodenet) + micropsi.save_nodenet(test_nodenet) + micropsi.revert_nodenet(test_nodenet) + micropsi.step_nodenet(test_nodenet) @pytest.mark.engine("theano_engine") def test_announce_nodes(test_nodenet): From 6a81137e1cdb2296b71ead75bfb80b7209f3952f Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 2 Mar 2016 22:39:27 +0100 Subject: [PATCH 129/306] make autoalignment take a set of entity_uids to align --- micropsi_core/nodenet/netapi.py | 6 ++++++ micropsi_core/nodenet/node_alignment.py | 28 ++++++++++++++++++------- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/micropsi_core/nodenet/netapi.py b/micropsi_core/nodenet/netapi.py index 57a2bda8..e8a6bcd6 100644 --- a/micropsi_core/nodenet/netapi.py +++ b/micropsi_core/nodenet/netapi.py @@ -378,6 +378,12 @@ def autoalign_nodespace(self, nodespace): if nodespace in self.__nodenet.get_nodespace_uids(): align(self.__nodenet, nodespace) + def autoalign_entities(self, nodespace, entity_uids): + """ Calls the autoalignment on the given entities in the given nodespace """ + from micropsi_core.nodenet.node_alignment import align + if nodespace in self.__nodenet.get_nodespace_uids(): + align(self.__nodenet, nodespace, entity_uids) + def get_modulator(self, modulator): """ Returns the numeric value of the given global modulator diff --git a/micropsi_core/nodenet/node_alignment.py b/micropsi_core/nodenet/node_alignment.py index f02f96fe..073917c5 100644 --- a/micropsi_core/nodenet/node_alignment.py +++ b/micropsi_core/nodenet/node_alignment.py @@ -18,11 +18,12 @@ PREFERRED_WIDTH = 8.0 -def align(nodenet, nodespace): +def align(nodenet, nodespace, entity_uids=False): """aligns the entities in the given nodenet. Arguments: nodenet: current node net nodespace: the nodespace in which the entities are to be aligned + entity_uids: optional list of entity uids that should be aligned. If set, other entities remain untouched Returns: True on success, False otherwise """ @@ -32,18 +33,30 @@ def align(nodenet, nodespace): key=lambda i: nodenet.get_nodespace(i).index) unaligned_nodes = sorted(nodenet.get_nodespace(nodespace).get_known_ids('nodes'), key=lambda i: nodenet.get_node(i).index) - sensors = [s for s in unaligned_nodes if nodenet.get_node(s).type == "Sensor"] - actors = [a for a in unaligned_nodes if nodenet.get_node(a).type == "Actor"] - activators = [a for a in unaligned_nodes if nodenet.get_node(a).type == "Activator"] - unaligned_nodes = [n for n in unaligned_nodes if not nodenet.get_node(n).type in ("Sensor", "Actor", "Activator")] + + if entity_uids: + unaligned_nodespaces = [id for id in unaligned_nodespaces if id in entity_uids] + unaligned_nodes = [id for id in unaligned_nodes if id in entity_uids] + sensors = [] + actors = [] + activators = [] + ymin = min(nodenet.get_node(n).position[1] for n in unaligned_nodes) + xmin = min(nodenet.get_node(n).position[0] for n in unaligned_nodes) + start_position = (xmin, ymin, 0) + + else: + sensors = [s for s in unaligned_nodes if nodenet.get_node(s).type == "Sensor"] + actors = [a for a in unaligned_nodes if nodenet.get_node(a).type == "Actor"] + activators = [a for a in unaligned_nodes if nodenet.get_node(a).type == "Activator"] + unaligned_nodes = [n for n in unaligned_nodes if not nodenet.get_node(n).type in ("Sensor", "Actor", "Activator")] + + start_position = (BORDER + GRID / 2, BORDER + (0.5 + math.ceil(len(unaligned_nodespaces) / PREFERRED_WIDTH)) * GRID, 0) # position nodespaces for i, id in enumerate(unaligned_nodespaces): nodenet.get_nodespace(id).position = calculate_grid_position(i) - start_position = (BORDER + GRID / 2, BORDER + (0.5 + math.ceil(len(unaligned_nodespaces) / PREFERRED_WIDTH)) * GRID, 0) - # simplify linkage group = unify_links(nodenet, unaligned_nodes) # connect all nodes that have por- and ret-links @@ -62,6 +75,7 @@ def align(nodenet, nodespace): return True + INVERSE_DIRECTIONS = {"s": "n", "w": "e", "nw": "se", "ne": "sw", "n": "s", "e": "w", "se": "nw", "sw": "ne", "o": "O", "O": "o", "b": "a", "a": "b"} From 3b0e8f0c0035aa896fdf459125ee37f7928fa2a2 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 2 Mar 2016 22:44:39 +0100 Subject: [PATCH 130/306] create operations folder, scan operations folder in runtime. --- micropsi_core/nodenet/operations/__init__.py | 11 +++ micropsi_core/runtime.py | 91 +++++++++++++++++--- 2 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 micropsi_core/nodenet/operations/__init__.py diff --git a/micropsi_core/nodenet/operations/__init__.py b/micropsi_core/nodenet/operations/__init__.py new file mode 100644 index 00000000..5233ab5c --- /dev/null +++ b/micropsi_core/nodenet/operations/__init__.py @@ -0,0 +1,11 @@ + + +def selectioninfo(nodetypes=[], mincount=0, maxcount=-1): + def _decorator(func): + func.selectioninfo = { + 'nodetypes': nodetypes if type(nodetypes) == list else [nodetypes], + 'mincount': mincount, + 'maxcount': maxcount + } + return func + return _decorator diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 6ce599e1..b882a40a 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -52,6 +52,7 @@ nodenets = {} native_modules = {} custom_recipes = {} +custom_operations = {} def add_signal_handler(handler): @@ -1148,6 +1149,21 @@ def get_available_recipes(): return recipes +def get_available_operations(): + """ Returns a dict of available user-operations """ + operations = {} + for name, data in custom_operations.items(): + if not name.startswith('_'): + operations[name] = { + 'name': name, + 'parameters': data['parameters'], + 'docstring': data['docstring'], + 'category': data['category'], + 'selection': data['selectioninfo'] + } + return operations + + def run_recipe(nodenet_uid, name, parameters): """ Calls the given recipe with the provided parameters, and returns the output, if any """ netapi = nodenets[nodenet_uid].netapi @@ -1176,6 +1192,34 @@ def run_recipe(nodenet_uid, name, parameters): return False, "Script not found" +def run_operation(nodenet_uid, name, parameters, selection_uids): + """ Calls the given operation on the selection""" + netapi = nodenets[nodenet_uid].netapi + params = {} + for key in parameters: + if parameters[key] != '': + params[key] = parameters[key] + if name in custom_operations: + func = custom_operations[name]['function'] + if cfg['micropsi2'].get('profile_runner'): + profiler = cProfile.Profile() + profiler.enable() + result = {'reload': True} + ret = func(netapi, selection_uids, **params) + if ret: + result.update(ret) + if cfg['micropsi2'].get('profile_runner'): + profiler.disable() + s = io.StringIO() + sortby = 'cumtime' + ps = pstats.Stats(profiler, stream=s).sort_stats(sortby) + ps.print_stats('nodenet') + logging.getLogger("agent.%s" % nodenet_uid).debug(s.getvalue()) + return True, result + else: + return False, "Operation not found" + + def get_agent_dashboard(nodenet_uid): from .emoexpression import calc_emoexpression_parameters net = nodenets[nodenet_uid] @@ -1285,9 +1329,11 @@ def load_user_files(path, reload_nodefunctions=False, errors=[]): elif f == 'nodetypes.json': err = parse_native_module_file(abspath) elif f == 'recipes.py': - err = parse_recipe_file(abspath, reload_nodefunctions) + err = parse_recipe_or_operations_file(abspath, reload_nodefunctions) elif f == 'nodefunctions.py' and reload_nodefunctions: err = reload_nodefunctions_file(abspath) + elif f == 'operations.py': + err = parse_recipe_or_operations_file(abspath, reload_nodefunctions) if err: errors.append(err) return errors @@ -1309,22 +1355,24 @@ def parse_native_module_file(path): native_modules[key] = modules[key] -def parse_recipe_file(path, reload=False): +def parse_recipe_or_operations_file(path, reload=False, category_overwrite=False): global custom_recipes import importlib import inspect - category = os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) + category = category_overwrite or os.path.relpath(os.path.dirname(path), start=RESOURCE_PATH) relpath = os.path.relpath(path, start=RESOURCE_PATH) - pyname = relpath.replace(os.path.sep, '.')[:-3] + name = os.path.basename(path)[:-3] + + mode = 'recipes' if os.path.basename(path).startswith('recipes') else 'operations' try: - loader = importlib.machinery.SourceFileLoader('recipes', path) + loader = importlib.machinery.SourceFileLoader(name, path) recipes = loader.load_module() # recipes = __import__(pyname, fromlist=['recipes']) # importlib.reload(sys.modules[pyname]) except SyntaxError as e: - return "%s in recipe file %s, line %d" % (e.__class__.__name__, relpath, e.lineno) + return "%s in %s file %s, line %d" % (e.__class__.__name__, mode, relpath, e.lineno) for name, module in inspect.getmembers(recipes, inspect.ismodule): if module.__file__.startswith(RESOURCE_PATH): @@ -1333,12 +1381,15 @@ def parse_recipe_file(path, reload=False): all_functions = inspect.getmembers(recipes, inspect.isfunction) for name, func in all_functions: filename = os.path.realpath(func.__code__.co_filename) - if filename != os.path.realpath(path) and os.path.basename(filename) == 'recipes.py': - # import from another recipes file. ignore, to avoid + if filename != os.path.realpath(path) and os.path.basename(filename) == os.path.basename(path): + # import from another file of the same mode. ignore, to avoid # false duplicate-function-name alerts continue argspec = inspect.getargspec(func) - arguments = argspec.args[1:] + if mode == 'recipes': + arguments = argspec.args[1:] + elif mode == 'operations': + arguments = argspec.args[2:] defaults = argspec.defaults or [] params = [] diff = len(arguments) - len(defaults) @@ -1351,9 +1402,11 @@ def parse_recipe_file(path, reload=False): 'name': arg, 'default': default }) - if name in custom_recipes and id(func) != id(custom_recipes[name]['function']): + if mode == 'recipes' and name in custom_recipes and id(func) != id(custom_recipes[name]['function']): logging.getLogger("system").warning("Recipe function names must be unique. %s is not." % name) - custom_recipes[name] = { + elif mode == 'operations' and name in custom_operations and id(func) != id(custom_operations[name]['function']): + logging.getLogger("system").warning("Operations function names must be unique. %s is not." % name) + data = { 'name': name, 'parameters': params, 'function': func, @@ -1362,6 +1415,13 @@ def parse_recipe_file(path, reload=False): 'path': path } + if mode == 'recipes': + custom_recipes[name] = data + elif mode == 'operations': + if hasattr(func, 'selectioninfo'): + data['selectioninfo'] = func.selectioninfo + custom_operations[name] = data + def reload_nodefunctions_file(path): import importlib @@ -1381,12 +1441,19 @@ def reload_nodefunctions_file(path): def reload_native_modules(): # stop nodenets, save state - global native_modules, custom_recipes + global native_modules, custom_recipes, custom_operations native_modules = {} custom_recipes = {} + custom_operations = {} runners = {} + # load builtins: from micropsi_core.nodenet.native_modules import nodetypes native_modules.update(nodetypes) + operationspath = os.path.abspath('micropsi_core/nodenet/operations/') + for file in os.listdir(operationspath): + import micropsi_core.nodenet.operations + if file != '__init__.py' and not file.startswith('.') and os.path.isfile(os.path.join(operationspath, file)): + parse_recipe_or_operations_file(os.path.join(operationspath, file), category_overwrite=file[:-3]) for uid in nodenets: if nodenets[uid].is_active: runners[uid] = True From ade39f50001a99254305d3a751ec9457149b2e0a Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 2 Mar 2016 22:45:22 +0100 Subject: [PATCH 131/306] json api for operations --- micropsi_server/micropsi_app.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index 87701f89..65ee012d 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -1218,6 +1218,16 @@ def get_available_recipes(): return True, runtime.get_available_recipes() +@rpc("run_operation") +def run_operation(nodenet_uid, name, parameters, selection_uids): + return runtime.run_operation(nodenet_uid, name, parameters, selection_uids) + + +@rpc('get_available_operations') +def get_available_operations(): + return True, runtime.get_available_operations() + + @rpc('get_agent_dashboard') def get_agent_dashboard(nodenet_uid): return True, runtime.get_agent_dashboard(nodenet_uid) From e0aeead787ed542500aa856bd343b5f63e1f86e3 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 2 Mar 2016 22:47:40 +0100 Subject: [PATCH 132/306] add a first operation and some tests. --- micropsi_core/nodenet/operations/layout.py | 19 ++++++++++++++++++ micropsi_core/tests/test_operations.py | 23 ++++++++++++++++++++++ micropsi_server/tests/test_json_api.py | 16 +++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 micropsi_core/nodenet/operations/layout.py create mode 100644 micropsi_core/tests/test_operations.py diff --git a/micropsi_core/nodenet/operations/layout.py b/micropsi_core/nodenet/operations/layout.py new file mode 100644 index 00000000..b6ddca16 --- /dev/null +++ b/micropsi_core/nodenet/operations/layout.py @@ -0,0 +1,19 @@ + +from micropsi_core.nodenet.operations import selectioninfo + + +@selectioninfo(mincount=2) +def autoalign(netapi, selection): + nodespace = None + if len(selection): + try: + nodespace = netapi.get_node(selection[0]).parent_nodespace + except: + pass + try: + nodespace = netapi.get_nodespace(selection[0]).parent_nodespace + except: + pass + if nodespace is None: + return {'error': 'unknown entity in selection'} + netapi.autoalign_entities(nodespace, selection) diff --git a/micropsi_core/tests/test_operations.py b/micropsi_core/tests/test_operations.py new file mode 100644 index 00000000..edb5a621 --- /dev/null +++ b/micropsi_core/tests/test_operations.py @@ -0,0 +1,23 @@ + +from micropsi_core import runtime + + +def test_autoalign_operation(test_nodenet): + ops = runtime.get_available_operations() + assert ops['autoalign']['selection']['nodetypes'] == [] + assert ops['autoalign']['selection']['mincount'] == 2 + assert ops['autoalign']['selection']['maxcount'] == -1 + assert ops['autoalign']['category'] == 'layout' + assert ops['autoalign']['parameters'] == [] + + api = runtime.nodenets[test_nodenet].netapi + p1 = api.create_node("Pipe", None, "p1") + p2 = api.create_node("Pipe", None, "p2") + p3 = api.create_node("Pipe", None, "p3") + api.link_with_reciprocal(p1, p2, 'subsur') + api.link_with_reciprocal(p1, p3, 'subsur') + api.link_with_reciprocal(p2, p3, 'porret') + runtime.run_operation(test_nodenet, "autoalign", {}, [p1.uid, p2.uid, p3.uid]) + assert p1.position[0] == p2.position[0] + assert p1.position[1] < p2.position[1] + assert p2.position[1] == p3.position[1] diff --git a/micropsi_server/tests/test_json_api.py b/micropsi_server/tests/test_json_api.py index 879238d4..2acf4262 100644 --- a/micropsi_server/tests/test_json_api.py +++ b/micropsi_server/tests/test_json_api.py @@ -1455,3 +1455,19 @@ def test_get_state_diff(app, test_nodenet, node): }) data = response.json_body['data']['nodenet_diff'] assert [node2] == list(data['changes']['nodes_dirty'].keys()) + + +def test_get_operations(app, test_nodenet): + response = app.get_json('/rpc/get_available_operations()') + data = response.json_body['data'] + assert data['autoalign']['selection']['nodetypes'] == [] + + +def test_run_operation(app, test_nodenet, node): + response = app.post_json('/rpc/run_operation', { + 'nodenet_uid': test_nodenet, + 'name': 'autoalign', + 'parameters': {}, + 'selection_uids': [node] + }) + assert response.json_body['status'] == 'success' From 4a558682818b327d5786ea8eb7bb17d13f6ccb43 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 3 Mar 2016 12:00:25 +0100 Subject: [PATCH 133/306] also test user provided operations --- micropsi_core/tests/test_operations.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/micropsi_core/tests/test_operations.py b/micropsi_core/tests/test_operations.py index edb5a621..07e3376f 100644 --- a/micropsi_core/tests/test_operations.py +++ b/micropsi_core/tests/test_operations.py @@ -2,6 +2,28 @@ from micropsi_core import runtime +def test_user_operation(test_nodenet, resourcepath): + import os + os.makedirs(os.path.join(resourcepath, 'foobar')) + with open(os.path.join(resourcepath, 'foobar', 'operations.py'), 'w+') as fp: + fp.write(""" +def delete_nodes(netapi, selection): + for uid in selection: + netapi.delete_node(netapi.get_node(uid)) + +delete_nodes.selectioninfo = { + 'nodetypes': [], + 'mincount': 1, + 'maxcount': -1 +}""") + runtime.reload_native_modules() + ops = runtime.get_available_operations() + assert ops['delete_nodes']['category'] == 'foobar' + res, uid = runtime.add_node(test_nodenet, "Register", [10, 10], None) + runtime.run_operation(test_nodenet, "delete_nodes", {}, [uid]) + assert uid not in runtime.nodenets[test_nodenet].get_node_uids() + + def test_autoalign_operation(test_nodenet): ops = runtime.get_available_operations() assert ops['autoalign']['selection']['nodetypes'] == [] From dac217e43251f665f7c10eac21fccf187c8676c5 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 3 Mar 2016 12:00:57 +0100 Subject: [PATCH 134/306] first shot at operations frontend --- micropsi_server/static/js/nodenet.js | 128 ++++++++++++++++++--------- 1 file changed, 87 insertions(+), 41 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index ea714c73..dab63a26 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -151,6 +151,10 @@ registerResizeHandler(); globalDataSources = []; globalDataTargets = []; +available_operations = {}; +operation_categories = {}; +sorted_operations = []; + $(document).on('load_nodenet', function(event, uid){ ns = 'Root'; if(uid == currentNodenet){ @@ -233,6 +237,31 @@ function setNodenetValues(data){ } } +function buildCategoryTree(item, path, idx){ + if (idx < path.length){ + name = path[idx]; + if (!item[name]){ + item[name] = {}; + } + buildCategoryTree(item[name], path, idx + 1); + } +} + + +api.call("get_available_operations", {}, function(data){ + available_operations = data + categories = []; + for(var key in available_operations){ + categories.push(available_operations[key].category.split('/')); + } + operation_categories = {} + for(var i =0; i < categories.length; i++){ + buildCategoryTree(operation_categories, categories[i], 0); + } + sorted_operations = Object.keys(available_operations).sort(); +}); + + function setCurrentNodenet(uid, nodespace, changed){ if(!nodespace){ nodespace = "Root"; @@ -288,16 +317,6 @@ function setCurrentNodenet(uid, nodespace, changed){ return 0; }); - function buildTreeRecursive(item, path, idx){ - if (idx < path.length){ - name = path[idx]; - if (!item[name]){ - item[name] = {}; - } - buildTreeRecursive(item[name], path, idx + 1); - } - } - categories = []; for(var key in native_modules){ nodetypes[key] = native_modules[key]; @@ -305,7 +324,7 @@ function setCurrentNodenet(uid, nodespace, changed){ } native_module_categories = {} for(var i =0; i < categories.length; i++){ - buildTreeRecursive(native_module_categories, categories[i], 0); + buildCategoryTree(native_module_categories, categories[i], 0); } available_gatetypes = []; @@ -2576,7 +2595,7 @@ function initializeDialogs(){ var clickPosition = null; -function buildNativeModuleDropdown(cat, html, current_category){ +function buildRecursiveDropdown(cat, html, current_category, generate_items){ if(!current_category){ current_category=''; } @@ -2589,17 +2608,14 @@ function buildNativeModuleDropdown(cat, html, current_category){ var newcategory = current_category || ''; if(current_category == '') newcategory += catentries[i] else newcategory += '/'+catentries[i]; - html += '
      • '+catentries[i]+''; + html += '
      • '+catentries[i]+''; html += '
          ' - html += buildNativeModuleDropdown(cat[catentries[i]], '', newcategory); + html += buildRecursiveDropdown(cat[catentries[i]], '', newcategory, generate_items); html += '
        • '; } - for(var idx in sorted_native_modules){ - key = sorted_native_modules[idx]; - if(native_modules[key].category == current_category){ - html += '
      • '+ native_modules[key].name +'
        • '; - } - } + + html += generate_items(current_category); + return html } @@ -2622,9 +2638,18 @@ function openContextMenu(menu_id, event) { html += '
      • Create ' + sorted_nodetypes[idx] +'
        • '; } if(Object.keys(native_modules).length){ - html += '
      • Create Native Module'; + html += '
      • Create Native Module'; html += '
          '; - html += buildNativeModuleDropdown(native_module_categories, '', '') + html += buildRecursiveDropdown(native_module_categories, '', '', function(current_category){ + items = ''; + for(var idx in sorted_native_modules){ + key = sorted_native_modules[idx]; + if(native_modules[key].category == current_category){ + items += '
        • '+ native_modules[key].name +'
          • '; + } + } + return items; + }); html += '
        • '; } html += '
      • Autoalign Nodes
        • '; @@ -2636,22 +2661,19 @@ function openContextMenu(menu_id, event) { list.html(html); } $(menu_id+" .dropdown-toggle").dropdown("toggle"); + $(menu_id+" li.noop").on('click', function(event){event.stopPropagation();}) } function openMultipleNodesContextMenu(event){ - var typecheck = null; - var sametype = true; var node = null; var compact = false; + var nodetypes = []; for(var uid in selection){ if(isCompact(nodes[uid])) { compact = true; } - if(typecheck == null || typecheck == nodes[uid].type){ - typecheck = nodes[uid].type; - node = nodes[uid]; - } else { - sametype = false; + if(nodetypes.indexOf(nodes[uid].type) > -1){ + nodetypes.push(nodes[uid].type); } } var menu = $('#multi_node_menu .nodenet_menu'); @@ -2662,7 +2684,20 @@ function openMultipleNodesContextMenu(event){ html += '
      • Copy nodes
        • '+ '
      • Paste nodes
        • '+ '
      • Delete nodes
        • '; - if(sametype){ + + html += '
      • Operations
          '; + html += buildRecursiveDropdown(operation_categories, '', '', function(current_category){ + items = ''; + for(var idx in sorted_operations){ + key = sorted_operations[idx]; + if(available_operations[key].category == current_category){ + items += '
        • '+ key +'
          • '; + } + } + return items; + }); + html += '
        • '; + if(nodetypes.length == 1){ html += '
      • ' + getNodeLinkageContextMenuHTML(node); } html += '
      • Generate netapi fragment
        • '; @@ -2835,18 +2870,29 @@ function handleContextMenu(event) { } break; default: - var linktype = $(event.target).attr('data-link-type'); - if (linktype) { - var forwardlinktype = linktype; - if(forwardlinktype.indexOf('/')){ - forwardlinktype = forwardlinktype.split('/')[0]; - } - for(var uid in selection){ - clickIndex = nodes[uid].gateIndexes.indexOf(forwardlinktype); - createLinkHandler(uid, clickIndex, linktype); - } + if($el.attr('data-run-operation')){ + api.call('run_operation', { + 'nodenet_uid': currentNodenet, + 'name': $el.attr('data-run-operation'), + 'parameters': {}, + 'selection_uids': Object.keys(selection)}, function(data){ + refreshNodespace() + } + ); } else { - openLinkCreationDialog(path.name) + var linktype = $(event.target).attr('data-link-type'); + if (linktype) { + var forwardlinktype = linktype; + if(forwardlinktype.indexOf('/')){ + forwardlinktype = forwardlinktype.split('/')[0]; + } + for(var uid in selection){ + clickIndex = nodes[uid].gateIndexes.indexOf(forwardlinktype); + createLinkHandler(uid, clickIndex, linktype); + } + } else { + openLinkCreationDialog(path.name) + } } } break; From 11fc1a6f6d2647c9bd74047c4d078216f71d2bc0 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 3 Mar 2016 12:01:21 +0100 Subject: [PATCH 135/306] recipes and operations are fetched explcitly --- micropsi_server/micropsi_app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index 65ee012d..73e2bba4 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -723,7 +723,6 @@ def load_nodenet(nodenet_uid, nodespace='Root', include_links=True): if result: data = runtime.get_nodenet_data(nodenet_uid, nodespace, -1, include_links) data['nodetypes'] = runtime.get_available_node_types(nodenet_uid) - data['recipes'] = runtime.get_available_recipes() return True, data else: return False, uid From 5e6613817518e888a87481d33ff6d378379a5dc5 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 3 Mar 2016 14:25:11 +0100 Subject: [PATCH 136/306] filter operations by nodetype --- micropsi_server/static/js/nodenet.js | 61 +++++++++++++++++----------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index dab63a26..d0439cc5 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -152,8 +152,6 @@ globalDataSources = []; globalDataTargets = []; available_operations = {}; -operation_categories = {}; -sorted_operations = []; $(document).on('load_nodenet', function(event, uid){ ns = 'Root'; @@ -250,15 +248,6 @@ function buildCategoryTree(item, path, idx){ api.call("get_available_operations", {}, function(data){ available_operations = data - categories = []; - for(var key in available_operations){ - categories.push(available_operations[key].category.split('/')); - } - operation_categories = {} - for(var i =0; i < categories.length; i++){ - buildCategoryTree(operation_categories, categories[i], 0); - } - sorted_operations = Object.keys(available_operations).sort(); }); @@ -2661,7 +2650,7 @@ function openContextMenu(menu_id, event) { list.html(html); } $(menu_id+" .dropdown-toggle").dropdown("toggle"); - $(menu_id+" li.noop").on('click', function(event){event.stopPropagation();}) + $(menu_id+" li.noop > a").on('click', function(event){event.stopPropagation();}) } function openMultipleNodesContextMenu(event){ @@ -2669,10 +2658,11 @@ function openMultipleNodesContextMenu(event){ var compact = false; var nodetypes = []; for(var uid in selection){ + if(!node) node = nodes[uid]; if(isCompact(nodes[uid])) { compact = true; } - if(nodetypes.indexOf(nodes[uid].type) > -1){ + if(nodetypes.indexOf(nodes[uid].type) == -1){ nodetypes.push(nodes[uid].type); } } @@ -2685,18 +2675,41 @@ function openMultipleNodesContextMenu(event){ '
      • Paste nodes
        • '+ '
      • Delete nodes
        • '; - html += '
      • Operations
          '; - html += buildRecursiveDropdown(operation_categories, '', '', function(current_category){ - items = ''; - for(var idx in sorted_operations){ - key = sorted_operations[idx]; - if(available_operations[key].category == current_category){ - items += '
        • '+ key +'
          • '; - } + operation_categories = {}; + sorted_operations = []; + + applicable_operations = {}; + for(var key in available_operations){ + var supported_types = available_operations[key].selection.nodetypes; + if(supported_types.length == 0 || $(nodetypes).not(supported_types).get().length == 0){ + applicable_operations[key] = available_operations[key] } - return items; - }); - html += '
        • '; + } + + categories = []; + for(var key in applicable_operations){ + categories.push(applicable_operations[key].category.split('/')); + } + operation_categories = {} + for(var i =0; i < categories.length; i++){ + buildCategoryTree(operation_categories, categories[i], 0); + } + sorted_operations = Object.keys(applicable_operations).sort(); + + if(len(sorted_operations)){ + html += '
      • Operations
          '; + html += buildRecursiveDropdown(operation_categories, '', '', function(current_category){ + items = ''; + for(var idx in sorted_operations){ + key = sorted_operations[idx]; + if(applicable_operations[key].category == current_category){ + items += '
        • '+ key +'
          • '; + } + } + return items; + }); + html += '
        • '; + } if(nodetypes.length == 1){ html += '
      • ' + getNodeLinkageContextMenuHTML(node); } From 9387d342b0774bbd33ec721f26e81bb5b8a01c0b Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 3 Mar 2016 14:35:40 +0100 Subject: [PATCH 137/306] filter operations by count criteria disable operations entry if none applicable --- micropsi_server/static/js/nodenet.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index d0439cc5..47e0bbc7 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -2657,6 +2657,7 @@ function openMultipleNodesContextMenu(event){ var node = null; var compact = false; var nodetypes = []; + var count = 0 for(var uid in selection){ if(!node) node = nodes[uid]; if(isCompact(nodes[uid])) { @@ -2665,6 +2666,7 @@ function openMultipleNodesContextMenu(event){ if(nodetypes.indexOf(nodes[uid].type) == -1){ nodetypes.push(nodes[uid].type); } + count += 1; } var menu = $('#multi_node_menu .nodenet_menu'); var html = ''; @@ -2680,9 +2682,11 @@ function openMultipleNodesContextMenu(event){ applicable_operations = {}; for(var key in available_operations){ - var supported_types = available_operations[key].selection.nodetypes; - if(supported_types.length == 0 || $(nodetypes).not(supported_types).get().length == 0){ - applicable_operations[key] = available_operations[key] + var conditions = available_operations[key].selection; + if((conditions.nodetypes.length == 0 || $(nodetypes).not(conditions.nodetypes).get().length == 0) && + (count >= conditions.mincount) && + (conditions.maxcount < 0 || count <= conditions.maxcount)){ + applicable_operations[key] = available_operations[key] } } @@ -2696,7 +2700,7 @@ function openMultipleNodesContextMenu(event){ } sorted_operations = Object.keys(applicable_operations).sort(); - if(len(sorted_operations)){ + if(sorted_operations.length){ html += '
      • Operations
          '; html += buildRecursiveDropdown(operation_categories, '', '', function(current_category){ items = ''; @@ -2709,6 +2713,8 @@ function openMultipleNodesContextMenu(event){ return items; }); html += '
        • '; + } else { + html += '
      • Operations
        • '; } if(nodetypes.length == 1){ html += '
      • ' + getNodeLinkageContextMenuHTML(node); From 9ab2e6a3a9187708d58473ff50cade668bd716c9 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 3 Mar 2016 14:36:20 +0100 Subject: [PATCH 138/306] up the coverage a bit :) --- micropsi_core/tests/test_operations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/micropsi_core/tests/test_operations.py b/micropsi_core/tests/test_operations.py index 07e3376f..71c943f9 100644 --- a/micropsi_core/tests/test_operations.py +++ b/micropsi_core/tests/test_operations.py @@ -33,13 +33,14 @@ def test_autoalign_operation(test_nodenet): assert ops['autoalign']['parameters'] == [] api = runtime.nodenets[test_nodenet].netapi + ns1 = api.create_nodespace(None, "foo") p1 = api.create_node("Pipe", None, "p1") p2 = api.create_node("Pipe", None, "p2") p3 = api.create_node("Pipe", None, "p3") api.link_with_reciprocal(p1, p2, 'subsur') api.link_with_reciprocal(p1, p3, 'subsur') api.link_with_reciprocal(p2, p3, 'porret') - runtime.run_operation(test_nodenet, "autoalign", {}, [p1.uid, p2.uid, p3.uid]) + runtime.run_operation(test_nodenet, "autoalign", {}, [p1.uid, p2.uid, p3.uid, ns1]) assert p1.position[0] == p2.position[0] assert p1.position[1] < p2.position[1] assert p2.position[1] == p3.position[1] From e08125f15af18126038f9a2c144ed6d65402559e Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 3 Mar 2016 15:37:43 +0100 Subject: [PATCH 139/306] add a test for generate_netapi_fragment and fix the bug it found --- micropsi_core/runtime.py | 6 +++--- micropsi_core/tests/test_runtime.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 6ce599e1..508f65c2 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -827,9 +827,9 @@ def generate_netapi_fragment(nodenet_uid, node_uids): else: lines.append("%s = netapi.create_nodespace(None)" % (varname)) idmap[nodespace.uid] = varname - xpos.append(node.position[0]) - ypos.append(node.position[1]) - zpos.append(node.position[2]) + xpos.append(nodespace.position[0]) + ypos.append(nodespace.position[1]) + zpos.append(nodespace.position[2]) # nodes and gates for i, node in enumerate(nodes): diff --git a/micropsi_core/tests/test_runtime.py b/micropsi_core/tests/test_runtime.py index 50583ea7..b7f8dc4d 100644 --- a/micropsi_core/tests/test_runtime.py +++ b/micropsi_core/tests/test_runtime.py @@ -166,3 +166,32 @@ def test_export_json_does_not_send_duplicate_links(fixed_nodenet): import json result = json.loads(micropsi.export_nodenet(fixed_nodenet)) assert len(result['links']) == 4 + + +def test_generate_netapi_fragment(test_nodenet, resourcepath): + import os + netapi = micropsi.nodenets[test_nodenet].netapi + # create a bunch of nodes and link them + linktypes = ['subsur', 'porret', 'catexp'] + nodes = [] + for t in linktypes: + p1 = netapi.create_node('Pipe', None, t) + p2 = netapi.create_node('Pipe', None, t + '2') + nodes.extend([p1, p2]) + netapi.link_with_reciprocal(p1, p2, t) + reg = netapi.create_node('Register', None, 'reg') + netapi.link(reg, 'gen', nodes[0], 'gen') + ns = netapi.create_nodespace(None, 'ns1') + nodes.extend([reg, ns]) + # remember their names + names = [n.name for n in nodes] + fragment = micropsi.generate_netapi_fragment(test_nodenet, [n.uid for n in nodes]) + micropsi.nodenets[test_nodenet].clear() + code = "def foo(netapi):\n " + "\n ".join(fragment.split('\n')) + # save the fragment as recipe & run + with open(os.path.join(resourcepath, 'recipes.py'), 'w+') as fp: + fp.write(code) + micropsi.reload_native_modules() + micropsi.run_recipe(test_nodenet, 'foo', {}) + # assert that all the nodes are there again + assert set(names) == set([n.name for n in netapi.get_nodes()] + ['ns1']) From 4157c2980ebd08478913f5f944c2b355aa054707 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 4 Mar 2016 13:25:19 +0100 Subject: [PATCH 140/306] operations don't do a full reload, they refresh the nodespace --- micropsi_core/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index b882a40a..5c532bed 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1204,7 +1204,7 @@ def run_operation(nodenet_uid, name, parameters, selection_uids): if cfg['micropsi2'].get('profile_runner'): profiler = cProfile.Profile() profiler.enable() - result = {'reload': True} + result = {} ret = func(netapi, selection_uids, **params) if ret: result.update(ret) From f7a30500dfbe2e6ad32b33353496c182c942c320 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 4 Mar 2016 13:25:56 +0100 Subject: [PATCH 141/306] track errors when parsing core operations --- micropsi_core/runtime.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 5c532bed..3f0d26fd 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1446,6 +1446,7 @@ def reload_native_modules(): custom_recipes = {} custom_operations = {} runners = {} + errors = [] # load builtins: from micropsi_core.nodenet.native_modules import nodetypes native_modules.update(nodetypes) @@ -1453,12 +1454,15 @@ def reload_native_modules(): for file in os.listdir(operationspath): import micropsi_core.nodenet.operations if file != '__init__.py' and not file.startswith('.') and os.path.isfile(os.path.join(operationspath, file)): - parse_recipe_or_operations_file(os.path.join(operationspath, file), category_overwrite=file[:-3]) + err = parse_recipe_or_operations_file(os.path.join(operationspath, file), category_overwrite=file[:-3]) + if err: + errors.append(err) + for uid in nodenets: if nodenets[uid].is_active: runners[uid] = True nodenets[uid].is_active = False - errors = load_user_files(RESOURCE_PATH, reload_nodefunctions=True, errors=[]) + errors.extend(load_user_files(RESOURCE_PATH, reload_nodefunctions=True, errors=[])) for nodenet_uid in nodenets: nodenets[nodenet_uid].reload_native_modules(filter_native_modules(nodenets[nodenet_uid].engine)) # restart previously active nodenets From c2f3ce34f69d134a8b2e2043c2f687ce37a83945 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 4 Mar 2016 13:39:18 +0100 Subject: [PATCH 142/306] display results in frontend --- micropsi_server/static/js/nodenet.js | 45 +++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index 47e0bbc7..c33792c1 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -2890,14 +2890,7 @@ function handleContextMenu(event) { break; default: if($el.attr('data-run-operation')){ - api.call('run_operation', { - 'nodenet_uid': currentNodenet, - 'name': $el.attr('data-run-operation'), - 'parameters': {}, - 'selection_uids': Object.keys(selection)}, function(data){ - refreshNodespace() - } - ); + runOperation($el.attr('data-run-operation')); } else { var linktype = $(event.target).attr('data-link-type'); if (linktype) { @@ -2961,6 +2954,42 @@ function handleContextMenu(event) { view.draw(); } +function runOperation(name){ + api.call('run_operation', { + 'nodenet_uid': currentNodenet, + 'name': $el.attr('data-run-operation'), + 'parameters': {}, + 'selection_uids': Object.keys(selection)}, function(data){ + refreshNodespace(); + if(!$.isEmptyObject(data)){ + html = ''; + if(data.content_type && data.content_type.indexOf("image") > -1){ + html += '

        '; + delete data.content_type + delete data.data + } + if(Object.keys(data).length){ + html += '
        '; + for(var key in data){ + html += '
        '+key+':
        '; + if(typeof data[key] == 'string'){ + html += '
        '+data[key]+'
        '; + } else { + html += '
        '+JSON.stringify(data[key])+'
        '; + } + } + html += '
        '; + } + if(html){ + $('#recipe_result .modal-body').html(html); + $('#recipe_result').modal('show'); + $('#recipe_result button').off(); + } + } + } + ); +} + function openLinkCreationDialog(nodeUid){ $("#link_target_node").html(''); $('#link_target_slot').html(''); From 790b3b8c5caf4395f392c6b6a35119d24b33cc90 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 4 Mar 2016 15:08:47 +0100 Subject: [PATCH 143/306] sensor_values & co should be numpy arrays --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index b5d1a160..5ec4f971 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -1279,16 +1279,21 @@ def set_sensors_and_actuator_feedback_to_values(self, sensor_values, actuator_fe """ Sets the sensors for the given data sources or modulators to the given values """ + # convert from python types: + if type(sensor_values).__module__ != 'numpy': + sensor_values = np.asarray(sensor_values) + if type(actuator_feedback_values).__module__ != 'numpy': + actuator_feedback_values = np.asarray(actuator_feedback_values) if self.use_modulators: # include modulators readables = [0 for _ in DoernerianEmotionalModulators.readable_modulators] for idx, key in enumerate(sorted(DoernerianEmotionalModulators.readable_modulators)): readables[idx] = self.get_modulator(key) - sensor_values = sensor_values + readables + sensor_values = np.concatenate((sensor_values, np.asarray(readables))) writeables = [0 for _ in DoernerianEmotionalModulators.writeable_modulators] for idx, key in enumerate(sorted(DoernerianEmotionalModulators.writeable_modulators)): writeables[idx] = 1 - actuator_feedback_values = actuator_feedback_values + writeables + actuator_feedback_values = np.concatenate((actuator_feedback_values, np.asarray(writeables))) for partition in self.partitions.values(): a_array = partition.a.get_value(borrow=True) From 1573256acab7f46745bbc81b08cc229abb473468 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 4 Mar 2016 16:15:12 +0100 Subject: [PATCH 144/306] fix: do not update nodes we didn't render in the first place --- micropsi_server/static/js/nodenet.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index ea714c73..79a90a36 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -507,18 +507,20 @@ function setNodespaceDiffData(data, changed){ } // activations: for(var uid in data.activations){ - activations = data.activations[uid]; - var gen = 0 - for(var i=0; i < nodes[uid].gateIndexes.length; i++){ - var type = nodes[uid].gateIndexes[i]; - nodes[uid].gates[type].sheaves['default'].activation = activations[i]; - if(type == 'gen'){ - gen = activations[i]; + if (uid in nodes){ + activations = data.activations[uid]; + var gen = 0 + for(var i=0; i < nodes[uid].gateIndexes.length; i++){ + var type = nodes[uid].gateIndexes[i]; + nodes[uid].gates[type].sheaves['default'].activation = activations[i]; + if(type == 'gen'){ + gen = activations[i]; + } } + nodes[uid].sheaves['default'].activation = gen; + setActivation(nodes[uid]); + redrawNodeLinks(nodes[uid]); } - nodes[uid].sheaves['default'].activation = gen; - setActivation(nodes[uid]); - redrawNodeLinks(nodes[uid]); } updateModulators(data.modulators); From 8f663f5d460eeaab5d5afc7a0f60486dea7b271b Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 4 Mar 2016 17:06:20 +0100 Subject: [PATCH 145/306] experimental nodenet display: * conventional display: display all nodes, display activation as red/green * feature-display: use activation as alpha, only color green activation --- micropsi_server/static/js/nodenet.js | 35 ++++++++++++++++++++++++---- micropsi_server/view/nodenet.tpl | 7 ++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index 79a90a36..4c405c6e 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -224,6 +224,7 @@ function setNodenetValues(data){ $('#nodenet_name').val(data.name); $('#nodenet_snap').attr('checked', data.snap_to_grid); $('#nodenet_renderlinks').val(nodenet_data.renderlinks); + $('#nodenet_vis').val(nodenet_data.vis); if (!jQuery.isEmptyObject(worldadapters)) { var worldadapter_select = $('#nodenet_worldadapter'); worldadapter_select.val(data.worldadapter); @@ -261,6 +262,7 @@ function setCurrentNodenet(uid, nodespace, changed){ nodenet_data = data; nodenet_data['renderlinks'] = $.cookie('renderlinks') || 'always'; nodenet_data['snap_to_grid'] = $.cookie('snap_to_grid') || viewProperties.snap_to_grid; + nodenet_data['vis'] = $.cookie('activation_display') || 'redgreen'; showDefaultForm(); currentNodeSpace = data['nodespace']; @@ -1239,7 +1241,13 @@ function renderLink(link, force) { linkItem.name = "link"; var linkContainer = new Group(linkItem); linkContainer.name = link.uid; - + if (nodenet_data['vis'] == 'alpha'){ + if(sourceNode){ + linkContainer.opacity = Math.max(0.1, sourceNode.sheaves[currentSheaf].activation) + } else { + linkContainer.opacity = 0.1 + } + } linkLayer.addChild(linkContainer); } @@ -1784,8 +1792,23 @@ function setActivation(node) { } if (node.uid in nodeLayer.children) { var nodeItem = nodeLayer.children[node.uid]; - node.fillColor = nodeItem.children["activation"].children["body"].fillColor = - activationColor(node.sheaves[currentSheaf].activation, viewProperties.nodeColor); + if((nodenet_data['vis'] != 'alpha') || node.sheaves[currentSheaf].activation > 0.5){ + node.fillColor = nodeItem.children["activation"].children["body"].fillColor = + activationColor(node.sheaves[currentSheaf].activation, viewProperties.nodeColor); + } + if(nodenet_data['vis'] == 'alpha'){ + for(var i in nodeItem.children){ + if(nodeItem.children[i].name == 'labelText'){ + nodeItem.children[i].opacity = 0; + if (node.sheaves[currentSheaf].activation > 0.5){ + nodeItem.children[i].opacity = node.sheaves[currentSheaf].activation; + } + } else { + nodeItem.children[i].opacity = Math.max(0.1, node.sheaves[currentSheaf].activation) + } + } + } + if (!isCompact(node) && (node.slotIndexes.length || node.gateIndexes.length)) { var i=0; var type; @@ -3615,13 +3638,17 @@ function handleEditNodenet(event){ $.cookie('renderlinks', nodenet_data.renderlinks || '', {path: '/', expires: 7}) nodenet_data.snap_to_grid = $('#nodenet_snap').attr('checked'); $.cookie('snap_to_grid', nodenet_data.snap_to_grid || '', {path: '/', expires: 7}) + if(nodenet_data.vis != $('#nodenet_vis').val()) reload = true; + nodenet_data.vis = $('#nodenet_vis').val(); + $.cookie('activation_display', nodenet_data.vis || '', {path: '/', expires: 7}) api.call("set_nodenet_properties", params, success=function(data){ dialogs.notification('Nodenet data saved', 'success'); if(reload){ window.location.reload(); } else { - setCurrentNodenet(currentNodenet, currentNodeSpace); + // setCurrentNodenet(currentNodenet, currentNodeSpace); + refreshNodespace(); } } ); diff --git a/micropsi_server/view/nodenet.tpl b/micropsi_server/view/nodenet.tpl index 41373170..7dc4c062 100644 --- a/micropsi_server/view/nodenet.tpl +++ b/micropsi_server/view/nodenet.tpl @@ -83,6 +83,13 @@
        + + + +% include netapi_console + - \ No newline at end of file + + \ No newline at end of file From c526b581856c291c4a4efb9071a497a96d231f8c Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 4 Apr 2016 16:33:22 +0200 Subject: [PATCH 250/306] autocomplete options for defined variables (if they are of type Node, Nodespace, Slot, Gate) --- micropsi_core/runtime.py | 87 ++++++++++++++----- micropsi_core/tests/test_runtime.py | 21 +++++ micropsi_server/micropsi_app.py | 4 +- micropsi_server/static/js/netapi_console.js | 92 +++++++++++++-------- 4 files changed, 149 insertions(+), 55 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 26896ea8..2f20d872 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1327,30 +1327,79 @@ def run_netapi_command(nodenet_uid, command): return shell.push(command) -def get_netapi_signatures(nodenet_uid): +def get_netapi_autocomplete_data(nodenet_uid, name=None): import inspect - netapi = get_nodenet(nodenet_uid).netapi - methods = inspect.getmembers(netapi, inspect.ismethod) - data = {} - for name, func in methods: - argspec = inspect.getargspec(func) - arguments = argspec.args[1:] - defaults = argspec.defaults or [] - params = [] - diff = len(arguments) - len(defaults) - for i, arg in enumerate(arguments): - if i >= diff: - default = defaults[i - diff] + nodenet = get_nodenet(nodenet_uid) + nodetypes = get_available_node_types(nodenet_uid) + + shell = netapi_consoles[nodenet_uid] + res, locs = shell.push("[k for k in locals() if not k.startswith('_')]") + locs = eval(locs) + + def parsemembers(members): + data = {} + for name, thing in members: + if name.startswith('_'): + continue + if inspect.isroutine(thing): + argspec = inspect.getargspec(thing) + arguments = argspec.args[1:] + defaults = argspec.defaults or [] + params = [] + diff = len(arguments) - len(defaults) + for i, arg in enumerate(arguments): + if i >= diff: + default = defaults[i - diff] + else: + default = None + params.append({ + 'name': arg, + 'default': default + }) + data[name] = params else: - default = None - params.append({ - 'name': arg, - 'default': default - }) - data[name] = params + data[name] = None + return data + + + data = { + 'types': {}, + 'autocomplete_options': {} + } + + for n in locs: + if name is None or n == name: + res, typedescript = shell.push(n) + if 'netapi' in typedescript: + data['types'][n] = 'netapi' + else: + # get type of thing. + match = re.search('^<([A-Za-z]+) ', typedescript) + if match: + typename = match.group(1) + if typename in ['Nodespace', 'Node', 'Gate', 'Slot']: + data['types'][n] = typename + elif typename in nodetypes['nodetypes'] or typename in nodetypes['native_modules']: + data['types'][n] = 'Node' + else: + logging.getLogger('system').debug('no autocomplete for %s' % typename) + + for t in set(data['types'].values()): + if t == 'netapi': + netapi = nodenet.netapi + methods = inspect.getmembers(netapi, inspect.ismethod) + data['autocomplete_options']['netapi'] = parsemembers(methods) + elif t == 'Nodespace': + from micropsi_core.nodenet.nodespace import Nodespace + data['autocomplete_options']['Nodespace'] = parsemembers(inspect.getmembers(Nodespace, inspect.isroutine)) + elif t in ['Node', 'Gate', 'Slot']: + from micropsi_core.nodenet import node + cls = getattr(node, t) + data['autocomplete_options'][t] = parsemembers(inspect.getmembers(cls, inspect.isroutine)) return data + # --- end of API def filter_native_modules(engine=None): diff --git a/micropsi_core/tests/test_runtime.py b/micropsi_core/tests/test_runtime.py index 1e57939b..c4afa880 100644 --- a/micropsi_core/tests/test_runtime.py +++ b/micropsi_core/tests/test_runtime.py @@ -244,3 +244,24 @@ def test_run_netapi_command(test_nodenet): result, msg = micropsi.run_netapi_command(test_nodenet, command) assert result assert len(netapi.get_nodes()) == 4 + + +def test_get_netapi_autocomplete(test_nodenet): + micropsi.run_netapi_command(test_nodenet, "foonode = netapi.create_node('Pipe', None, 'foo')") + micropsi.run_netapi_command(test_nodenet, "foogate = foonode.get_gate('gen')") + micropsi.run_netapi_command(test_nodenet, "fooslot = foonode.get_slot('gen')") + micropsi.run_netapi_command(test_nodenet, "nodespace = netapi.create_nodespace(None, 'foospace')") + micropsi.run_netapi_command(test_nodenet, "barnode = netapi.create_node('Register', None, 'foo')") + data = micropsi.get_netapi_autocomplete_data(test_nodenet) + data['types']['foonode'] = 'Node' + data['types']['foogate'] = 'Gate' + data['types']['fooslot'] = 'Slot' + data['types']['nodespace'] = 'Nodespace' + data['types']['barnode'] = 'Node' + assert data['autocomplete_options']['Node']["get_gate"][0]['name'] == 'type' + assert data['autocomplete_options']['Gate']["get_links"] == [] + assert data['autocomplete_options']['Slot']["get_links"] == [] + assert data['autocomplete_options']['Nodespace']["get_known_ids"][0]['name'] == 'entitytype' + data = micropsi.get_netapi_autocomplete_data(test_nodenet, name='foonode') + assert list(data['types'].keys()) == ['foonode'] + assert list(data['autocomplete_options'].keys()) == ['Node'] diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index 36ff7349..628ba79a 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -1250,8 +1250,8 @@ def run_netapi_command(nodenet_uid, command): @rpc("get_netapi_signatures") -def get_netapi_signatures(nodenet_uid): - return True, runtime.get_netapi_signatures(nodenet_uid) +def get_netapi_autocomplete_data(nodenet_uid, name=None): + return True, runtime.get_netapi_autocomplete_data(nodenet_uid, name=None) # ----------------------------------------------------------------------------------------------- diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index fc1699c1..0e206a9a 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -123,7 +123,8 @@ $(function(){ registerResizeHandler(); - var autocomplete_data = {}; + var nametypes = {}; + var autocomplete_options = {}; var autocomplete_open = false; var autocomplete_pointer = -1; @@ -141,11 +142,27 @@ $(function(){ function enable(){ input.removeAttr("disabled"); - if($.isEmptyObject(autocomplete_data)){ - api.call('get_netapi_signatures', {nodenet_uid: currentNodenet}, function(data){ - autocomplete_data = data; - }); + if($.isEmptyObject(autocomplete_options)){ + getAutocompleteOptions(); + } + } + + function getAutocompleteOptions(name){ + params = {nodenet_uid: currentNodenet}; + if(name){ + params['name'] = name; } + api.call('get_netapi_signatures', params, function(data){ + if(name){ + var type = data.types[name] + nametypes[name] = type + autocomplete_options[type] = data.autocomplete_options[type]; + } else { + nametypes = data.types; + autocomplete_options = data.autocomplete_options; + } + }); + } function disable(){ @@ -161,10 +178,28 @@ $(function(){ if(isDisabled()) return; autocomplete_select(event); }); - input.keydown(function(event){ + input.keyup(function(event){ if(isDisabled()) return; var code = input.val(); switch(event.keyCode){ + case 13: // Enter + if(autocomplete_open){ + autocomplete_select(event); + } else { + submitInput(code); + } + break; + case 190: // dot. autocomplete + autocomplete(); + break; + case 32: // spacebar + if(event.ctrlKey && !autocomplete_open){ + autocomplete(); + } + break; + case 27: // escape + stop_autocomplete(); + break; case 38: // arrow up if(autocomplete_open){ event.preventDefault(); @@ -193,30 +228,6 @@ $(function(){ input.putCursorAtEnd() } break; - } - }); - input.keyup(function(event){ - if(isDisabled()) return; - var code = input.val(); - switch(event.keyCode){ - case 13: // Enter - if(autocomplete_open){ - autocomplete_select(event); - } else { - submitInput(code); - } - break; - case 190: // dot. autocomplete - autocomplete(); - break; - case 32: // spacebar - if(event.ctrlKey && !autocomplete_open){ - autocomplete(); - } - break; - case 27: - stop_autocomplete(); - break; default: history_pointer = -1 if(autocomplete_open){ @@ -255,12 +266,15 @@ $(function(){ if(code.indexOf('.') > -1){ var last = parts[parts.length - 1]; var obj = parts[parts.length - 2]; + obj = obj.match(/([a-zA-Z0-9_]+)/g); + obj = obj[obj.length - 1]; } - if(!obj || obj != "netapi"){ + if(!obj || !(obj in nametypes)){ return stop_autocomplete(); } html = []; - for(var key in autocomplete_data){ + var type = nametypes[obj]; + for(var key in autocomplete_options[type]){ if(key && (last == "" || key.startsWith(last))){ html.push('
      • '+key+'
        • '); } @@ -284,11 +298,20 @@ $(function(){ var selected = $(event.target).text(); } var val = input.val() - parts = val.split('.'); + var parts = input.val().split('.'); + var last = null; + var obj = null; + if(val.indexOf('.') > -1){ + var last = parts[parts.length - 1]; + var obj = parts[parts.length - 2]; + obj = obj.match(/([a-zA-Z0-9_]+)/g); + obj = obj[obj.length - 1]; + } parts.pop() val = parts.join('.') + '.' + selected; var params = []; - var data = autocomplete_data[selected]; + var type = nametypes[obj]; + var data = autocomplete_options[type][selected]; for(var i=0; i < data.length; i++){ if(!data[i].default){ params.push(data[i].name); @@ -314,6 +337,7 @@ $(function(){ function submitInput(code){ api.call('run_netapi_command', {nodenet_uid: currentNodenet, command: code}, function(data){ + getAutocompleteOptions(); data = data.replace(/\n+/g, '\n') var hist = history.text(); hist += "\n" + ">>> " + code; From 56d4017eb511f55e320873a4b7322e7d6c46404a Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 4 Apr 2016 16:47:19 +0200 Subject: [PATCH 251/306] fix test --- micropsi_core/tests/test_runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/tests/test_runtime.py b/micropsi_core/tests/test_runtime.py index c4afa880..c45732f9 100644 --- a/micropsi_core/tests/test_runtime.py +++ b/micropsi_core/tests/test_runtime.py @@ -239,7 +239,7 @@ def test_run_netapi_command(test_nodenet): command = "netapi.create_node()" result, msg = micropsi.run_netapi_command(test_nodenet, command) assert not result - assert msg.startswith("Traceback") + assert msg.startswith("TypeError") command = "for i in range(3): netapi.create_node('Register', None, 'test%d' % i)" result, msg = micropsi.run_netapi_command(test_nodenet, command) assert result From 1415b7d0a2af6091bf1aac068dabd2600ed1011a Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 4 Apr 2016 17:12:11 +0200 Subject: [PATCH 252/306] include properties in autocomplete, sort --- micropsi_core/runtime.py | 6 ++-- micropsi_core/tests/test_runtime.py | 1 + micropsi_server/static/js/netapi_console.js | 32 +++++++++++++-------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 2f20d872..756fdf02 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1381,8 +1381,6 @@ def parsemembers(members): data['types'][n] = typename elif typename in nodetypes['nodetypes'] or typename in nodetypes['native_modules']: data['types'][n] = 'Node' - else: - logging.getLogger('system').debug('no autocomplete for %s' % typename) for t in set(data['types'].values()): if t == 'netapi': @@ -1391,11 +1389,11 @@ def parsemembers(members): data['autocomplete_options']['netapi'] = parsemembers(methods) elif t == 'Nodespace': from micropsi_core.nodenet.nodespace import Nodespace - data['autocomplete_options']['Nodespace'] = parsemembers(inspect.getmembers(Nodespace, inspect.isroutine)) + data['autocomplete_options']['Nodespace'] = parsemembers(inspect.getmembers(Nodespace)) elif t in ['Node', 'Gate', 'Slot']: from micropsi_core.nodenet import node cls = getattr(node, t) - data['autocomplete_options'][t] = parsemembers(inspect.getmembers(cls, inspect.isroutine)) + data['autocomplete_options'][t] = parsemembers(inspect.getmembers(cls)) return data diff --git a/micropsi_core/tests/test_runtime.py b/micropsi_core/tests/test_runtime.py index c45732f9..ac7b216b 100644 --- a/micropsi_core/tests/test_runtime.py +++ b/micropsi_core/tests/test_runtime.py @@ -262,6 +262,7 @@ def test_get_netapi_autocomplete(test_nodenet): assert data['autocomplete_options']['Gate']["get_links"] == [] assert data['autocomplete_options']['Slot']["get_links"] == [] assert data['autocomplete_options']['Nodespace']["get_known_ids"][0]['name'] == 'entitytype' + assert data['autocomplete_options']['Node']['name'] is None data = micropsi.get_netapi_autocomplete_data(test_nodenet, name='foonode') assert list(data['types'].keys()) == ['foonode'] assert list(data['autocomplete_options'].keys()) == ['Node'] diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 0e206a9a..e756e6fd 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -274,9 +274,15 @@ $(function(){ } html = []; var type = nametypes[obj]; - for(var key in autocomplete_options[type]){ + var sorted = Object.keys(autocomplete_options[type]).sort(); + for(var i in sorted){ + var key = sorted[i]; if(key && (last == "" || key.startsWith(last))){ - html.push('
      • '+key+'
        • '); + if(autocomplete_options[type][key] == null){ + html.push('
      • '+key+'
        • '); + } else { + html.push('
      • '+key+'()
        • '); + } } } if (html.length == 0){ @@ -293,9 +299,9 @@ $(function(){ function autocomplete_select(event){ if($(event.target).attr('id') == 'console_input'){ - var selected = $('a.selected', autocomplete_container).text(); + var selected = $('a.selected', autocomplete_container).attr('data'); } else { - var selected = $(event.target).text(); + var selected = $(event.target).attr('data'); } var val = input.val() var parts = input.val().split('.'); @@ -312,18 +318,20 @@ $(function(){ var params = []; var type = nametypes[obj]; var data = autocomplete_options[type][selected]; - for(var i=0; i < data.length; i++){ - if(!data[i].default){ - params.push(data[i].name); - } else { - if(isNaN(data[i].default)){ - params.push(data[i].name+'='+'"'+data[i].default+'"') + if(data != null){ + for(var i=0; i < data.length; i++){ + if(!data[i].default){ + params.push(data[i].name); } else { - params.push(data[i].name+'='+data[i].default) + if(isNaN(data[i].default)){ + params.push(data[i].name+'='+'"'+data[i].default+'"') + } else { + params.push(data[i].name+'='+data[i].default) + } } } + val += '(' + params.join(', ') + ')'; } - val += '(' + params.join(', ') + ')'; input.val(val); stop_autocomplete(); } From 25d2f6a52be53e03f757420293b931adf1a65f83 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 4 Apr 2016 17:12:28 +0200 Subject: [PATCH 253/306] fix navigating history with arrowkeys --- micropsi_server/static/js/netapi_console.js | 66 +++++++++++++-------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index e756e6fd..21939efd 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -178,28 +178,8 @@ $(function(){ if(isDisabled()) return; autocomplete_select(event); }); - input.keyup(function(event){ - if(isDisabled()) return; - var code = input.val(); + input.keydown(function(event){ switch(event.keyCode){ - case 13: // Enter - if(autocomplete_open){ - autocomplete_select(event); - } else { - submitInput(code); - } - break; - case 190: // dot. autocomplete - autocomplete(); - break; - case 32: // spacebar - if(event.ctrlKey && !autocomplete_open){ - autocomplete(); - } - break; - case 27: // escape - stop_autocomplete(); - break; case 38: // arrow up if(autocomplete_open){ event.preventDefault(); @@ -228,6 +208,34 @@ $(function(){ input.putCursorAtEnd() } break; + } + }); + input.keyup(function(event){ + if(isDisabled()) return; + var code = input.val(); + switch(event.keyCode){ + case 13: // Enter + if(autocomplete_open){ + autocomplete_select(event); + } else { + submitInput(code); + } + break; + case 190: // dot. autocomplete + autocomplete(); + break; + case 32: // spacebar + if(event.ctrlKey && !autocomplete_open){ + autocomplete(); + } + break; + case 27: // escape + stop_autocomplete(); + break; + case 38: // arrow up + case 40: // arrow down + // do nothing. + break; default: history_pointer = -1 if(autocomplete_open){ @@ -244,7 +252,13 @@ $(function(){ if(autocomplete_pointer < autocomplete_container.children().length - 1){ autocomplete_pointer += 1; $('a.selected', autocomplete_container).removeClass('selected') - $($(autocomplete_container.children()[autocomplete_pointer]).children()).addClass('selected'); + var child = $(autocomplete_container.children()[autocomplete_pointer]); + $(child.children()).addClass('selected'); + var pos = child.offset().top; + + autocomplete_container.scrollTop( + autocomplete_container.scrollTop() + child.position().top + - autocomplete_container.height()/2 + child.height()/2); } } @@ -252,8 +266,12 @@ $(function(){ if(autocomplete_pointer > 0){ autocomplete_pointer -= 1; $('a.selected', autocomplete_container).removeClass('selected') - var item = $(autocomplete_container.children()[autocomplete_pointer]); - $(item.children()).addClass('selected'); + var child = $(autocomplete_container.children()[autocomplete_pointer]); + $(child.children()).addClass('selected'); + autocomplete_container.scrollTop( + autocomplete_container.scrollTop() + child.position().top + - autocomplete_container.height()/2 + child.height()/2); + } } From 1649cd3ba824b3673ed49ed74fca272a244a7b22 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 4 Apr 2016 21:47:01 +0200 Subject: [PATCH 254/306] fix missing assignment --- micropsi_server/static/js/netapi_console.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 21939efd..911f80c3 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -179,6 +179,8 @@ $(function(){ autocomplete_select(event); }); input.keydown(function(event){ + if(isDisabled()) return; + var code = input.val(); switch(event.keyCode){ case 38: // arrow up if(autocomplete_open){ From 9e81467b9dac04dacaee9cd53ae22d983bfe6666 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 5 Apr 2016 15:30:55 +0200 Subject: [PATCH 255/306] fix autocompletion for default-to-None-parameters --- micropsi_core/runtime.py | 13 ++++++------- micropsi_server/static/js/netapi_console.js | 6 ++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 756fdf02..98905ad3 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1349,19 +1349,18 @@ def parsemembers(members): diff = len(arguments) - len(defaults) for i, arg in enumerate(arguments): if i >= diff: - default = defaults[i - diff] + params.append({ + 'name': arg, + 'default': defaults[i - diff] + }) else: - default = None - params.append({ - 'name': arg, - 'default': default - }) + params.append({'name': arg}) + data[name] = params else: data[name] = None return data - data = { 'types': {}, 'autocomplete_options': {} diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 911f80c3..417225aa 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -340,10 +340,12 @@ $(function(){ var data = autocomplete_options[type][selected]; if(data != null){ for(var i=0; i < data.length; i++){ - if(!data[i].default){ + if(!('default' in data[i])){ params.push(data[i].name); } else { - if(isNaN(data[i].default)){ + if(data[i].default == null){ + params.push(data[i].name+'=None') + } else if(isNaN(data[i].default)){ params.push(data[i].name+'='+'"'+data[i].default+'"') } else { params.push(data[i].name+'='+data[i].default) From e289820b317a6dfefc97844b42851f077f522501 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 5 Apr 2016 15:31:10 +0200 Subject: [PATCH 256/306] do not submit empty commands --- micropsi_server/static/js/netapi_console.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 417225aa..6c59a274 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -220,7 +220,9 @@ $(function(){ if(autocomplete_open){ autocomplete_select(event); } else { - submitInput(code); + if(code.trim().length){ + submitInput(code); + } } break; case 190: // dot. autocomplete From 0ba92f004ab9ed436725e957f4277ee99f5e74ae Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 5 Apr 2016 15:31:35 +0200 Subject: [PATCH 257/306] fix last-thing-to-autocomplete-detection --- micropsi_server/static/js/netapi_console.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 6c59a274..9fa77063 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -289,7 +289,8 @@ $(function(){ var last = parts[parts.length - 1]; var obj = parts[parts.length - 2]; obj = obj.match(/([a-zA-Z0-9_]+)/g); - obj = obj[obj.length - 1]; + if(obj) + obj = obj[obj.length - 1]; } if(!obj || !(obj in nametypes)){ return stop_autocomplete(); From 2c67637f38cefa3ce670b7b027f4ee4efd6a1110 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 5 Apr 2016 15:34:59 +0200 Subject: [PATCH 258/306] increase copy-paste-ability comment return values, and remove the python console's >>> --- micropsi_server/static/js/netapi_console.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 9fa77063..7aac12cb 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -373,9 +373,9 @@ $(function(){ getAutocompleteOptions(); data = data.replace(/\n+/g, '\n') var hist = history.text(); - hist += "\n" + ">>> " + code; + hist += "\n" + code; if(data){ - hist += "\n" + data; + hist += "\n# " + data.replace(/\n/g, "\n# "); } history.text(hist); input.val(''); @@ -386,8 +386,8 @@ $(function(){ }, function(error){ var hist = history.text(); if(error.data){ - hist += "\n" + ">>> " + code; - hist += '\nERROR: '+error.data; + hist += "\n" + code; + hist += '\n# ERROR: '+error.data.replace(/\n/g, "\n# "); } history.text(hist); input.val(''); From 5624e4355fdc1fcac22fd2c32cd2649bc71b3bdd Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 5 Apr 2016 15:37:53 +0200 Subject: [PATCH 259/306] remove cruft --- micropsi_core/runtime.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 98905ad3..0e3ca216 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1317,13 +1317,7 @@ def get_agent_dashboard(nodenet_uid): def run_netapi_command(nodenet_uid, command): - import re shell = netapi_consoles[nodenet_uid] - name = None - match = re.search("^([a-zA-Z0-9_])+ ?=", command) - if match: - name = match.group(1) - return shell.push(command) From a2145750a1984920e7cb9a7917a6c10e51b3fdab Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 5 Apr 2016 21:50:35 +0200 Subject: [PATCH 260/306] autocomplete names as well --- micropsi_server/static/js/netapi_console.js | 45 ++++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 7aac12cb..6d2ed143 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -225,9 +225,6 @@ $(function(){ } } break; - case 190: // dot. autocomplete - autocomplete(); - break; case 32: // spacebar if(event.ctrlKey && !autocomplete_open){ autocomplete(); @@ -242,9 +239,7 @@ $(function(){ break; default: history_pointer = -1 - if(autocomplete_open){ - autocomplete(); - } + autocomplete(); } }); input.blur(function(){ @@ -291,10 +286,35 @@ $(function(){ obj = obj.match(/([a-zA-Z0-9_]+)/g); if(obj) obj = obj[obj.length - 1]; + } else if(code.match(/^([a-z0-9]+)?$/m)){ + return autocomplete_names(code); } if(!obj || !(obj in nametypes)){ return stop_autocomplete(); } + return autocomplete_properties(obj, last); + } + + function autocomplete_names(start){ + html = []; + for(var key in nametypes){ + if(start.length == 0 || key.startsWith(start)){ + html.push('
      • '+key+'
        • ') + } + } + if(html.length == 0){ + return stop_autocomplete(); + } + autocomplete_container.html(html.join('')); + autocomplete_container.css({ + 'position': 'absolute', + 'top': input.offset().top + input.height(), + 'left': input.offset().left + (input.val().length * 4), + 'display': 'block' + }); + } + + function autocomplete_properties(obj, last){ html = []; var type = nametypes[obj]; var sorted = Object.keys(autocomplete_options[type]).sort(); @@ -302,9 +322,9 @@ $(function(){ var key = sorted[i]; if(key && (last == "" || key.startsWith(last))){ if(autocomplete_options[type][key] == null){ - html.push('
      • '+key+'
        • '); + html.push('
      • '+key+'
        • '); } else { - html.push('
      • '+key+'()
        • '); + html.push('
      • '+key+'()
        • '); } } } @@ -322,10 +342,15 @@ $(function(){ function autocomplete_select(event){ if($(event.target).attr('id') == 'console_input'){ - var selected = $('a.selected', autocomplete_container).attr('data'); + var el = $('a.selected', autocomplete_container) } else { - var selected = $(event.target).attr('data'); + var el = $(event.target); + } + if(el.attr('data-complete') == 'name'){ + input.val(el.attr('data')); + return stop_autocomplete(); } + var selected = el.attr('data'); var val = input.val() var parts = input.val().split('.'); var last = null; From 4b171df117e99f69363e5e251a43c2c86ac39c46 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 5 Apr 2016 22:11:36 +0200 Subject: [PATCH 261/306] put cursor after opening bracket when autocompleting methods --- micropsi_server/static/js/netapi_console.js | 28 +++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 6d2ed143..3a2158cd 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -1,16 +1,24 @@ // thanks to Chris Coyier at css-tricks.com -jQuery.fn.putCursorAtEnd = function() { +jQuery.fn.putCursorAt = function(index=-1) { return this.each(function() { $(this).focus() // If this function exists... if (this.setSelectionRange) { // ... then use it (Doesn't work in IE) // Double the length because Opera is inconsistent about whether a carriage return is one character or two. Sigh. - var len = $(this).val().length * 2; + var len = index; + if(len < 0){ + len = $(this).val().length * 2; + } this.setSelectionRange(len, len); - } else { + } else if(this.createTextRange){ + var range = this.createTextRange(); + range.move('character', index); + range.select(); + } + else { // ... otherwise replace the contents with itself // (Doesn't work in Google Chrome) $(this).val($(this).val()); @@ -190,12 +198,12 @@ $(function(){ event.preventDefault(); history_pointer = command_history.length - 1; input.val(command_history[history_pointer]) - input.putCursorAtEnd() + input.putCursorAt(-1) } else if(history_pointer > 0 && code == command_history[history_pointer]) { event.preventDefault(); history_pointer -= 1; input.val(command_history[history_pointer]) - input.putCursorAtEnd() + input.putCursorAt(-1) } break; @@ -207,7 +215,7 @@ $(function(){ event.preventDefault(); history_pointer += 1; input.val(command_history[history_pointer]) - input.putCursorAtEnd() + input.putCursorAt(-1) } break; } @@ -380,9 +388,15 @@ $(function(){ } } } + var length = val.length; val += '(' + params.join(', ') + ')'; + input.val(val); + input.putCursorAt(length + 1); + + } else { + input.val(val); + input.putCursorAt(-1); } - input.val(val); stop_autocomplete(); } From 7e436e5e428e447eca69138955343be1d27c948b Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 6 Apr 2016 12:47:32 +0200 Subject: [PATCH 262/306] always autocomplete on ctrl+space. autocomplete after blanks, use ctrl+space to select only autocopmlete option --- micropsi_server/static/js/netapi_console.js | 50 ++++++++++++++------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 3a2158cd..0cad73f0 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -234,8 +234,8 @@ $(function(){ } break; case 32: // spacebar - if(event.ctrlKey && !autocomplete_open){ - autocomplete(); + if(event.ctrlKey){ + autocomplete(true); } break; case 27: // escape @@ -282,25 +282,28 @@ $(function(){ } } - function autocomplete(){ + function autocomplete(do_autoselect){ autocomplete_open = true; var code = input.val(); - var parts = input.val().split('.'); - var last = null; - var obj = null; if(code.indexOf('.') > -1){ + var parts = input.val().split('.'); var last = parts[parts.length - 1]; var obj = parts[parts.length - 2]; obj = obj.match(/([a-zA-Z0-9_]+)/g); if(obj) obj = obj[obj.length - 1]; - } else if(code.match(/^([a-z0-9]+)?$/m)){ - return autocomplete_names(code); + if(!obj || !(obj in nametypes)){ + stop_autocomplete(); + } + autocomplete_properties(obj, last); + } else if(code.match(/ ?([a-z0-9]+)?$/m)){ + var parts = input.val().split(' '); + var last = parts[parts.length - 1]; + autocomplete_names(last); } - if(!obj || !(obj in nametypes)){ - return stop_autocomplete(); + if (do_autoselect && autocomplete_container.children().length == 1){ + autocomplete_select(); } - return autocomplete_properties(obj, last); } function autocomplete_names(start){ @@ -349,18 +352,33 @@ $(function(){ } function autocomplete_select(event){ - if($(event.target).attr('id') == 'console_input'){ + if(event && $(event.target).attr('id') == 'console_input'){ var el = $('a.selected', autocomplete_container) } else { - var el = $(event.target); + if(event){ + var el = $(event.target); + } else if (autocomplete_container.children().length == 1) { + var el = $('a', autocomplete_container); + } } + var val = input.val() if(el.attr('data-complete') == 'name'){ - input.val(el.attr('data')); + if(val[val.length-1] == " "){ + console.log(val); + input.val(val + el.attr('data')); + } else { + var parts = val.split(" "); + parts.pop(); + var pre = ''; + if(parts.length){ + pre = parts.join(" ") +" "; + } + input.val(pre + el.attr('data')); + } return stop_autocomplete(); } var selected = el.attr('data'); - var val = input.val() - var parts = input.val().split('.'); + var parts = val.split('.'); var last = null; var obj = null; if(val.indexOf('.') > -1){ From c599deeacb5c22117a90bd56c742e7a5c630d039 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 7 Apr 2016 12:00:26 +0200 Subject: [PATCH 263/306] store a bunch of ui-properties for each nodespace, offer setter&getter --- .../nodenet/dict_engine/dict_nodenet.py | 3 +++ micropsi_core/nodenet/nodenet.py | 21 ++++++++++++++++++- .../nodenet/theano_engine/theano_nodenet.py | 4 +++- micropsi_core/runtime.py | 10 +++++++++ .../tests/test_runtime_nodenet_basics.py | 11 ++++++++++ micropsi_server/micropsi_app.py | 20 ++++++++++++++++++ 6 files changed, 67 insertions(+), 2 deletions(-) diff --git a/micropsi_core/nodenet/dict_engine/dict_nodenet.py b/micropsi_core/nodenet/dict_engine/dict_nodenet.py index 67d9f356..9083cc62 100644 --- a/micropsi_core/nodenet/dict_engine/dict_nodenet.py +++ b/micropsi_core/nodenet/dict_engine/dict_nodenet.py @@ -316,6 +316,8 @@ def initialize_nodenet(self, initfrom): if initfrom.get('runner_condition'): self.set_runner_condition(initfrom['runner_condition']) + self._nodespace_ui_properties = initfrom.get('nodespace_ui_properties', {}) + # set up nodespaces; make sure that parent nodespaces exist before children are initialized self._nodespaces = {} self._nodespaces["Root"] = DictNodespace(self, None, [0, 0, 0], name="Root", uid="Root") @@ -438,6 +440,7 @@ def delete_node(self, node_uid): self._track_deletion('nodes', node_uid) def delete_nodespace(self, uid): + self._nodespace_ui_properties.pop(uid, None) self.delete_node(uid) def clear(self): diff --git a/micropsi_core/nodenet/nodenet.py b/micropsi_core/nodenet/nodenet.py index d2d610a4..1dab43c4 100644 --- a/micropsi_core/nodenet/nodenet.py +++ b/micropsi_core/nodenet/nodenet.py @@ -66,7 +66,8 @@ def metadata(self): 'worldadapter': self._worldadapter_uid, 'version': NODENET_VERSION, 'runner_condition': self._runner_condition, - 'use_modulators': self.use_modulators + 'use_modulators': self.use_modulators, + 'nodespace_ui_properties': self._nodespace_ui_properties } return data @@ -154,6 +155,7 @@ def __init__(self, name="", worldadapter="Default", world=None, owner="", uid=No self.owner = owner self._monitors = {} + self._nodespace_ui_properties = {} self.netlock = Lock() @@ -296,6 +298,23 @@ def is_nodespace(self, uid): """ pass # pragma: no cover + def set_nodespace_properties(self, nodespace_uid, data): + """ + Sets a persistent property for UI purposes for the given nodespace + """ + if nodespace_uid not in self._nodespace_ui_properties: + self._nodespace_ui_properties[nodespace_uid] = {} + self._nodespace_ui_properties[nodespace_uid].update(data) + + def get_nodespace_properties(self, nodespace_uid=None): + """ + Return the nodespace properties of all or only the given nodespace + """ + if nodespace_uid: + return self._nodespace_ui_properties.get(nodespace_uid, {}) + else: + return self._nodespace_ui_properties + @abstractmethod def set_entity_positions(self, positions): """ Sets the position of nodes or nodespaces. diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 2e1cd558..2666d3b8 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -433,6 +433,8 @@ def initialize_nodenet(self, initfrom): if initfrom.get('runner_condition'): self.set_runner_condition(initfrom['runner_condition']) + self._nodespace_ui_properties = initfrom.get('nodespace_ui_properties', {}) + if len(initfrom) != 0: # now merge in all init data (from the persisted file typically) self.merge_data(initfrom, keep_uids=True, native_module_instances_only=True) @@ -971,7 +973,7 @@ def create_nodespace(self, parent_uid, position, name="", uid=None, options=None def delete_nodespace(self, nodespace_uid): if nodespace_uid is None or nodespace_uid == self.get_nodespace(None).uid: raise ValueError("The root nodespace cannot be deleted.") - + self._nodespace_ui_properties.pop(uid, None) partition = self.get_partition(nodespace_uid) nodespace_id = nodespace_from_id(nodespace_uid) if nodespace_id == 1 and partition.pid != self.rootpartition.pid: diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 1fc33e69..383dfbd3 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -818,6 +818,16 @@ def get_nodespace_changes(nodenet_uid, ndoespaces, since_step): return get_nodenet(nodenet_uid).get_nodespace_changes(ndoespaces, since_step) +def get_nodespace_properties(nodenet_uid, nodespace_uid): + """ retrieve the ui properties for the given nodespace""" + return get_nodenet(nodenet_uid).get_nodespace_properties(nodespace_uid) + + +def set_nodespace_properties(nodenet_uid, nodespace_uid, properties): + """ sets the ui properties for the given nodespace""" + return get_nodenet(nodenet_uid).set_nodespace_properties(nodespace_uid, properties) + + def __pythonify(name): s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name).lower() return re.sub('([\s+\W])', '_', s1) diff --git a/micropsi_core/tests/test_runtime_nodenet_basics.py b/micropsi_core/tests/test_runtime_nodenet_basics.py index c6bab61b..c4a4f8c4 100644 --- a/micropsi_core/tests/test_runtime_nodenet_basics.py +++ b/micropsi_core/tests/test_runtime_nodenet_basics.py @@ -449,3 +449,14 @@ def test_get_nodespace_changes_cycles(fixed_nodenet): net.step() result = micropsi.get_nodespace_changes(fixed_nodenet, [None], 1) assert nodes['B2'].uid not in result['nodes_deleted'] + + +def test_nodespace_properties(test_nodenet): + data = {'testvalue': 'foobar'} + rootns = micropsi.get_nodenet(test_nodenet).get_nodespace(None) + micropsi.set_nodespace_properties(test_nodenet, rootns.uid, data) + assert micropsi.nodenets[test_nodenet].metadata['nodespace_ui_properties'][rootns.uid] == data + assert micropsi.get_nodespace_properties(test_nodenet, rootns.uid) == data + micropsi.save_nodenet(test_nodenet) + micropsi.revert_nodenet(test_nodenet) + assert micropsi.get_nodespace_properties(test_nodenet, rootns.uid) == data diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index 584745be..8e4f5717 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -1055,6 +1055,16 @@ def get_nodespace_changes(nodenet_uid, nodespaces, since_step): return runtime.get_nodespace_changes(nodenet_uid, nodespaces, since_step) +@rpc("get_nodespace_properties") +def get_nodespace_properties(nodenet_uid, nodespace_uid=None): + return True, runtime.get_nodespace_properties(nodenet_uid, nodespace_uid) + + +@rpc("set_nodespace_properties") +def set_nodespace_properties(nodenet_uid, nodespace_uid, properties): + return True, runtime.set_nodespace_properties(nodenet_uid, nodespace_uid, properties) + + @rpc("get_node") def get_node(nodenet_uid, node_uid): return runtime.get_node(nodenet_uid, node_uid) @@ -1244,6 +1254,16 @@ def get_agent_dashboard(nodenet_uid): return True, runtime.get_agent_dashboard(nodenet_uid) +@rpc("run_netapi_command", permission_required="manage nodenets") +def run_netapi_command(nodenet_uid, command): + return runtime.run_netapi_command(nodenet_uid, command) + + +@rpc("get_netapi_signatures") +def get_netapi_autocomplete_data(nodenet_uid, name=None): + return True, runtime.get_netapi_autocomplete_data(nodenet_uid, name=None) + + # ----------------------------------------------------------------------------------------------- def main(host=None, port=None): From b36ff3c93aee299b1495e3da93b49fd1a79803ed Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 7 Apr 2016 12:00:46 +0200 Subject: [PATCH 264/306] some cleanup --- micropsi_core/nodenet/dict_engine/dict_nodespace.py | 9 --------- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 1 - micropsi_core/runtime.py | 4 ++-- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/micropsi_core/nodenet/dict_engine/dict_nodespace.py b/micropsi_core/nodenet/dict_engine/dict_nodespace.py index f7f8a265..547f1e7f 100644 --- a/micropsi_core/nodenet/dict_engine/dict_nodespace.py +++ b/micropsi_core/nodenet/dict_engine/dict_nodespace.py @@ -33,15 +33,6 @@ def __init__(self, nodenet, parent_nodespace, position, name="", uid=None, index self.contents_last_changed = nodenet.current_step nodenet._register_nodespace(self) - def get_data(self): - return { - "uid": self.uid, - "index": self.index, - "name": self.name, - "position": self.position, - "parent_nodespace": self.parent_nodespace, - } - def get_known_ids(self, entitytype=None): if entitytype: if entitytype not in self.__netentities: diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 2666d3b8..a02fec1c 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -890,7 +890,6 @@ def delete_partition(self, pid): for s in node.get_slot_types(): node.get_slot(s).invalidate_caches() - def create_nodespace(self, parent_uid, position, name="", uid=None, options=None): if options is None: options = {} diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 383dfbd3..0908e28a 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -811,11 +811,11 @@ def clone_nodes(nodenet_uid, node_uids, clonemode, nodespace=None, offset=[50, 5 return False, "Could not clone nodes. See log for details." -def get_nodespace_changes(nodenet_uid, ndoespaces, since_step): +def get_nodespace_changes(nodenet_uid, nodespaces, since_step): """ Returns a dict of changes that happened in the nodenet in the given nodespace since the given step. Contains uids of deleted nodes and nodespaces and the datadicts for changed or added nodes and nodespaces """ - return get_nodenet(nodenet_uid).get_nodespace_changes(ndoespaces, since_step) + return get_nodenet(nodenet_uid).get_nodespace_changes(nodespaces, since_step) def get_nodespace_properties(nodenet_uid, nodespace_uid): From a3165a1bb8ba2cf3ab85e2f72d6127aefd1216d1 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 7 Apr 2016 15:14:38 +0200 Subject: [PATCH 265/306] fix safari js syntax error --- micropsi_server/static/js/netapi_console.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_server/static/js/netapi_console.js b/micropsi_server/static/js/netapi_console.js index 0cad73f0..bb161371 100644 --- a/micropsi_server/static/js/netapi_console.js +++ b/micropsi_server/static/js/netapi_console.js @@ -1,7 +1,7 @@ // thanks to Chris Coyier at css-tricks.com -jQuery.fn.putCursorAt = function(index=-1) { +jQuery.fn.putCursorAt = function(index) { return this.each(function() { $(this).focus() // If this function exists... From 6303751e83793b5efc3290d78530059ea2facb1f Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 7 Apr 2016 15:53:27 +0200 Subject: [PATCH 266/306] make sure nodenet is loaded --- micropsi_core/runtime.py | 1 + 1 file changed, 1 insertion(+) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 0e3ca216..bee65c00 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1317,6 +1317,7 @@ def get_agent_dashboard(nodenet_uid): def run_netapi_command(nodenet_uid, command): + get_nodenet(nodenet_uid) shell = netapi_consoles[nodenet_uid] return shell.push(command) From 5c17f631e09557fb5f157530c01e731a5216e758 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 7 Apr 2016 16:25:30 +0200 Subject: [PATCH 267/306] remember netapi collapsed stated --- micropsi_server/static/js/dialogs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_server/static/js/dialogs.js b/micropsi_server/static/js/dialogs.js index 642b74cf..4b74b5ba 100644 --- a/micropsi_server/static/js/dialogs.js +++ b/micropsi_server/static/js/dialogs.js @@ -657,7 +657,7 @@ var listeners = {} var calculationRunning = false; var currentNodenet; var runner_properties = {}; -var sections = ['nodenet_editor', 'monitor', 'world_editor']; +var sections = ['nodenet_editor', 'netapi_console', 'monitor', 'world_editor']; register_stepping_function = function(type, input, callback){ From 869ff124d43abd60597659637535d673a5966eab Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 7 Apr 2016 18:14:19 +0200 Subject: [PATCH 268/306] frontend: make renderlinks and activation_display nodespace_properties --- micropsi_core/runtime.py | 1 + micropsi_server/static/js/dialogs.js | 4 +- micropsi_server/static/js/nodenet.js | 75 ++++++++++++++++------------ micropsi_server/view/nodenet.tpl | 30 +++++------ 4 files changed, 62 insertions(+), 48 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 0908e28a..9bf8cb6b 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -670,6 +670,7 @@ def get_nodespace_list(nodenet_uid): 'name': nodespace.name, 'parent': nodespace.parent_nodespace, 'nodes': {}, + 'properties': nodenet.get_nodespace_properties(uid) } for nid in nodespace.get_known_ids('nodes'): data[uid]['nodes'][nid] = { diff --git a/micropsi_server/static/js/dialogs.js b/micropsi_server/static/js/dialogs.js index 642b74cf..1e8ebeea 100644 --- a/micropsi_server/static/js/dialogs.js +++ b/micropsi_server/static/js/dialogs.js @@ -890,7 +890,7 @@ $(document).ready(function() { $('#nodenet_mgr').dataTable( { "sDom": "<'row'<'span6'l><'span6'f>r>t<'row'<'span6'i><'span6'p>>", "sPaginationType": "bootstrap" - } ); + }); $('textarea.loc').autogrow(); if($('.frontend_section').length == 1){ $('.frontend_section').addClass('in'); @@ -900,7 +900,7 @@ $(document).ready(function() { refreshNodenetList(); setButtonStates(false); if(currentNodenet){ - fetch_stepping_info(); + $(document).trigger('nodenet_changed', currentNodenet); } }); diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index 2ba144c6..247546f4 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -101,9 +101,7 @@ if (nodenetcookie && nodenetcookie.indexOf('/') > 0){ currentNodeSpace = ''; } -if(!$.cookie('renderlinks')){ - $.cookie('renderlinks', 'always'); -} +nodespaceProperties = {}; currentWorldadapter = null; @@ -161,14 +159,6 @@ globalDataTargets = []; available_operations = {}; -$(document).on('load_nodenet', function(event, uid){ - ns = 'Root'; - if(uid == currentNodenet){ - ns = currentNodeSpace; - } - setCurrentNodenet(uid, ns); -}); - $(document).on('nodenet_changed', function(event, new_nodenet){ setCurrentNodenet(new_nodenet, null, true); }); @@ -238,8 +228,6 @@ function setNodenetValues(data){ $('#nodenet_uid').val(currentNodenet); $('#nodenet_name').val(data.name); $('#nodenet_snap').attr('checked', data.snap_to_grid); - $('#nodenet_renderlinks').val(nodenet_data.renderlinks); - $('#nodenet_vis').val(nodenet_data.vis); if (!jQuery.isEmptyObject(worldadapters)) { var worldadapter_select = $('#nodenet_worldadapter'); worldadapter_select.val(data.worldadapter); @@ -280,6 +268,14 @@ function setCurrentNodenet(uid, nodespace, changed){ currentNodenet = uid; currentNodeSpace = data.rootnodespace; + nodespaceProperties = data.nodespace_ui_properties; + for(var key in data.nodespaces){ + if(!(key in nodespaceProperties)){ + nodespaceProperties[key] = {}; + } + if(!nodespaceProperties[key].renderlinks) nodespaceProperties[key].renderlinks = 'always'; + if(!nodespaceProperties[key].activation_display) nodespaceProperties[key].activation_display = 'redgreen'; + } if(nodenetChanged){ clipboard = {}; selection = {}; @@ -292,9 +288,7 @@ function setCurrentNodenet(uid, nodespace, changed){ $(document).trigger('nodenet_loaded', uid); nodenet_data = data; - nodenet_data['renderlinks'] = $.cookie('renderlinks') || 'always'; nodenet_data['snap_to_grid'] = $.cookie('snap_to_grid') || viewProperties.snap_to_grid; - nodenet_data['vis'] = $.cookie('activation_display') || 'redgreen'; showDefaultForm(); @@ -361,6 +355,7 @@ function getNodespaceList(){ for(var i=0; i < sorted.length; i++){ nodespaces[sorted[i].uid] = sorted[i]; html += '
      • '+sorted[i].name+'
        • '; + nodespaceProperties[sorted[i].uid] = sorted[i].properties; } $('#nodespace_control ul').html(html); $("#current_nodespace_name").text(nodespaces[currentNodeSpace].name); @@ -428,7 +423,7 @@ function setNodespaceData(data, changed){ } } - if(nodenet_data.renderlinks == 'selection'){ + if(nodespaceProperties[currentNodeSpace].renderlinks == 'selection'){ loadLinksForSelection(function(data){ for(var uid in links) { if(!(uid in data)) { @@ -587,7 +582,7 @@ function get_nodenet_params(){ return { 'nodespaces': [currentNodeSpace], 'step': currentSimulationStep - 1, - 'include_links': $.cookie('renderlinks') == 'always', + 'include_links': nodespaceProperties[currentNodeSpace].renderlinks == 'always', } } function get_nodenet_diff_params(){ @@ -618,7 +613,7 @@ function refreshNodespace(nodespace, step, callback){ nodenet_uid: currentNodenet, nodespaces: [nodespace], }; - params.include_links = nodenet_data['renderlinks'] == 'always'; + params.include_links = nodespaceProperties[nodespace].renderlinks == 'always'; api.call('get_nodes', params , success=function(data){ var changed = nodespace != currentNodeSpace; if(changed){ @@ -1204,10 +1199,10 @@ function createPlaceholder(node, direction, point){ // draw link function renderLink(link, force) { - if(nodenet_data.renderlinks == 'no' && !force){ + if(nodespaceProperties[currentNodeSpace].renderlinks == 'no'){ return; } - if(nodenet_data.renderlinks == 'selection' && !force){ + if(nodespaceProperties[currentNodeSpace] == 'selection'){ var is_hovered = hoverNode && (link.sourceNodeUid == hoverNode.uid || link.targetNodeUid == hoverNode.uid); var is_selected = selection && (link.sourceNodeUid in selection || link.targetNodeUid in selection); if(!is_hovered && !is_selected){ @@ -1253,7 +1248,7 @@ function renderLink(link, force) { linkItem.name = "link"; var linkContainer = new Group(linkItem); linkContainer.name = link.uid; - if (nodenet_data['vis'] == 'alpha'){ + if (nodespaceProperties[currentNodeSpace].activation_display == 'alpha'){ if(sourceNode){ linkContainer.opacity = Math.max(0.1, sourceNode.sheaves[currentSheaf].activation) } else { @@ -1804,11 +1799,11 @@ function setActivation(node) { } if (node.uid in nodeLayer.children) { var nodeItem = nodeLayer.children[node.uid]; - if((nodenet_data['vis'] != 'alpha') || node.sheaves[currentSheaf].activation > 0.5){ + if((nodespaceProperties[currentNodeSpace].activation_display != 'alpha') || node.sheaves[currentSheaf].activation > 0.5){ node.fillColor = nodeItem.children["activation"].children["body"].fillColor = activationColor(node.sheaves[currentSheaf].activation, viewProperties.nodeColor); } - if(nodenet_data['vis'] == 'alpha'){ + if(nodespaceProperties[currentNodeSpace].activation_display == 'alpha'){ for(var i in nodeItem.children){ if(nodeItem.children[i].name == 'labelText'){ nodeItem.children[i].opacity = 0; @@ -1904,7 +1899,7 @@ function deselectLink(linkUid) { delete selection[linkUid]; if(linkUid in linkLayer.children){ var linkShape = linkLayer.children[linkUid].children["link"]; - if(nodenet_data.renderlinks == 'no' || nodenet_data.renderlinks == 'selection'){ + if(nodespaceProperties[currentNodeSpace].renderlinks == 'no' || nodespaceProperties[currentNodeSpace].renderlinks == 'selection'){ linkLayer.children[linkUid].remove(); } linkShape.children["line"].strokeColor = links[linkUid].strokeColor; @@ -2410,7 +2405,7 @@ function onMouseUp(event) { selectionRectangle.width = selectionRectangle.height = 1; selectionBox.setBounds(selectionRectangle); } - if(currentNodenet && nodenet_data && nodenet_data['renderlinks'] == 'selection'){ + if(currentNodenet && nodenet_data && nodespaceProperties[currentNodeSpace].renderlinks == 'selection'){ loadLinksForSelection(); } } @@ -3520,7 +3515,7 @@ function finalizeLinkHandler(nodeUid, slotIndex) { nodes[link.sourceNodeUid].linksToOutside.push(link.uid); } } - if(nodenet_data.renderlinks == 'always'){ + if(nodespaceProperties[currentNodeSpace].renderlinks == 'always'){ addLink(link); } }); @@ -3773,13 +3768,8 @@ function handleEditNodenet(event){ if(worldadapter){ params.worldadapter = worldadapter; } - nodenet_data.renderlinks = $('#nodenet_renderlinks').val(); - $.cookie('renderlinks', nodenet_data.renderlinks || '', {path: '/', expires: 7}) nodenet_data.snap_to_grid = $('#nodenet_snap').attr('checked'); $.cookie('snap_to_grid', nodenet_data.snap_to_grid || '', {path: '/', expires: 7}) - if(nodenet_data.vis != $('#nodenet_vis').val()) reload = true; - nodenet_data.vis = $('#nodenet_vis').val(); - $.cookie('activation_display', nodenet_data.vis || '', {path: '/', expires: 7}) api.call("set_nodenet_properties", params, success=function(data){ dialogs.notification('Nodenet data saved', 'success'); @@ -3799,6 +3789,27 @@ function handleEditNodespace(event){ if(name != nodespaces[currentNodeSpace].name){ renameNode(currentNodeSpace, name); } + properties = {}; + properties['renderlinks'] = $('#nodespace_renderlinks').val(); + properties['activation_display'] = $('#nodespace_activation_display').val(); + var update = false; + for(var key in properties){ + if(properties[key] != nodespaceProperties[currentNodeSpace][key]){ + update = true; + nodespaceProperties[currentNodeSpace][key] = properties[key]; + } else { + delete properties[key]; + } + } + if(update){ + params = {nodenet_uid: currentNodenet, nodespace_uid: currentNodeSpace, properties: properties} + api.call('set_nodespace_properties', params); + if ('renderlinks' in properties){ + refreshNodespace(); + } else { + redrawNodeNet(); + } + } } @@ -4106,6 +4117,8 @@ function updateNodespaceForm(){ } else { $('#nodespace_name').removeAttr('disabled'); } + $('#nodespace_renderlinks').val(nodespaceProperties[currentNodeSpace].renderlinks); + $('#nodespace_activation_display').val(nodespaceProperties[currentNodeSpace].activation_display); } } diff --git a/micropsi_server/view/nodenet.tpl b/micropsi_server/view/nodenet.tpl index 0d7801d0..0d2b348b 100644 --- a/micropsi_server/view/nodenet.tpl +++ b/micropsi_server/view/nodenet.tpl @@ -75,21 +75,6 @@ - - - - - - - - @@ -117,6 +102,21 @@ + + + + + + + +
        From e4075e40805e5fde293ae5745493fb08b7c083e7 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 7 Apr 2016 18:37:17 +0200 Subject: [PATCH 269/306] fix renderlinks --- micropsi_server/static/js/nodenet.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index 247546f4..35ac9613 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -1202,10 +1202,9 @@ function renderLink(link, force) { if(nodespaceProperties[currentNodeSpace].renderlinks == 'no'){ return; } - if(nodespaceProperties[currentNodeSpace] == 'selection'){ - var is_hovered = hoverNode && (link.sourceNodeUid == hoverNode.uid || link.targetNodeUid == hoverNode.uid); + if(nodespaceProperties[currentNodeSpace].renderlinks == 'selection'){ var is_selected = selection && (link.sourceNodeUid in selection || link.targetNodeUid in selection); - if(!is_hovered && !is_selected){ + if(!is_selected){ return; } } From d62053c5882e0dc7f0d62119d04e2587039b0ac2 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 8 Apr 2016 11:48:06 +0200 Subject: [PATCH 270/306] offer netapi methods as well --- micropsi_core/nodenet/netapi.py | 8 ++++++++ micropsi_core/tests/test_node_netapi.py | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/micropsi_core/nodenet/netapi.py b/micropsi_core/nodenet/netapi.py index b6c72d9b..a35c6e52 100644 --- a/micropsi_core/nodenet/netapi.py +++ b/micropsi_core/nodenet/netapi.py @@ -573,5 +573,13 @@ def decay_por_links(self, nodespace_uid): if link.weight > 0: link._set_weight(max(link.weight * decay_factor, 0)) + def get_nodespace_properties(self, nodespace_uid=None): + """ retrieve the ui properties for the given nodespace""" + return self.__nodenet.get_nodespace_properties(nodespace_uid) + + def set_nodespace_properties(self, nodespace_uid, properties): + """ sets the ui properties for the given nodespace""" + self.__nodenet.set_nodespace_properties(nodespace_uid, properties) + def announce_nodes(self, nodespace_uid, numer_of_nodes, average_element_per_node): pass diff --git a/micropsi_core/tests/test_node_netapi.py b/micropsi_core/tests/test_node_netapi.py index 87935a9a..d7ae7d86 100644 --- a/micropsi_core/tests/test_node_netapi.py +++ b/micropsi_core/tests/test_node_netapi.py @@ -1141,3 +1141,12 @@ def test_unlink_slot(test_nodenet): netapi.unlink_slot(node, 'por', source_gate_name='sur') assert len(node.get_slot('por').get_links()) == 2 # pipe1:gen, pipe2:gen assert len(node.get_slot('sur').get_links()) == 2 # only sur->por unlinked + + +def test_nodespace_properties(test_nodenet): + nodenet = micropsi.get_nodenet(test_nodenet) + netapi = nodenet.netapi + rootns = netapi.get_nodespace(None) + netapi.set_nodespace_properties(None, {'foo': 'bar'}) + data = netapi.get_nodespace_properties() + assert data[rootns.uid] == {'foo': 'bar'} From be66902927e94037b3462e9e5084662543846795 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 8 Apr 2016 11:48:19 +0200 Subject: [PATCH 271/306] fix parameter naming --- micropsi_core/nodenet/dict_engine/dict_nodenet.py | 6 +++--- micropsi_core/nodenet/nodenet.py | 2 +- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/micropsi_core/nodenet/dict_engine/dict_nodenet.py b/micropsi_core/nodenet/dict_engine/dict_nodenet.py index 9083cc62..15e185b5 100644 --- a/micropsi_core/nodenet/dict_engine/dict_nodenet.py +++ b/micropsi_core/nodenet/dict_engine/dict_nodenet.py @@ -439,9 +439,9 @@ def delete_node(self, node_uid): del self._nodes[node_uid] self._track_deletion('nodes', node_uid) - def delete_nodespace(self, uid): - self._nodespace_ui_properties.pop(uid, None) - self.delete_node(uid) + def delete_nodespace(self, nodespace_uid): + self._nodespace_ui_properties.pop(nodespace_uid, None) + self.delete_node(nodespace_uid) def clear(self): super(DictNodenet, self).clear() diff --git a/micropsi_core/nodenet/nodenet.py b/micropsi_core/nodenet/nodenet.py index 1dab43c4..8fb3c16b 100644 --- a/micropsi_core/nodenet/nodenet.py +++ b/micropsi_core/nodenet/nodenet.py @@ -329,7 +329,7 @@ def create_nodespace(self, parent_uid, position, name="", uid=None): pass # pragma: no cover @abstractmethod - def delete_nodespace(self, uid): + def delete_nodespace(self, nodespace_uid): """ Deletes the nodespace with the given UID, and everything it contains """ diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index a02fec1c..a54e4fee 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -972,7 +972,7 @@ def create_nodespace(self, parent_uid, position, name="", uid=None, options=None def delete_nodespace(self, nodespace_uid): if nodespace_uid is None or nodespace_uid == self.get_nodespace(None).uid: raise ValueError("The root nodespace cannot be deleted.") - self._nodespace_ui_properties.pop(uid, None) + self._nodespace_ui_properties.pop(nodespace_uid, None) partition = self.get_partition(nodespace_uid) nodespace_id = nodespace_from_id(nodespace_uid) if nodespace_id == 1 and partition.pid != self.rootpartition.pid: From dbee990dce27801a17a512dbfca76b8bfaf83390 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 8 Apr 2016 11:49:08 +0200 Subject: [PATCH 272/306] allow to set_parameter for None-rootnodespace, fix parameter default for getter --- micropsi_core/nodenet/nodenet.py | 1 + micropsi_core/runtime.py | 2 +- micropsi_core/tests/test_runtime_nodenet_basics.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/micropsi_core/nodenet/nodenet.py b/micropsi_core/nodenet/nodenet.py index 8fb3c16b..dcfbbf53 100644 --- a/micropsi_core/nodenet/nodenet.py +++ b/micropsi_core/nodenet/nodenet.py @@ -302,6 +302,7 @@ def set_nodespace_properties(self, nodespace_uid, data): """ Sets a persistent property for UI purposes for the given nodespace """ + nodespace_uid = self.get_nodespace(nodespace_uid).uid if nodespace_uid not in self._nodespace_ui_properties: self._nodespace_ui_properties[nodespace_uid] = {} self._nodespace_ui_properties[nodespace_uid].update(data) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index 6f3dade5..9643a416 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -875,7 +875,7 @@ def get_nodespace_changes(nodenet_uid, nodespaces, since_step): return get_nodenet(nodenet_uid).get_nodespace_changes(nodespaces, since_step) -def get_nodespace_properties(nodenet_uid, nodespace_uid): +def get_nodespace_properties(nodenet_uid, nodespace_uid=None): """ retrieve the ui properties for the given nodespace""" return get_nodenet(nodenet_uid).get_nodespace_properties(nodespace_uid) diff --git a/micropsi_core/tests/test_runtime_nodenet_basics.py b/micropsi_core/tests/test_runtime_nodenet_basics.py index c4a4f8c4..ac8eee83 100644 --- a/micropsi_core/tests/test_runtime_nodenet_basics.py +++ b/micropsi_core/tests/test_runtime_nodenet_basics.py @@ -460,3 +460,5 @@ def test_nodespace_properties(test_nodenet): micropsi.save_nodenet(test_nodenet) micropsi.revert_nodenet(test_nodenet) assert micropsi.get_nodespace_properties(test_nodenet, rootns.uid) == data + properties = micropsi.get_nodespace_properties(test_nodenet) + assert properties[rootns.uid] == data From 41f48ca56f6b911b2140b33c727341d1d854777b Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 8 Apr 2016 12:19:09 +0200 Subject: [PATCH 273/306] default renderlinks to old cookie setting --- micropsi_server/static/js/nodenet.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index 35ac9613..5b7dcfbc 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -103,6 +103,9 @@ if (nodenetcookie && nodenetcookie.indexOf('/') > 0){ nodespaceProperties = {}; +// compatibility +renderlinks_default = $.cookie('renderlinks') || 'always'; + currentWorldadapter = null; var currentSheaf = "default"; @@ -273,7 +276,7 @@ function setCurrentNodenet(uid, nodespace, changed){ if(!(key in nodespaceProperties)){ nodespaceProperties[key] = {}; } - if(!nodespaceProperties[key].renderlinks) nodespaceProperties[key].renderlinks = 'always'; + if(!nodespaceProperties[key].renderlinks) nodespaceProperties[key].renderlinks = renderlinks_default; if(!nodespaceProperties[key].activation_display) nodespaceProperties[key].activation_display = 'redgreen'; } if(nodenetChanged){ From 5915160a1d3c4a23c39bfff507263c04a4e86c68 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Mon, 11 Apr 2016 16:12:59 +0200 Subject: [PATCH 274/306] fix group_nodes_by_names --- micropsi_core/nodenet/theano_engine/theano_nodenet.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/micropsi_core/nodenet/theano_engine/theano_nodenet.py b/micropsi_core/nodenet/theano_engine/theano_nodenet.py index 2e1cd558..7cafbe6c 100644 --- a/micropsi_core/nodenet/theano_engine/theano_nodenet.py +++ b/micropsi_core/nodenet/theano_engine/theano_nodenet.py @@ -1429,14 +1429,16 @@ def group_nodes_by_names(self, nodespace_uid, node_name_prefix=None, gatetype="g if nodespace_uid is None: nodespace_uid = self.get_nodespace(None).uid + partition = self.get_partition(nodespace_uid) + if group_name is None: group_name = node_name_prefix ids = [] for uid, name in self.names.items(): - partition = self.get_partition(uid) - if self.is_node(uid) and name.startswith(node_name_prefix) and \ - (partition.allocated_node_parents[node_from_id(uid)] == nodespace_from_id(nodespace_uid)): + parentpartition = self.get_partition(uid) + if parentpartition == partition and self.is_node(uid) and name.startswith(node_name_prefix) and \ + (parentpartition.allocated_node_parents[node_from_id(uid)] == nodespace_from_id(nodespace_uid)): ids.append(uid) self.group_nodes_by_ids(nodespace_uid, ids, group_name, gatetype, sortby) From 6ab1ef46eb62b5604027d93a8652d3359900c27c Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Tue, 12 Apr 2016 15:59:49 +0200 Subject: [PATCH 275/306] fix z-axis --- micropsi_core/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index bee65c00..d7ca5f37 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -1009,7 +1009,7 @@ def generate_netapi_fragment(nodenet_uid, node_uids): lines.append("") # positions - origin = [100, 100, 100] + origin = [100, 100, 0] factor = [int(min(xpos)), int(min(ypos)), int(min(zpos))] lines.append("origin_pos = (%d, %d, %d)" % (origin[0], origin[1], origin[2])) for node in nodes + nodespaces: From f0bd05268ee45edf0fd4be6d62af3314c9545c89 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 13 Apr 2016 15:12:09 +0200 Subject: [PATCH 276/306] get_world_view for timeseries world --- micropsi_core/world/timeseries/timeseries.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 4e5e4efe..a0e83609 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -156,6 +156,15 @@ def get_config_options(): 'options': ["True", "False"]} ] + def get_world_view(self, step): + return { + 'first_timestamp': str(self.timestamps[0]), + 'last_timestamp': str(self.timestamps[-1]), + 'total_timestamps': len(self.timestamps), + 'current_timestamp': str(self.timestamps[self.current_step]), + 'current_step': self.current_step, + } + class TimeSeriesRunner(ArrayWorldAdapter): From 85126481ca73355e7ee471ffbf9d1ce82f4c669e Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 13 Apr 2016 15:26:36 +0200 Subject: [PATCH 277/306] change asset key to differentiate between paperjs and normal js --- micropsi_core/world/island/island.py | 2 +- micropsi_core/world/minecraft/minecraft.py | 4 ++-- micropsi_server/view/world.tpl | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/micropsi_core/world/island/island.py b/micropsi_core/world/island/island.py index 62f29323..065e0bc5 100644 --- a/micropsi_core/world/island/island.py +++ b/micropsi_core/world/island/island.py @@ -21,7 +21,7 @@ class Island(World): assets = { 'background': "island/psi_1.png", 'template': 'island/island.tpl', - 'js': "island/island.js", + 'paperjs': "island/island.js", 'x': 2048, 'y': 2048, 'icons': { diff --git a/micropsi_core/world/minecraft/minecraft.py b/micropsi_core/world/minecraft/minecraft.py index 032db7fb..f21ad200 100644 --- a/micropsi_core/world/minecraft/minecraft.py +++ b/micropsi_core/world/minecraft/minecraft.py @@ -30,7 +30,7 @@ class Minecraft(World): assets = { 'template': 'minecraft/minecraft.tpl', - 'js': 'minecraft/minecraft.js', + 'paperjs': 'minecraft/minecraft.js', 'x': 256, 'y': 256, } @@ -144,7 +144,7 @@ class Minecraft2D(Minecraft): assets = { 'template': 'minecraft/minecraft.tpl', - 'js': 'minecraft/minecraft2d.js', + 'paperjs': 'minecraft/minecraft2d.js', } def step(self): diff --git a/micropsi_server/view/world.tpl b/micropsi_server/view/world.tpl index dee7dd99..9bb133c4 100644 --- a/micropsi_server/view/world.tpl +++ b/micropsi_server/view/world.tpl @@ -46,6 +46,8 @@ -%if world_assets.get('js'): +%if world_assets.get('paperjs'): +%elif world_assets.get('js'): + %end From 40fe5472a4d068d20907648983c97fc771c95b68 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 13 Apr 2016 15:27:38 +0200 Subject: [PATCH 278/306] worldjs does not need paperscript --- micropsi_server/static/js/world.js | 98 +++++++++++++++--------------- micropsi_server/view/world.tpl | 2 +- 2 files changed, 51 insertions(+), 49 deletions(-) diff --git a/micropsi_server/static/js/world.js b/micropsi_server/static/js/world.js index 3a5a7d65..7822e6db 100644 --- a/micropsi_server/static/js/world.js +++ b/micropsi_server/static/js/world.js @@ -1,56 +1,58 @@ -var canvas = $('#world'); +$(function(){ + currentWorld = $.cookie('selected_world') || null; + currentWorldSimulationStep = 0; -currentWorld = $.cookie('selected_world') || null; -currentWorldSimulationStep = 0; + worldRunning = false; + wasRunning = false; -worldRunning = false; -wasRunning = false; + registerResizeHandler(); -registerResizeHandler(); - -function get_world_data(){ - return {step: currentWorldSimulationStep}; -} + function get_world_data(){ + return {step: currentWorldSimulationStep}; + } -function set_world_data(data){ - if(!jQuery.isEmptyObject(data)){ - currentWorldSimulationStep = data.current_step; + function set_world_data(data){ + if(!jQuery.isEmptyObject(data)){ + currentWorldSimulationStep = data.current_step; + } } -} - -register_stepping_function('world', get_world_data, set_world_data); - -function updateViewSize() { - view.draw(true); -} - -function registerResizeHandler(){ - // resize handler for nodenet viewer: - var isDragging = false; - var container = $('.section.world .editor_field'); - if($.cookie('world_editor_height')){ - container.height($.cookie('world_editor_height')); - try{ - updateViewSize(); - } catch(err){} + + register_stepping_function('world', get_world_data, set_world_data); + + function updateViewSize() { + if(typeof view != 'undefined'){ + view.draw(true); + } } - var startHeight, startPos, newHeight; - $("a#worldSizeHandle").mousedown(function(event) { - startHeight = container.height(); - startPos = event.pageY; - $(window).mousemove(function(event) { - isDragging = true; - newHeight = startHeight + (event.pageY - startPos); - container.height(newHeight); - updateViewSize(); - }); - }); - $(window).mouseup(function(event) { - if(isDragging){ - $.cookie('world_editor_height', container.height(), {expires:7, path:'/'}); + + function registerResizeHandler(){ + // resize handler for nodenet viewer: + var isDragging = false; + var container = $('.section.world .editor_field'); + if($.cookie('world_editor_height')){ + container.height($.cookie('world_editor_height')); + try{ + updateViewSize(); + } catch(err){} } - isDragging = false; - $(window).unbind("mousemove"); - }); -} + var startHeight, startPos, newHeight; + $("a#worldSizeHandle").mousedown(function(event) { + startHeight = container.height(); + startPos = event.pageY; + $(window).mousemove(function(event) { + isDragging = true; + newHeight = startHeight + (event.pageY - startPos); + container.height(newHeight); + updateViewSize(); + }); + }); + $(window).mouseup(function(event) { + if(isDragging){ + $.cookie('world_editor_height', container.height(), {expires:7, path:'/'}); + } + isDragging = false; + $(window).unbind("mousemove"); + }); + } +}); \ No newline at end of file diff --git a/micropsi_server/view/world.tpl b/micropsi_server/view/world.tpl index 9bb133c4..67c3d6c1 100644 --- a/micropsi_server/view/world.tpl +++ b/micropsi_server/view/world.tpl @@ -44,7 +44,7 @@
        - + %if world_assets.get('paperjs'): From 21c580ab09f29887eb3ee03dd66b9ca5ebdf8537 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 13 Apr 2016 15:28:02 +0200 Subject: [PATCH 279/306] timeseries controls frontend --- micropsi_core/world/timeseries/timeseries.py | 7 +++ .../static/css/micropsi-styles.css | 10 ++++ .../static/timeseries/timeseries.js | 55 +++++++++++++++++++ .../static/timeseries/timeseries.tpl | 11 ++++ micropsi_server/view/boilerplate.tpl | 2 + 5 files changed, 85 insertions(+) create mode 100644 micropsi_server/static/timeseries/timeseries.js create mode 100644 micropsi_server/static/timeseries/timeseries.tpl diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index a0e83609..ca9e3786 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -23,9 +23,16 @@ class TimeSeries(World): """ supported_worldadapters = ['TimeSeriesRunner'] + assets = { + 'js': "timeseries/timeseries.js", + 'template': 'timeseries/timeseries.tpl' + } + def __init__(self, filename, world_type="TimeSeries", name="", owner="", engine=None, uid=None, version=1, config={}): World.__init__(self, filename, world_type=world_type, name=name, owner=owner, uid=uid, version=version, config=config) + self.data['assets'] = self.assets + filename = config.get('time_series_data_file', "timeseries.npz") if os.path.isabs(filename): path = filename diff --git a/micropsi_server/static/css/micropsi-styles.css b/micropsi_server/static/css/micropsi-styles.css index cfcf39e0..b32556e4 100644 --- a/micropsi_server/static/css/micropsi-styles.css +++ b/micropsi_server/static/css/micropsi-styles.css @@ -663,4 +663,14 @@ p.clear { color: #ffffff !important; text-decoration: none; background-color: #0081c2 !important; +} + +#timeseries_controls label{ + padding-top: 2em; +} +#timeseries_controls .slider.slider-horizontal { + width: 100% !important; +} +#timeseries_controls .tooltip.top { + opacity: 0.8; } \ No newline at end of file diff --git a/micropsi_server/static/timeseries/timeseries.js b/micropsi_server/static/timeseries/timeseries.js new file mode 100644 index 00000000..22a4732e --- /dev/null +++ b/micropsi_server/static/timeseries/timeseries.js @@ -0,0 +1,55 @@ + +$(function(){ + + var container = $('#timeseries_controls'); + + var slider = $('#timeseries_slider'); + + var initialized = false; + + var first, last, total; + + function get_world_data(){ + return {step: currentWorldSimulationStep}; + } + + function set_world_data(data){ + if(!initialized){ + first = new Date(data['first_timestamp']); + last = new Date(data['last_timestamp']); + total = data['total_timestamps']; + slider.slider({ + 'min': 0, + 'max': data['total_timestamps'], + 'width': '100%', + 'step': 1, + 'value': data['current_step'], + 'tooltip': 'show', + 'handle': 'triangle', + 'selection': 'none', + 'formater': function(index){ + if (index > 0){ + var interval = parseInt((last.getTime() - first.getTime()) / total); + return new Date(first.getTime() + (interval * index)).toLocaleString('de'); + } else { + return first.toLocaleString('de'); + } + } + + }); + $('.firstval', container).text(first.toLocaleString('de')); + $('.lastval', container).text(last.toLocaleString('de')); + initialized = true; + slider.on('slideStop', set_world_state); + } + slider.slider('setValue', data['current_step']); + } + + function set_world_state(event){ + console.log(event); + } + + register_stepping_function('world', get_world_data, set_world_data); + + api.call('get_world_view', {'world_uid': currentWorld, 'step': 0}, set_world_data); +}); \ No newline at end of file diff --git a/micropsi_server/static/timeseries/timeseries.tpl b/micropsi_server/static/timeseries/timeseries.tpl new file mode 100644 index 00000000..5253d646 --- /dev/null +++ b/micropsi_server/static/timeseries/timeseries.tpl @@ -0,0 +1,11 @@ + + +
        +
        +
        +
        +
        +
        +
        +
        +
        \ No newline at end of file diff --git a/micropsi_server/view/boilerplate.tpl b/micropsi_server/view/boilerplate.tpl index fa05cffb..27e1915b 100644 --- a/micropsi_server/view/boilerplate.tpl +++ b/micropsi_server/view/boilerplate.tpl @@ -23,6 +23,7 @@ + @@ -37,6 +38,7 @@ + From 1c5b8ba42a92bddd9760236159ef75e2baeb4c12 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 13 Apr 2016 16:41:51 +0200 Subject: [PATCH 280/306] api to set world data from frontend --- micropsi_core/_runtime_api_world.py | 8 ++++++++ micropsi_core/world/timeseries/timeseries.py | 6 ++++++ micropsi_core/world/world.py | 5 +++++ micropsi_server/micropsi_app.py | 7 ++++++- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/micropsi_core/_runtime_api_world.py b/micropsi_core/_runtime_api_world.py index c2fb9fc2..c8a088ed 100644 --- a/micropsi_core/_runtime_api_world.py +++ b/micropsi_core/_runtime_api_world.py @@ -158,6 +158,14 @@ def set_world_properties(world_uid, world_name=None, owner=None): return True +def set_world_data(world_uid, data): + """ Sets some data for the world. Whatever the world supports""" + if world_uid not in micropsi_core.runtime.worlds: + raise KeyError("World not found") + micropsi_core.runtime.worlds[world_uid].set_user_data(data) + return True + + def revert_world(world_uid): """Reverts the world to the last saved state.""" data = micropsi_core.runtime.world_data[world_uid] diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index ca9e3786..17b30dbd 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -163,6 +163,12 @@ def get_config_options(): 'options': ["True", "False"]} ] + def set_user_data(self, data): + """ Allow the user to set the step of this world""" + if 'step' in data: + self.last_realtime_step = datetime.utcnow().timestamp() * 1000 + self.current_step = data['step'] + def get_world_view(self, step): return { 'first_timestamp': str(self.timestamps[0]), diff --git a/micropsi_core/world/world.py b/micropsi_core/world/world.py index c2e1b130..788a585f 100644 --- a/micropsi_core/world/world.py +++ b/micropsi_core/world/world.py @@ -321,6 +321,11 @@ def set_agent_properties(self, uid, position=None, orientation=None, name=None, return True return False + def set_user_data(self, data): + """ Sets some data from the user. Implement this in your worldclass to allow + the user to set certain properties of this world""" + pass + def __del__(self): """Empty destructor""" pass diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index 628ba79a..f91c215f 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -957,10 +957,15 @@ def get_world_view(world_uid, step): @rpc("set_world_properties", permission_required="manage worlds") -def set_world_data(world_uid, world_name=None, owner=None): +def set_world_properties(world_uid, world_name=None, owner=None): return runtime.set_world_properties(world_uid, world_name, owner) +@rpc("set_world_data") +def set_world_data(world_uid, data): + return runtime.set_world_data(world_uid, data) + + @rpc("revert_world", permission_required="manage worlds") def revert_world(world_uid): return runtime.revert_world(world_uid) From eaf58419deb6b5e1ada77826fa62f3c5fed17158 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 13 Apr 2016 16:47:32 +0200 Subject: [PATCH 281/306] set world step on slider change --- micropsi_server/static/timeseries/timeseries.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/micropsi_server/static/timeseries/timeseries.js b/micropsi_server/static/timeseries/timeseries.js index 22a4732e..c88e7ee2 100644 --- a/micropsi_server/static/timeseries/timeseries.js +++ b/micropsi_server/static/timeseries/timeseries.js @@ -20,7 +20,7 @@ $(function(){ total = data['total_timestamps']; slider.slider({ 'min': 0, - 'max': data['total_timestamps'], + 'max': total - 1, 'width': '100%', 'step': 1, 'value': data['current_step'], @@ -37,8 +37,8 @@ $(function(){ } }); - $('.firstval', container).text(first.toLocaleString('de')); - $('.lastval', container).text(last.toLocaleString('de')); + $('.firstval', container).html(first.toLocaleString('de').replace(', ', '
        ')); + $('.lastval', container).html(last.toLocaleString('de').replace(', ', '
        ')); initialized = true; slider.on('slideStop', set_world_state); } @@ -46,7 +46,10 @@ $(function(){ } function set_world_state(event){ - console.log(event); + var value = parseInt(slider.val()); + api.call('set_world_data', {world_uid: currentWorld, data: {step: value}}, function(){ + $(document).trigger('runner_stepped'); + }); } register_stepping_function('world', get_world_data, set_world_data); From ad97fd1f107d9754e9f81d7f1c7b5b131f45988b Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 13 Apr 2016 17:52:52 +0200 Subject: [PATCH 282/306] change step_nodenet to allow for nodenet_only stepping --- micropsi_core/runtime.py | 11 ++++++----- micropsi_server/micropsi_app.py | 5 +++++ micropsi_server/tests/test_json_api.py | 12 ++++++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/micropsi_core/runtime.py b/micropsi_core/runtime.py index d7ca5f37..1f599406 100755 --- a/micropsi_core/runtime.py +++ b/micropsi_core/runtime.py @@ -626,17 +626,18 @@ def stop_nodenetrunner(nodenet_uid): return True -def step_nodenet(nodenet_uid): +def step_nodenet(nodenet_uid, nodenet_only=False, amount=1): """Advances the given nodenet by one calculation step. Arguments: nodenet_uid: The uid of the nodenet """ nodenet = get_nodenet(nodenet_uid) - nodenet.timed_step() - nodenet.update_monitors() - if nodenet.world and nodenet.current_step % configs['runner_factor'] == 0: - worlds[nodenet.world].step() + for i in range(amount): + nodenet.timed_step() + nodenet.update_monitors() + if not nodenet_only and nodenet.world and nodenet.current_step % configs['runner_factor'] == 0: + worlds[nodenet.world].step() return nodenet.current_step diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index f91c215f..d84d835f 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -844,6 +844,11 @@ def step_calculation(nodenet_uid): return True, runtime.step_nodenet(nodenet_uid) +@rpc("step_nodenet", permission_required="manage nodenets") +def step_nodenet(nodenet_uid, amount=1): + return True, runtime.step_nodenet(nodenet_uid, True, amount) + + @rpc("revert_calculation", permission_required="manage nodenets") def revert_calculation(nodenet_uid): return runtime.revert_nodenet(nodenet_uid, True) diff --git a/micropsi_server/tests/test_json_api.py b/micropsi_server/tests/test_json_api.py index aeb632e8..845a20d4 100644 --- a/micropsi_server/tests/test_json_api.py +++ b/micropsi_server/tests/test_json_api.py @@ -217,6 +217,18 @@ def test_step_calculation(app, test_nodenet): assert response.json_body['data']['current_step'] == 1 +def test_step_nodenet(app, test_nodenet, test_world): + app.set_auth() + app.post_json('/rpc/set_nodenet_properties', params=dict(nodenet_uid=test_nodenet, nodenet_name="new_name", worldadapter="Braitenberg", world_uid=test_world)) + app.post_json('/rpc/step_nodenet', {'nodenet_uid': test_nodenet}) + from micropsi_core import runtime + assert runtime.nodenets[test_nodenet].current_step == 1 + assert runtime.worlds[test_world].current_step == 0 + app.post_json('/rpc/step_nodenet', {'nodenet_uid': test_nodenet, 'amount': 10}) + assert runtime.nodenets[test_nodenet].current_step == 11 + assert runtime.worlds[test_world].current_step == 0 + + def test_get_calculation_state(app, test_nodenet, test_world, node): from time import sleep app.set_auth() From 85391531f136908616be3acb5f6e6e4627425ab9 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 13 Apr 2016 17:53:17 +0200 Subject: [PATCH 283/306] update agents after setting world step --- micropsi_core/world/timeseries/timeseries.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/micropsi_core/world/timeseries/timeseries.py b/micropsi_core/world/timeseries/timeseries.py index 17b30dbd..1090c379 100644 --- a/micropsi_core/world/timeseries/timeseries.py +++ b/micropsi_core/world/timeseries/timeseries.py @@ -168,6 +168,9 @@ def set_user_data(self, data): if 'step' in data: self.last_realtime_step = datetime.utcnow().timestamp() * 1000 self.current_step = data['step'] + for uid in self.agents: + with self.agents[uid].datasource_lock: + self.agents[uid].update() def get_world_view(self, step): return { From 15a8b5a94ebae650dd789a42adfd9b5fbac1e1f9 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Wed, 13 Apr 2016 17:53:28 +0200 Subject: [PATCH 284/306] frontend for choosing whether to step nodenet after timestamp change --- micropsi_server/static/css/micropsi-styles.css | 3 --- micropsi_server/static/timeseries/timeseries.js | 13 ++++++++++++- micropsi_server/static/timeseries/timeseries.tpl | 13 +++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/micropsi_server/static/css/micropsi-styles.css b/micropsi_server/static/css/micropsi-styles.css index b32556e4..e2b5c4c0 100644 --- a/micropsi_server/static/css/micropsi-styles.css +++ b/micropsi_server/static/css/micropsi-styles.css @@ -665,9 +665,6 @@ p.clear { background-color: #0081c2 !important; } -#timeseries_controls label{ - padding-top: 2em; -} #timeseries_controls .slider.slider-horizontal { width: 100% !important; } diff --git a/micropsi_server/static/timeseries/timeseries.js b/micropsi_server/static/timeseries/timeseries.js index c88e7ee2..803e1794 100644 --- a/micropsi_server/static/timeseries/timeseries.js +++ b/micropsi_server/static/timeseries/timeseries.js @@ -9,10 +9,15 @@ $(function(){ var first, last, total; + var advance_nodenet = $('#timeseries_controls_nodenet'); + var nodenet_amount = $('#timeseries_controls_nodenet_amount') + function get_world_data(){ return {step: currentWorldSimulationStep}; } + $('.section.world .editor_field').height('auto'); + function set_world_data(data){ if(!initialized){ first = new Date(data['first_timestamp']); @@ -48,7 +53,13 @@ $(function(){ function set_world_state(event){ var value = parseInt(slider.val()); api.call('set_world_data', {world_uid: currentWorld, data: {step: value}}, function(){ - $(document).trigger('runner_stepped'); + if(advance_nodenet.attr('checked')){ + api.call('step_nodenet', {nodenet_uid: currentNodenet, amount: parseInt(nodenet_amount.val())}, function(){ + $(document).trigger('runner_stepped'); + }); + } else { + $(document).trigger('runner_stepped'); + } }); } diff --git a/micropsi_server/static/timeseries/timeseries.tpl b/micropsi_server/static/timeseries/timeseries.tpl index 5253d646..e5d3d89c 100644 --- a/micropsi_server/static/timeseries/timeseries.tpl +++ b/micropsi_server/static/timeseries/timeseries.tpl @@ -7,5 +7,18 @@
        +
        +
        +
        +

        Drag the slider to change the current point in time of the timeseries

        +

        + +

        + +
        +
        + \ No newline at end of file From 3977d8b643a1ffe3f3ff3b90e6c81f02d24a9552 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 14 Apr 2016 12:12:13 +0200 Subject: [PATCH 285/306] deactivate netapi console if not serving to localhost only --- micropsi_server/micropsi_app.py | 11 ++++++++--- micropsi_server/view/nodenet.tpl | 5 +++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/micropsi_server/micropsi_app.py b/micropsi_server/micropsi_app.py index 628ba79a..7573f473 100755 --- a/micropsi_server/micropsi_app.py +++ b/micropsi_server/micropsi_app.py @@ -33,6 +33,8 @@ VERSION = cfg['micropsi2']['version'] APPTITLE = cfg['micropsi2']['apptitle'] +INCLUDE_CONSOLE = cfg['micropsi2']['host'] == 'localhost' + APP_PATH = os.path.dirname(__file__) micropsi_app = Bottle() @@ -170,13 +172,13 @@ def server_static(filepath): def index(): first_user = usermanager.users == {} user_id, permissions, token = get_request_data() - return _add_world_list("viewer", mode="all", first_user=first_user, logging_levels=runtime.get_logging_levels(), version=VERSION, user_id=user_id, permissions=permissions) + return _add_world_list("viewer", mode="all", first_user=first_user, logging_levels=runtime.get_logging_levels(), version=VERSION, user_id=user_id, permissions=permissions, console=INCLUDE_CONSOLE) @micropsi_app.route("/nodenet") def nodenet(): user_id, permissions, token = get_request_data() - return template("viewer", mode="nodenet", version=VERSION, user_id=user_id, permissions=permissions) + return template("viewer", mode="nodenet", version=VERSION, user_id=user_id, permissions=permissions, console=INCLUDE_CONSOLE) @micropsi_app.route("/monitors") @@ -1246,7 +1248,10 @@ def get_agent_dashboard(nodenet_uid): @rpc("run_netapi_command", permission_required="manage nodenets") def run_netapi_command(nodenet_uid, command): - return runtime.run_netapi_command(nodenet_uid, command) + if INCLUDE_CONSOLE: + return runtime.run_netapi_command(nodenet_uid, command) + else: + raise RuntimeError("Netapi console only available if serving to localhost only") @rpc("get_netapi_signatures") diff --git a/micropsi_server/view/nodenet.tpl b/micropsi_server/view/nodenet.tpl index 50b133f8..83c25644 100644 --- a/micropsi_server/view/nodenet.tpl +++ b/micropsi_server/view/nodenet.tpl @@ -253,8 +253,9 @@
        - -% include netapi_console +% if console: + % include netapi_console +% end % if console: - % include netapi_console + % include("netapi_console") % end @@ -102,5 +105,7 @@ $('#world_type').on('change', function(event){ var val = $(event.target).val(); $('.world_config').hide(); $('.world_config_'+val).show(); + $('.world_docstring').hide(); + $('.world_docstring_'+val).show(); }) \ No newline at end of file From fab1dde408ad5b0f9bbf32962ee2f0340e1daca0 Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Thu, 21 Apr 2016 22:35:36 +0200 Subject: [PATCH 301/306] fail silently if dependencies not installed --- micropsi_core/world/world.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/micropsi_core/world/world.py b/micropsi_core/world/world.py index 788a585f..39503e3d 100644 --- a/micropsi_core/world/world.py +++ b/micropsi_core/world/world.py @@ -354,4 +354,8 @@ def __del__(self): try: from micropsi_core.world.timeseries import timeseries except ImportError as e: - sys.stdout.write("Could not import timeseries world.\nError: %s \n\n" % e.msg) + if e.msg == "No module named 'numpy'": + # ignore silently + pass + else: + sys.stdout.write("Could not import timeseries world.\nError: %s \n\n" % e.msg) From c88621428c5f11837dfa1c71e159ee4770cd5e9d Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 22 Apr 2016 13:51:15 +0200 Subject: [PATCH 302/306] fix nodespace properties for new nodespaces --- micropsi_server/static/js/nodenet.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index f27ca957..52d61308 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -104,7 +104,10 @@ if (nodenetcookie && nodenetcookie.indexOf('/') > 0){ nodespaceProperties = {}; // compatibility -renderlinks_default = $.cookie('renderlinks') || 'always'; +nodespace_property_defaults = { + 'renderlinks': ($.cookie('renderlinks') || 'always'), + 'activation_display': 'redgreen' +} currentWorldadapter = null; @@ -277,10 +280,10 @@ function setCurrentNodenet(uid, nodespace, changed){ nodespaceProperties[key] = {}; } if(!nodespaceProperties[key].renderlinks){ - nodespaceProperties[key].renderlinks = renderlinks_default; + nodespaceProperties[key].renderlinks = nodespace_property_defaults.renderlinks; } if(!nodespaceProperties[key].activation_display){ - nodespaceProperties[key].activation_display = 'redgreen'; + nodespaceProperties[key].activation_display = nodespace_property_defaults.activation_display; } } if(nodenetChanged){ @@ -617,8 +620,11 @@ function refreshNodespace(nodespace, step, callback){ params = { nodenet_uid: currentNodenet, nodespaces: [nodespace], + include_links: true }; - params.include_links = nodespaceProperties[nodespace].renderlinks == 'always'; + if(nodespaceProperties[nodespace] && nodespaceProperties[nodespace].renderlinks != 'always'){ + params.include_links = false; + } api.call('get_nodes', params , success=function(data){ var changed = nodespace != currentNodeSpace; if(changed){ @@ -3160,6 +3166,9 @@ function createNodeHandler(x, y, name, type, parameters, callback) { } api.call(method, params, success=function(uid){ + if(type == 'Nodespace'){ + nodespaceProperties[uid] = nodespace_property_defaults + } addNode(new Node(uid, x, y, currentNodeSpace, name || '', type, null, null, parameters)); view.draw(); selectNode(uid); From 5f4352b69172eb191cafe480c0805e05154e559c Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 22 Apr 2016 13:54:19 +0200 Subject: [PATCH 303/306] prune label if no emotional modulators present --- micropsi_server/static/js/nodenet.js | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/micropsi_server/static/js/nodenet.js b/micropsi_server/static/js/nodenet.js index 52d61308..53cd1ee1 100644 --- a/micropsi_server/static/js/nodenet.js +++ b/micropsi_server/static/js/nodenet.js @@ -109,6 +109,7 @@ nodespace_property_defaults = { 'activation_display': 'redgreen' } + currentWorldadapter = null; var currentSheaf = "default"; @@ -3108,11 +3109,13 @@ function get_datasource_options(worldadapter, value){ } html += ''; } - html += ''; - for(var i in globalDataSources){ - html += ''; + if(globalDataSources.length){ + html += ''; + for(var i in globalDataSources){ + html += ''; + } + html += ''; } - html += ''; return html; } @@ -3129,11 +3132,13 @@ function get_datatarget_options(worldadapter, value){ } html += ''; } - html += ''; - for(var i in globalDataTargets){ - html += ''; + if(globalDataTargets.length){ + html += ''; + for(var i in globalDataTargets){ + html += ''; + } + html += ''; } - html += ''; return html; } From 97599503a82c48a36737774e011a2abd4692e35a Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 22 Apr 2016 14:13:05 +0200 Subject: [PATCH 304/306] use http to avoid certificate error no way to get rid of this in an iframe --- micropsi_server/static/minecraft/minecraft.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropsi_server/static/minecraft/minecraft.js b/micropsi_server/static/minecraft/minecraft.js index 01375f5f..1bbf4e8b 100644 --- a/micropsi_server/static/minecraft/minecraft.js +++ b/micropsi_server/static/minecraft/minecraft.js @@ -88,7 +88,7 @@ function setCurrentWorld(uid) { world_uid: currentWorld }, success = function (data) { refreshWorldView(); - $('#world').parent().html(''); + $('#world').parent().html(''); }, error = function (data) { $.cookie('selected_world', '', { expires: -1, From 7fca1f99f3d1e88be393105e838c81d739a1c1fa Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 22 Apr 2016 14:17:59 +0200 Subject: [PATCH 305/306] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d4e93fb..8b75a505 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ * Nodenet/native module data structure changed * Faster sensors for theano * Configurable worlds + * New timeseries world + * Netapi console if serving for localhost only 0.7-alpha5 (2016-02-04) From 082d7be0bf919c442aa3e4d722b964994413cafb Mon Sep 17 00:00:00 2001 From: Dominik Welland Date: Fri, 22 Apr 2016 14:18:10 +0200 Subject: [PATCH 306/306] release 0.8-alpha6 --- CHANGELOG.md | 2 +- configuration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b75a505..61248979 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ -0.8-alpha6 (unreleased) +0.8-alpha6 (2016-04-22) ========== * Operations for selections of nodes/nodespaces diff --git a/configuration.py b/configuration.py index 9648565d..6ff5d240 100644 --- a/configuration.py +++ b/configuration.py @@ -24,7 +24,7 @@ warnings.warn('Can not read config from inifile %s' % filename) raise RuntimeError('Can not read config from inifile %s' % filename) -config['micropsi2']['version'] = "0.8-alpha6-dev" +config['micropsi2']['version'] = "0.8-alpha6" config['micropsi2']['apptitle'] = "MicroPsi" homedir = config['micropsi2']['data_directory'].startswith('~')