Merge remote-tracking branch 'mi/release'

.coveragerc

@@ -1,4 +1,5 @@
 [run]
 omit =
     */tests/*
-    micropsi_server/bottle.py
+    micropsi_server/bottle.py
+    */conftest.py

.gitignore

@@ -25,3 +25,4 @@
 /*.sublime-workspace
 micropsi.log
 /test-data/
+pip-selfcheck.json

CHANGELOG.md

@@ -1,4 +1,17 @@
 
+0.8-alpha6 (2016-04-22)
+==========
+
+ * Operations for selections of nodes/nodespaces
+ * Test infrastructure for nodenets and native modules
+ * Gradient descent native modules built-in
+ * Nodenet/native module data structure changed
+ * Faster sensors for theano
+ * Configurable worlds
+ * New timeseries world
+ * Netapi console if serving for localhost only
+
+
 0.7-alpha5 (2016-02-04)
 ==========
 

Makefile

@@ -17,9 +17,17 @@ clean:
 	rm -rf include lib .Python bin
 
 tests:
-	bin/py.test
+	-bin/py.test
+	-bin/py.test --agents
 
 test-coverage:
 	bin/py.test --cov micropsi_core --cov micropsi_server --cov-report html
 
-.PHONY: run
+test-toolkit:
+	bin/py.test
+
+test-agents:
+	bin/py.test --agents
+
+
+.PHONY: run

README.md

@@ -7,7 +7,7 @@ For more information on the cognitive architecture visit [micropsi.com](http://w
 
 Prerequisites
 -----
-* Python3
+* Python3 (tested with 3.4.3 and 3.5.1)
 * On Windows, we recommend downloading and installing [WinPython 3.4.3.7](http://winpython.github.io/)
 
 

README_additional_configuration.md

@@ -37,7 +37,7 @@ Windows support for advanced features is experimental.
 * From the mingw Folder, add the `bin` Folder to your PATH environment variable
 * Install pycrypto for python3.4. Get one of the [pycrypto windows binaries](https://github.com/axper/python3-pycrypto-windows-installer), open the WinPython Control Panel, click "Add Packages", select the downloaded pycrypto installer, and click "Install packages"
 * now you can install our modified spock via
-`pip install -e -e git+https://github.com/micropsi-industries/spock.git#egg=spock-dev`
+`pip install -e git+https://github.com/micropsi-industries/spock.git#egg=spock-dev`
 * this should lead to a working MicroPsi with Theano and minecraft support.
 * install the optional packages with `pip install cherrypy pytest mock webtest`
 * run `python start_micropsi_server.py`

configuration.py

@@ -24,7 +24,7 @@
     warnings.warn('Can not read config from inifile %s' % filename)
     raise RuntimeError('Can not read config from inifile %s' % filename)
 
-config['micropsi2']['version'] = "0.7-alpha5"
+config['micropsi2']['version'] = "0.8-alpha6"
 config['micropsi2']['apptitle'] = "MicroPsi"
 
 homedir = config['micropsi2']['data_directory'].startswith('~')
@@ -43,6 +43,6 @@
         config['logging'][level] = 'WARNING'
 
 config.add_section('paths')
-config['paths']['resource_path'] = os.path.join(os.path.dirname(__file__), data_path)
+config['paths']['data_directory'] = os.path.join(os.path.dirname(__file__), data_path)
 config['paths']['usermanager_path'] = os.path.join(os.path.dirname(__file__), 'resources', 'user-db.json')
 config['paths']['server_settings_path'] = os.path.join(os.path.dirname(__file__), 'resources', 'server-config.json')

conftest.py

@@ -4,59 +4,63 @@
 import pytest
 import logging
 
+try:
+    import theano
+    engine_defaults = "dict_engine,theano_engine"
+except:
+    engine_defaults = "dict_engine"
 
-testpath = os.path.join('.', 'test-data')
 
+testpath = os.path.abspath(os.path.join('.', 'test-data'))
 try:
     shutil.rmtree(testpath)
 except OSError:
     pass
 
-# override config
-from configuration import config
-config['paths']['resource_path'] = testpath
-config['paths']['server_settings_path'] = os.path.join(config['paths']['resource_path'], 'server_config.json')
-config['paths']['usermanager_path'] = os.path.join(config['paths']['resource_path'], 'user-db.json')
-config['micropsi2']['single_agent_mode'] = ''
-if 'theano' in config:
-    config['theano']['initial_number_of_nodes'] = '50'
-
 from micropsi_core import runtime as micropsi
+from micropsi_core.runtime import cfg
+original_ini_data_directory = cfg['paths']['data_directory']
 
-# create testuser
-from micropsi_server.micropsi_app import usermanager
+cfg['paths']['data_directory'] = testpath
+cfg['paths']['server_settings_path'] = os.path.join(testpath, 'server_cfg.json')
+cfg['paths']['usermanager_path'] = os.path.join(testpath, 'user-db.json')
+cfg['micropsi2']['single_agent_mode'] = ''
+if 'theano' in cfg:
+    cfg['theano']['initial_number_of_nodes'] = '50'
 
-usermanager.create_user('Pytest User', 'test', 'Administrator', uid='Pytest User')
-user_token = usermanager.start_session('Pytest User', 'test', True)
-
-# reset logging levels
-logging.getLogger('system').setLevel(logging.WARNING)
-logging.getLogger('world').setLevel(logging.WARNING)
 
 world_uid = 'WorldOfPain'
 nn_uid = 'Testnet'
 
-nodetype_file = os.path.join(config['paths']['resource_path'], 'nodetypes.json')
-nodefunc_file = os.path.join(config['paths']['resource_path'], 'nodefunctions.py')
-recipes_file = os.path.join(config['paths']['resource_path'], 'recipes.py')
 
+def pytest_addoption(parser):
+    """register argparse-style options and ini-style config values."""
+    parser.addoption("--engine", action="store", default=engine_defaults,
+        help="The engine that should be used for this testrun.")
+    parser.addoption("--agents", action="store_true",
+        help="Only test agents-code from the data_directory")
 
-try:
-    import theano
-    engine_defaults = "dict_engine,theano_engine"
-except:
-    engine_defaults = "dict_engine"
 
+def pytest_cmdline_main(config):
+    """ called for performing the main command line action. The default
+    implementation will invoke the configure hooks and runtest_mainloop. """
+    if config.getoption('agents'):
+        config.args = [original_ini_data_directory]
+        micropsi.initialize(persistency_path=testpath, resource_path=original_ini_data_directory)
+    else:
+        micropsi.initialize(persistency_path=testpath)
+        from micropsi_server.micropsi_app import usermanager
 
-def set_logging_levels():
-    logging.getLogger('system').setLevel(logging.WARNING)
-    logging.getLogger('world').setLevel(logging.WARNING)
-    micropsi.cfg['logging']['level_agent'] = 'WARNING'
+        usermanager.create_user('Pytest User', 'test', 'Administrator', uid='Pytest User')
+        usermanager.start_session('Pytest User', 'test', True)
 
+    set_logging_levels()
 
-def pytest_addoption(parser):
-    parser.addoption("--engine", action="store", default=engine_defaults,
-        help="The engine that should be used for this testrun.")
+
+def pytest_configure(config):
+    # register an additional marker
+    config.addinivalue_line("markers",
+        "engine(name): mark test to run only on the specified engine")
 
 
 def pytest_generate_tests(metafunc):
@@ -70,99 +74,102 @@ def pytest_generate_tests(metafunc):
         metafunc.parametrize("engine", engines, scope="session")
 
 
-def pytest_configure(config):
-    # register an additional marker
-    config.addinivalue_line("markers",
-        "engine(name): mark test to run only on the specified engine")
-
-
 def pytest_runtest_setup(item):
     engine_marker = item.get_marker("engine")
     if engine_marker is not None:
         engine_marker = engine_marker.args[0]
         if engine_marker != item.callspec.params['engine']:
             pytest.skip("test requires engine %s" % engine_marker)
+    for item in os.listdir(testpath):
+        if item != 'worlds' and item != 'nodenets':
+            path = os.path.join(testpath, item)
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            else:
+                os.remove(path)
+    os.mkdir(os.path.join(testpath, 'Test'))
+    open(os.path.join(testpath, 'Test', '__init__.py'), 'w').close()
+    micropsi.reload_native_modules()
     micropsi.logger.clear_logs()
+    micropsi.set_runner_properties(1, 1)
+    set_logging_levels()
 
 
-def pytest_runtest_teardown(item, nextitem):
-    if nextitem is None:
-        print("DELETING ALL STUFF")
-        shutil.rmtree(config['paths']['resource_path'])
-    else:
-        uids = list(micropsi.nodenets.keys())
-        for uid in uids:
-            micropsi.delete_nodenet(uid)
-        if os.path.isfile(nodetype_file):
-            os.remove(nodetype_file)
-        if os.path.isfile(nodefunc_file):
-            os.remove(nodefunc_file)
-        if os.path.isfile(recipes_file):
-            os.remove(recipes_file)
-        micropsi.reload_native_modules()
-        set_logging_levels()
+def pytest_internalerror(excrepr, excinfo):
+    """ called for internal errors. """
+    micropsi.kill_runners()
+    shutil.rmtree(testpath)
 
 
-@pytest.fixture(scope="session")
-def resourcepath():
-    return config['paths']['resource_path']
+def pytest_keyboard_interrupt(excinfo):
+    """ called for keyboard interrupt. """
+    micropsi.kill_runners()
+    shutil.rmtree(testpath)
 
 
-@pytest.fixture()
-def nodetype_def():
-    return nodetype_file
+def set_logging_levels():
+    """ sets the logging levels of the default loggers back to WARNING """
+    logging.getLogger('system').setLevel(logging.WARNING)
+    logging.getLogger('world').setLevel(logging.WARNING)
+    micropsi.cfg['logging']['level_agent'] = 'WARNING'
 
 
-@pytest.fixture
-def nodefunc_def():
-    return nodefunc_file
+@pytest.fixture(scope="session")
+def resourcepath():
+    """ Fixture: the resource path """
+    return micropsi.RESOURCE_PATH
 
 
-@pytest.fixture
-def recipes_def():
-    return recipes_file
+@pytest.fixture(scope="session")
+def runtime():
+    """ Fixture: The micropsi runtime """
+    return micropsi
 
 
-@pytest.fixture(scope="function")
+@pytest.yield_fixture(scope="function")
 def test_world(request):
+    """
+    Fixture: A test world of type Island
+    """
     global world_uid
-    worlds = micropsi.get_available_worlds("Pytest User")
-    if world_uid not in worlds:
-        success, world_uid = micropsi.new_world("World of Pain", "Island", "Pytest User", uid=world_uid)
-
-    def fin():
-        try:
-            micropsi.delete_world(world_uid)
-        except:
-            pass  # world was deleted in test
-    request.addfinalizer(fin)
-    return world_uid
+    success, world_uid = micropsi.new_world("World of Pain", "Island", "Pytest User", uid=world_uid)
+    yield world_uid
+    try:
+        micropsi.delete_world(world_uid)
+    except:
+        pass
 
 
 @pytest.fixture(scope="function")
+def default_world(request):
+    """
+    Fixture: A test world of type Island
+    """
+    for uid in micropsi.worlds:
+        if micropsi.worlds[uid].data['world_type'] == 'World':
+            return uid
+
+
+@pytest.yield_fixture(scope="function")
 def test_nodenet(request, test_world, engine):
+    """
+    Fixture: A completely empty nodenet without a worldadapter
+    """
     global nn_uid
-    nodenets = micropsi.get_available_nodenets("Pytest User") or {}
-    if nn_uid not in nodenets:
-        success, nn_uid = micropsi.new_nodenet("Testnet", engine=engine, owner="Pytest User", uid='Testnet')
-        micropsi.save_nodenet(nn_uid)
-    if nn_uid not in micropsi.nodenets:
-        micropsi.load_nodenet(nn_uid)
-    return nn_uid
+    success, nn_uid = micropsi.new_nodenet("Testnet", engine=engine, owner="Pytest User", uid='Testnet')
+    micropsi.save_nodenet(nn_uid)
+    yield nn_uid
+    try:
+        micropsi.delete_nodenet(nn_uid)
+    except:
+        pass
 
 
 @pytest.fixture(scope="function")
 def node(request, test_nodenet):
+    """
+    Fixture: A Pipe node with a genloop
+    """
     res, uid = micropsi.add_node(test_nodenet, 'Pipe', [10, 10, 10], name='N1')
     micropsi.add_link(test_nodenet, uid, 'gen', uid, 'gen')
     return uid
-
-
-def pytest_internalerror(excrepr, excinfo):
-    """ called for internal errors. """
-    shutil.rmtree(config['paths']['resource_path'])
-
-
-def pytest_keyboard_interrupt(excinfo):
-    """ called for keyboard interrupt. """
-    shutil.rmtree(config['paths']['resource_path'])