Merge pull request #691 from NextCenturyCorporation/development

MCS Merge development to master

.github/workflows/docs-publish.yaml

@@ -13,16 +13,18 @@ jobs:
       pull-requests: write
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
       with:
         fetch-depth: 0
     - name: Build and Commit
+      id: sphinxbuild
       uses: sphinx-notes/pages@v3
       with:
         documentation_path: docs/source
-    - name: Push changes
-      uses: ad-m/github-push-action@master
+        publish: false
+        requirements_path: ./requirements.txt
+    - uses: peaceiris/actions-gh-pages@v3
       with:
-        github_token: ${{ secrets.PLE_PAT_ACTIONS }}
-        branch: gh-pages
-        force: true
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ${{ steps.sphinxbuild.outputs.artifact }}
+        force_orphan: true

CPU_Container.dockerfile

@@ -9,7 +9,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 ENV TZ=America/New_York
 
 # --build-arg mcsversion=x.y.z to override default in docker build command
-ARG mcsversion=0.7.0
+ARG mcsversion=0.7.1
 ARG mcs_library_version=master
 
 WORKDIR /mcs

Dockerfile

@@ -19,7 +19,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display
 # ENV LANG C.UTF-8
 
 # --build-arg mcsversion=x.y.z to override default in docker build command
-ARG mcsversion=0.7.0
+ARG mcsversion=0.7.1
 ARG mcs_library_version=master
 
 WORKDIR /mcs

README.md

@@ -16,7 +16,7 @@ Publish to PyPI: [![Publish PyPI](https://github.com/NextCenturyCorporation/MCS/
 
 # MCS Python Package
 
-Python interface for interacting with MCS AI2Thor environment and running scenes. The latest release of the MCS Python library is `0.7.0`. You can find the latest documentation [here](https://nextcenturycorporation.github.io/MCS).
+Python interface for interacting with MCS AI2Thor environment and running scenes. The latest release of the MCS Python library is `0.7.1`. You can find the latest documentation [here](https://nextcenturycorporation.github.io/MCS).
 
 - [Quickstart Installation](#quickstart-installation)
 - [Usage](#usage)

docs/source/install.rst

@@ -2,8 +2,8 @@
 Installation and Setup
 =======================
 
-.. _Download and unzip the Mac ZIP: https://github.com/NextCenturyCorporation/MCS/releases/download/0.7.0/MCS-AI2-THOR-Unity-App-v0.7.0-mac.zip
-.. _Download and unzip the Linux ZIP: https://github.com/NextCenturyCorporation/MCS/releases/download/0.7.0/MCS-AI2-THOR-Unity-App-v0.7.0-linux.zip
+.. _Download and unzip the Mac ZIP: https://github.com/NextCenturyCorporation/MCS/releases/download/0.7.1/MCS-AI2-THOR-Unity-App-v0.7.1-mac.zip
+.. _Download and unzip the Linux ZIP: https://github.com/NextCenturyCorporation/MCS/releases/download/0.7.1/MCS-AI2-THOR-Unity-App-v0.7.1-linux.zip
 
 Virtual Environments
 ------------------------

integration_tests/README.md

@@ -6,6 +6,12 @@ The goal of the integration tests is to:
 
 ## Run Handmade Tests
 
+```
+python run_handmade_tests.py
+```
+
+### Run Specific Unity Build
+
 ```
 python run_handmade_tests.py --mcs_unity_build_file_path <mcs_unity_build_file_path>
 ```
@@ -26,12 +32,16 @@ Replace `001` with any test scene.
 
 ## Run Prefab Tests
 
-TODO MCS-432
+Note that this test requires an internet connection to run.
 
-## Run All Tests
+```
+python run_prefab_tests.py
+```
+
+### Run on Development Branch
 
 ```
-python run_tests.py --mcs_unity_build_file_path <mcs_unity_build_file_path>
+python run_prefab_tests.py --mcs_unity_version development
 ```
 
 ## Run the Autofixer
@@ -50,12 +60,98 @@ Create the following files (Replace `<X>` with your test name):
 
 - `./data/<X>.scene.json`: An MCS scene configuration JSON file. Give it a useful `name` that describes the specific test case for future developers.
 - `./data/<X>.actions.txt`: The list of specific actions to take in the scene, each action and its corresponding parameters on its own line, with the following format: `action<,parameter=value>\n`
-- `./data/<X>.<level1|level2|oracle>.outputs.json`: Each file is a JSON list of expected step and object output metadata per action step, starting at step 0 (initialization), for runs with the file's corresponding metadata tier. Please do not skip any step. See the Python code for the available JSON test properties. One separate file per metadata tier, and each metadata tier should have its own corresponding file.
+- `./data/<X>.<level1|level2|oracle>.outputs.json`: Each file is a JSON list of expected step and object output metadata per action step, starting at step 0 (initialization), for runs with the file's corresponding metadata tier. Please do not skip any step. See the [Outputs File Schema](#Outputs-File-Schema) below for the available JSON test properties. One separate file per metadata tier, and each metadata tier should have its own corresponding file.
+- (Optionally) `./data/<X>.<level1|level2|oracle>.config.ini`: MCS config files to use for running this specific scene at each metadata level, overriding the default config files in `integration_tests/`.
 
 While adding a new test scene, you can run with the `--dev` flag to run the whole scene and print the errors at each action step (by default, a test will stop immediately if it finds an error).
 
-### Future Handmade Test Scene Ideas
+### Outputs File Schema
+
+#### Example
+
+```json
+[{
+    "step_number": 0,
+    "head_tilt": 0.0,
+    "objects_count": 1,
+    "position_x": 0.0,
+    "position_y": 0.0,
+    "position_z": 0.0,
+    "return_status": "SUCCESSFUL",
+    "rotation_y": 0.0,
+    "structural_objects_count": 5,
+    "objects": [{
+        "id": "testObject",
+        "held": true,
+        "position_x": 0.0,
+        "position_y": 0.0,
+        "position_z": 1.0,
+        "rotation_x": 0.0,
+        "rotation_y": 0.0,
+        "rotation_z": 1.0,
+        "shape": "ball",
+        "texture_color_list": ["blue"],
+        "visible": true
+    }]
+}]
+```
 
-- PickupObject on a container with an object positioned inside of it (MCS-473)
-- Circumnavigating objects (MCS-541)
-- PickupObject on a "pickupable" object that is too large or heavy (when needed by an evaluation task)
+#### Notes
+
+- Please have validation for each step.
+- If you do not include checks for properties, it does not mean those properties are null, it just means you are not checking them in the particular test. This is also true for objects: if you put an empty array for `objects`, it does not mean there are no objects, it just means you are not checking any object properties (so use `objects_count: 0` instead).
+- Any numerical property can be an array instead of a single number. The array must contain two numbers. The integration tests will use the numbers in the array as minimum and maximum values for validating the property at that step.
+- All numerical properties will be rounded to the hundredths place.
+
+#### Step Validation
+
+- `action_list`: Expected action restrictions at this step, as an array of nested [action, parameters] array 
+- `camera_height`: Expected camera height
+- `haptic_feedback`: Expected haptick feedback
+- `head_tilt`: Expected head tilt
+- `holes`: Expected holes, as an array of nested [X, Z] arrays
+- `lava`: Expected lava, as an array of nested [X, Z] arrays
+- `objects`: Validation of specific objects; see [Object Validation](#Object-Validation) below. Note that each element in this array is required to have an `id` corresponding to the `id` of an object in the scene file.
+- `objects_count`: Expected number of objects
+- `physics_frames_per_second`: Expected physics frames per second
+- `position_x`: Expected performer agent X position
+- `position_y`: Expected performer agent Y position
+- `position_z`: Expected performer agent Z position
+- `return_status`: Expected action return status
+- `resolved_object`: Expected resolved object
+- `resolved_receptacle`: Expected resolved receptacle object
+- `reward`: Expected reward
+- `rotation_y`: Expected performer agent Y rotation
+- `room_dimensions`: Expected room dimensions
+- `step_number`: Expected step number
+- `structural_objects`: Validation of specific structural objects; see [Object Validation](#Object-Validation) below. Note that each element in this array is required to have an `id` corresponding to the `id` of an object in the scene file.
+- `structural_objects_count`: Expected number of structural objects
+- `triggered_by_sequence_incorrect`: Expected "triggered_by_sequence_incorrect"
+
+#### Object Validation
+
+- `id` (Required): The "id" of the corresponding object in the scene file
+- `associated_with_agent`: Whether this object starts held by a simulation-controlled agent
+- `direction_x`: Expected X direction from the performer agent to this object
+- `direction_y`: Expected Y direction from the performer agent to this object
+- `direction_z`: Expected Z direction from the performer agent to this object
+- `distance`: Expected distance between the performer agent and this object
+- `held`: Whether this object is being held
+- `is_open`: Whether this object is currently open
+- `locked`: Whether this object is currently locked
+- `mass`: Expected object mass
+- `material_list`: Expected object salient material list
+- `position_x`: Expected object X position
+- `position_y`: Expected object Y position
+- `position_z`: Expected object Z position
+- `rotation_x`: Expected object X rotation
+- `rotation_y`: Expected object Y rotation
+- `rotation_z`: Expected object Z rotation
+- `shape`: Expected object shape string (not to be confused with its "type" string)
+- `simulation_agent_held_object`: Expected object this simulation-controlled agent starts holding
+- `simulation_agent_is_holding_held_object`: Whether this simulation-controlled agent is still holding its object
+- `state_list`: Expected object "states" at this step
+- `texture_color_list`: Expected object texture colors
+- `visible`: Whether this object is currently visible
+
+Copyright 2023 CACI (formerly Next Century Corporation)

integration_tests/additional_integration_tests.py

@@ -4,13 +4,29 @@
 import os.path
 
 import numpy as np
+from integration_test_utils import DEFAULT_TEST_CONFIGS
 
 import machine_common_sense as mcs
 
 INTEGRATION_TESTS_FOLDER = os.path.dirname(os.path.abspath(__file__))
+
 DEPTH_AND_SEGMENTATION_SCENE = (
     f'{INTEGRATION_TESTS_FOLDER}/depth_and_segmentation.scene.json'
 )
+DEPTH_AND_SEGMENTATION_TEST_CONFIGS = {
+    'level1': {
+        'history_enabled': False,
+        'metadata': 'level1'
+    },
+    'level2': {
+        'history_enabled': False,
+        'metadata': 'level2'
+    },
+    'oracle': {
+        'history_enabled': False,
+        'metadata': 'oracle'
+    },
+}
 
 DEPTH_DATA = (
     f'{INTEGRATION_TESTS_FOLDER}/depth_map.outputs.json'
@@ -472,9 +488,9 @@ def run_restricted_action_list_test(controller, metadata_tier):
 
 
 FUNCTION_LIST = [
-    run_depth_and_segmentation_test,
-    run_habituation_trial_counts_test,
-    run_position_by_step_test,
-    run_public_sample_scenes_test,
-    run_restricted_action_list_test
+    (run_depth_and_segmentation_test, DEPTH_AND_SEGMENTATION_TEST_CONFIGS),
+    (run_habituation_trial_counts_test, DEFAULT_TEST_CONFIGS),
+    (run_position_by_step_test, DEFAULT_TEST_CONFIGS),
+    (run_public_sample_scenes_test, DEFAULT_TEST_CONFIGS),
+    (run_restricted_action_list_test, DEFAULT_TEST_CONFIGS),
 ]

integration_tests/config_level1.ini

@@ -1,4 +1,6 @@
 [MCS]
+disable_depth_maps: true
+disable_object_masks: true
 history_enabled: false
 metadata: level1
 steps_allowed_in_lava: 1000

integration_tests/config_level2.ini

@@ -1,4 +1,6 @@
 [MCS]
+disable_depth_maps: true
+disable_object_masks: true
 history_enabled: false
 metadata: level2
 steps_allowed_in_lava: 1000

integration_tests/config_oracle.ini

@@ -1,4 +1,6 @@
 [MCS]
+disable_depth_maps: true
+disable_object_masks: true
 history_enabled: false
 metadata: oracle
 steps_allowed_in_lava: 1000

integration_tests/data/081.move_into_light_object_intuitive_reset.oracle.outputs.json

@@ -323,7 +323,7 @@
             {
                 "direction_x": 0.0,
                 "direction_y": -0.734,
-                "direction_z": 0.679,
+                "direction_z": 0.69,
                 "distance": 0.697,
                 "held": false,
                 "id": "testBall",
@@ -353,17 +353,17 @@
         "objects": [
             {
                 "direction_x": 0.0,
-                "direction_y": -0.731,
-                "direction_z": 0.683,
-                "distance": 0.7,
+                "direction_y": -0.72,
+                "direction_z": 0.69,
+                "distance": 0.71,
                 "held": false,
                 "id": "testBall",
                 "mass": 1.9,
                 "material_list": [
                     "RUBBER"
                 ],
                 "position_x": 0.0,
-                "position_z": 1.478,
+                "position_z": 1.49,
                 "shape": "ball",
                 "texture_color_list": [
                     "blue"