Add test description and remove duplicated tests (omg-dds#43)

.github/workflows/1_run_interoperability_tests.yml

@@ -25,6 +25,8 @@ jobs:
       - name: Generate timestamp file
         run: date '+%Y-%m-%d-%H_%M_%S' > timestamp
       - name: Run Interoperability script
+        # The test descriptions used are the generated for the last execution.
+        # This shouldn't be an issue because all test are run always
         run: |
           source .venv/bin/activate
           cd executables

.github/workflows/3_generate_doc.yml

@@ -33,6 +33,7 @@ jobs:
       - name: Setting up environment
         run: |
           pip install -r requirements_doc.txt
+          pip install -r requirements.txt
 
       - name: Get latest XLSX and ZIP urls
         env:

.gitignore

@@ -72,3 +72,5 @@ gdrive_url.py
 html/
 doc/detailed_report.rst
 doc/test_results.rst
+test_description.rst
+doc/test_description.rst

doc/conf.py

@@ -18,12 +18,15 @@
 import os
 import sys
 import glob
+import re
 # sys.path.insert(0, os.path.abspath('.'))
 import sphinx_rtd_theme
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
 from gdrive_url import xlsx_url, zip_url
+import test_suite
+
 
 def find_index_html():
     # Get the directory of the script
@@ -41,6 +44,28 @@ def find_index_html():
         return os.path.relpath(matching_files[0], script_directory)
     return None
 
+def generate_test_description_rst():
+    last_test_group = ''
+    test_description_rst = ''
+
+    for test_name, test_value in test_suite.rtps_test_suite_1.items():
+        current_test_group = re.search(r'Test_(.*?)_\d+', test_name).group(1)
+
+        # write new group heading
+        if last_test_group != current_test_group:
+            test_description_rst += f'{current_test_group}\n'
+            test_description_rst += '-' * len(current_test_group) + '\n'
+            last_test_group = current_test_group
+
+        # write test name heading
+        test_description_rst += f'{test_name}\n'
+        test_description_rst += '~' * len(test_name) + '\n'
+
+        # write test name and description
+        test_description_rst += f"{test_value['title']}\n\n"
+        test_description_rst += f"{test_value['description']}\n"
+
+    return test_description_rst
 
 # replacement is defined as
 # replacements = {
@@ -172,6 +197,10 @@ def setup(app):
     ]
 }
 
+
+# Add test description
+TEST_DESCRIPTION = generate_test_description_rst()
+
 # -- links
 LINK_XLSX_URL = xlsx_url
 LINK_ZIP_URL = zip_url
@@ -183,6 +212,7 @@ def setup(app):
 replacements = {
     '|LINK_XLSX_URL|': LINK_XLSX_URL,
     '|INDEX_HTML_PATH|': INDEX_HTML_PATH,
+    '|TEST_DESCRIPTION|': TEST_DESCRIPTION,
     # Add more replacements as needed
 }
 replace_in_rst_files(replacements)

doc/index.rst

@@ -12,5 +12,6 @@ Welcome to |INTEROPERABILITY_TESTS|
 
    introduction
    test_results
+   test_description
    detailed_report
    copyright

doc/test_description.template.rst

@@ -0,0 +1,50 @@
+.. include:: vars.rst
+
+.. _section-test-descriptions:
+
+Test Descriptions
+=================
+
+Default Values
+--------------
+
+This section describes the different test performed. When the test description
+mentions 'default settings' or if the test does not mention any QoS modification,
+then, it refers to the default values for different QoS settings and other
+parameters the ShapeDemo Application configures:
+
+* **Domain ID**: 0
+* **RELIABILITY QoS**: reliable
+* **DURABILITY QoS**: volatile
+* **HISTORY QoS**: default middleware HISTORY kind (should be KEEP_LAST with depth 1)
+* **DATA_REPRESENTATION QoS**: XCDR1
+* **OWNERSHIP QoS**: shared
+* **PARTITION QoS**: default middleware partition
+* **DEADLINE QoS**: disabled
+* **TIME_BASED_FILTER QoS**: disabled
+* **Instance (color value)**: BLUE
+* **Topic name**: if not mentioned, the topic used is "Square"
+* **Writing period**: 33ms
+* **Reading period**: 100ms
+* **Delay when creating entities**: at least 1s
+
+The type used in these tests is the following:
+
+.. code-block:: C
+
+    @appendable
+    struct ShapeType {
+        @key
+        string<128> color;
+        int32 x;
+        int32 y;
+        int32 shapesize;
+    };
+
+Additionally, the test description may mention 'Publisher' and 'Subscriber',
+this refers to the publisher/subscriber applications. Qos policies are set
+to the corresponding entity: DomainParticipant, Publisher, Subscriber, Topic,
+DataWriter or DataReader.
+
+
+|TEST_DESCRIPTION|

doc/test_results.template.rst

@@ -2,23 +2,28 @@
 
 .. _section-test-results:
 
+
 Test Results
 ============
 
-This page contains a summary of the interoperability test results. The test
-results are organized in the following spreadsheet in different tabs. The first
-tab presents a comprehensive summary of the tests per product. The first table
-delineates the number of passed tests versus total tests, offering a quick
-overview of vendor compliance. The second table delineates the tests performed
-between products acting as **Publishers** (rows) and **Subscribers** (columns),
-providing insights into interoperability between different DDS implementations.
+The test results are organized in the following spreadsheet in different tabs.
+The first tab presents a comprehensive summary of the tests per product. The
+first table delineates the number of passed tests versus total tests, offering a
+quick overview of vendor compliance. The second table delineates the tests
+performed between products acting as **Publishers** (rows) and **Subscribers**
+(columns), providing insights into interoperability between different DDS
+implementations.
 
 The subsequent tabs represent individual test case results per product. Each tab
 is named after the respective product and contains two tables:
 
 * Left-side table: Current product as publisher and all products as subscribers.
 * Righ-side table: Current product as subscriber and all products as publishers.
 
+Access the report at: |LINK_XLSX_URL|
+
+**NOTE**: for a detailed test description visit :ref:`section-test-descriptions`
+
 .. raw:: html
 
     <iframe src="|LINK_XLSX_URL|" width="100%" height="800"></iframe>

generate_xlsx_report.py

@@ -18,6 +18,7 @@
 import re
 import datetime
 from rtps_test_utilities import log_message
+import test_suite
 
 class XlxsReportArgumentParser:
     """Class that parse the arguments of the application."""
@@ -310,6 +311,7 @@ class XlsxReport:
     __data: JunitData
     __formats: dict = {} # contains the format name and formats objects
     REPO_LINK = 'https://github.com/omg-dds/dds-rtps'
+    REPO_DOC = 'https://omg-dds.github.io/dds-rtps/'
 
     def __init__(self, output: pathlib.Path, data: JunitData):
         """
@@ -322,6 +324,7 @@ def __init__(self, output: pathlib.Path, data: JunitData):
         self.__data = data
         self.add_formats()
         self.create_summary_worksheet()
+        self.create_description_worksheet()
         self.add_data_test_worksheet()
         self.workbook.close()
 
@@ -349,6 +352,22 @@ def create_summary_worksheet(self, name: str = 'Summary'):
         summary_worksheet.autofit()
         self.add_static_data_summary_worksheet(summary_worksheet)
 
+    def create_description_worksheet(self, name: str = 'Test Descriptions'):
+        """
+        Creates a test description worksheet from test_suite.py.
+        """
+        description_worksheet = self.workbook.add_worksheet(name=name)
+        self.set_worksheet_defaults(description_worksheet)
+
+        # Set the test names and static data before doing the autofit.
+        self.add_static_data_description_worksheet(description_worksheet)
+        self.add_test_name_description_worksheet(description_worksheet)
+
+        description_worksheet.autofit()
+
+        # Add the title of the test after doing the autofit
+        self.add_title_description_worksheet(description_worksheet)
+
     def add_formats(self):
         """Add the specific format"""
         self.__formats['title'] = self.workbook.add_format({
@@ -373,6 +392,8 @@ def add_formats(self):
             'valign': 'vcenter'
         })
 
+        self.__formats['bold'] = self.workbook.add_format(properties={'bold': True})
+
         self.__formats['bold_w_border'] = self.workbook.add_format(
             properties={'bold': True, 'border': 1})
 
@@ -718,16 +739,71 @@ def add_static_data_summary_worksheet(self,
             filename=dds_logo_path,
             options={'x_scale': 0.4, 'y_scale': 0.4, 'decorative': True, 'object_position': 2})
 
+        # Add date
+        current_row += 1
+        worksheet.write(current_row, starting_column + 1, 'Date')
+        date_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        worksheet.write(current_row, starting_column + 2, date_time)
+
         # Add repo link
         current_row += 1
         worksheet.write(current_row, starting_column + 1,'Repo')
         worksheet.write(current_row, starting_column + 2, self.REPO_LINK)
 
-        # Add date
+        # Add repo doc link
         current_row += 1
-        worksheet.write(current_row, starting_column + 1, 'Date')
-        date_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-        worksheet.write(current_row, starting_column + 2, date_time)
+        worksheet.write(current_row, starting_column + 1,'Documentation')
+        worksheet.write(current_row, starting_column + 2, self.REPO_DOC)
+
+    def add_static_data_description_worksheet(self,
+            worksheet: xlsxwriter.Workbook.worksheet_class,
+            name: str = 'Test Descriptions',
+            starting_row: int = 0, # row 1
+            starting_column: int = 1): # B column
+        """
+        Add header to the test descriptions worksheet, it includes the headers
+        of the columns.
+        """
+
+        current_row = starting_row
+
+        # Add column headers
+        worksheet.write(
+            current_row, starting_column,
+            'Test Name', self.__formats['product_subtitle'])
+        worksheet.write(
+            current_row, starting_column + 1,
+            'Test Title', self.__formats['product_subtitle'])
+        worksheet.write_url(
+            current_row, starting_column + 2,
+            'https://omg-dds.github.io/dds-rtps/test_description.html',
+            string="Click here for full test descriptions")
+
+    def add_test_name_description_worksheet(self,
+            worksheet: xlsxwriter.Workbook.worksheet_class,
+            starting_row: int = 1, # row 2
+            col: int = 1): # B column
+        """Add test names to the test descriptions worksheet."""
+
+        current_row = starting_row
+
+        # Add test name
+        for test_name in test_suite.rtps_test_suite_1.keys():
+            worksheet.write(current_row, col, test_name, self.__formats['bold'])
+            current_row += 1
+
+    def add_title_description_worksheet(self,
+            worksheet: xlsxwriter.Workbook.worksheet_class,
+            starting_row: int = 1, # row 2
+            col: int = 2): # C column
+        """Add short test description (aka title) to the test descriptions worksheet."""
+
+        current_row = starting_row
+
+        # Add test title
+        for value in test_suite.rtps_test_suite_1.values():
+            worksheet.write(current_row, col, value['title'])
+            current_row += 1
 
 def get_file_extension(input) -> str:
     """Get file extension from the input as Path or str"""

interoperability_report.py

@@ -665,8 +665,7 @@ def main():
         'test_suite': args.suite,
         'test_cases': args.test,
         'test_cases_disabled': args.disable_test,
-        'data_representation': args.data_representation
-
+        'data_representation': args.data_representation,
     }
 
     # The executables's names are supposed to follow the pattern: name_shape_main
@@ -740,20 +739,16 @@ def main():
                     continue
                 else:
                     # if the test case is processed
-                    parameters = test_case_parameters[0]
-                    expected_codes = test_case_parameters[1]
-                    if len(test_case_parameters) == 3:
-                        if callable(test_case_parameters[2]):
-                            check_function = test_case_parameters[2]
+                    parameters = test_case_parameters['apps']
+                    expected_codes = test_case_parameters['expected_codes']
+                    if ('check_function' in test_case_parameters):
+                        if callable(test_case_parameters['check_function']):
+                            check_function = test_case_parameters['check_function']
                         else:
                             raise RuntimeError('Cannot process function of '
                                 f'test case: {test_case_name}')
-                    elif len(test_case_parameters) == 2:
-                        check_function = no_check
                     else:
-                        print('Error in the definition of the Test Suite. '
-                                'Number of arguments incorrect.')
-                        break
+                        check_function = no_check
 
                     assert(len(parameters) == len(expected_codes))