From 3b0db76f641c0efd97b31c1f0547612756a763e2 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Sun, 23 Jun 2024 14:39:54 +0200 Subject: [PATCH] Add test description and remove duplicated tests (#43) --- .../1_run_interoperability_tests.yml | 2 + .github/workflows/3_generate_doc.yml | 1 + .gitignore | 2 + doc/conf.py | 30 + doc/index.rst | 1 + doc/test_description.template.rst | 50 + doc/test_results.template.rst | 19 +- generate_xlsx_report.py | 84 +- interoperability_report.py | 19 +- test_suite.py | 908 +++++++++++++++--- 10 files changed, 956 insertions(+), 160 deletions(-) create mode 100644 doc/test_description.template.rst diff --git a/.github/workflows/1_run_interoperability_tests.yml b/.github/workflows/1_run_interoperability_tests.yml index ed21967..28cf555 100644 --- a/.github/workflows/1_run_interoperability_tests.yml +++ b/.github/workflows/1_run_interoperability_tests.yml @@ -25,6 +25,8 @@ jobs: - name: Generate timestamp file run: date '+%Y-%m-%d-%H_%M_%S' > timestamp - name: Run Interoperability script + # The test descriptions used are the generated for the last execution. + # This shouldn't be an issue because all test are run always run: | source .venv/bin/activate cd executables diff --git a/.github/workflows/3_generate_doc.yml b/.github/workflows/3_generate_doc.yml index 3922dd0..5cc6334 100644 --- a/.github/workflows/3_generate_doc.yml +++ b/.github/workflows/3_generate_doc.yml @@ -33,6 +33,7 @@ jobs: - name: Setting up environment run: | pip install -r requirements_doc.txt + pip install -r requirements.txt - name: Get latest XLSX and ZIP urls env: diff --git a/.gitignore b/.gitignore index 8a7084c..de5f673 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,5 @@ gdrive_url.py html/ doc/detailed_report.rst doc/test_results.rst +test_description.rst +doc/test_description.rst diff --git a/doc/conf.py b/doc/conf.py index 8e4bd73..e7250e6 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -18,12 +18,15 @@ import os import sys import glob +import re # sys.path.insert(0, os.path.abspath('.')) import sphinx_rtd_theme sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from gdrive_url import xlsx_url, zip_url +import test_suite + def find_index_html(): # Get the directory of the script @@ -41,6 +44,28 @@ def find_index_html(): return os.path.relpath(matching_files[0], script_directory) return None +def generate_test_description_rst(): + last_test_group = '' + test_description_rst = '' + + for test_name, test_value in test_suite.rtps_test_suite_1.items(): + current_test_group = re.search(r'Test_(.*?)_\d+', test_name).group(1) + + # write new group heading + if last_test_group != current_test_group: + test_description_rst += f'{current_test_group}\n' + test_description_rst += '-' * len(current_test_group) + '\n' + last_test_group = current_test_group + + # write test name heading + test_description_rst += f'{test_name}\n' + test_description_rst += '~' * len(test_name) + '\n' + + # write test name and description + test_description_rst += f"{test_value['title']}\n\n" + test_description_rst += f"{test_value['description']}\n" + + return test_description_rst # replacement is defined as # replacements = { @@ -172,6 +197,10 @@ def setup(app): ] } + +# Add test description +TEST_DESCRIPTION = generate_test_description_rst() + # -- links LINK_XLSX_URL = xlsx_url LINK_ZIP_URL = zip_url @@ -183,6 +212,7 @@ def setup(app): replacements = { '|LINK_XLSX_URL|': LINK_XLSX_URL, '|INDEX_HTML_PATH|': INDEX_HTML_PATH, + '|TEST_DESCRIPTION|': TEST_DESCRIPTION, # Add more replacements as needed } replace_in_rst_files(replacements) diff --git a/doc/index.rst b/doc/index.rst index 1940983..c434164 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -12,5 +12,6 @@ Welcome to |INTEROPERABILITY_TESTS| introduction test_results + test_description detailed_report copyright diff --git a/doc/test_description.template.rst b/doc/test_description.template.rst new file mode 100644 index 0000000..cc2e2bd --- /dev/null +++ b/doc/test_description.template.rst @@ -0,0 +1,50 @@ +.. include:: vars.rst + +.. _section-test-descriptions: + +Test Descriptions +================= + +Default Values +-------------- + +This section describes the different test performed. When the test description +mentions 'default settings' or if the test does not mention any QoS modification, +then, it refers to the default values for different QoS settings and other +parameters the ShapeDemo Application configures: + +* **Domain ID**: 0 +* **RELIABILITY QoS**: reliable +* **DURABILITY QoS**: volatile +* **HISTORY QoS**: default middleware HISTORY kind (should be KEEP_LAST with depth 1) +* **DATA_REPRESENTATION QoS**: XCDR1 +* **OWNERSHIP QoS**: shared +* **PARTITION QoS**: default middleware partition +* **DEADLINE QoS**: disabled +* **TIME_BASED_FILTER QoS**: disabled +* **Instance (color value)**: BLUE +* **Topic name**: if not mentioned, the topic used is "Square" +* **Writing period**: 33ms +* **Reading period**: 100ms +* **Delay when creating entities**: at least 1s + +The type used in these tests is the following: + +.. code-block:: C + + @appendable + struct ShapeType { + @key + string<128> color; + int32 x; + int32 y; + int32 shapesize; + }; + +Additionally, the test description may mention 'Publisher' and 'Subscriber', +this refers to the publisher/subscriber applications. Qos policies are set +to the corresponding entity: DomainParticipant, Publisher, Subscriber, Topic, +DataWriter or DataReader. + + +|TEST_DESCRIPTION| diff --git a/doc/test_results.template.rst b/doc/test_results.template.rst index 3c00d6b..4351a03 100644 --- a/doc/test_results.template.rst +++ b/doc/test_results.template.rst @@ -2,16 +2,17 @@ .. _section-test-results: + Test Results ============ -This page contains a summary of the interoperability test results. The test -results are organized in the following spreadsheet in different tabs. The first -tab presents a comprehensive summary of the tests per product. The first table -delineates the number of passed tests versus total tests, offering a quick -overview of vendor compliance. The second table delineates the tests performed -between products acting as **Publishers** (rows) and **Subscribers** (columns), -providing insights into interoperability between different DDS implementations. +The test results are organized in the following spreadsheet in different tabs. +The first tab presents a comprehensive summary of the tests per product. The +first table delineates the number of passed tests versus total tests, offering a +quick overview of vendor compliance. The second table delineates the tests +performed between products acting as **Publishers** (rows) and **Subscribers** +(columns), providing insights into interoperability between different DDS +implementations. The subsequent tabs represent individual test case results per product. Each tab is named after the respective product and contains two tables: @@ -19,6 +20,10 @@ is named after the respective product and contains two tables: * Left-side table: Current product as publisher and all products as subscribers. * Righ-side table: Current product as subscriber and all products as publishers. +Access the report at: |LINK_XLSX_URL| + +**NOTE**: for a detailed test description visit :ref:`section-test-descriptions` + .. raw:: html diff --git a/generate_xlsx_report.py b/generate_xlsx_report.py index f392318..0805f67 100644 --- a/generate_xlsx_report.py +++ b/generate_xlsx_report.py @@ -18,6 +18,7 @@ import re import datetime from rtps_test_utilities import log_message +import test_suite class XlxsReportArgumentParser: """Class that parse the arguments of the application.""" @@ -310,6 +311,7 @@ class XlsxReport: __data: JunitData __formats: dict = {} # contains the format name and formats objects REPO_LINK = 'https://github.com/omg-dds/dds-rtps' + REPO_DOC = 'https://omg-dds.github.io/dds-rtps/' def __init__(self, output: pathlib.Path, data: JunitData): """ @@ -322,6 +324,7 @@ def __init__(self, output: pathlib.Path, data: JunitData): self.__data = data self.add_formats() self.create_summary_worksheet() + self.create_description_worksheet() self.add_data_test_worksheet() self.workbook.close() @@ -349,6 +352,22 @@ def create_summary_worksheet(self, name: str = 'Summary'): summary_worksheet.autofit() self.add_static_data_summary_worksheet(summary_worksheet) + def create_description_worksheet(self, name: str = 'Test Descriptions'): + """ + Creates a test description worksheet from test_suite.py. + """ + description_worksheet = self.workbook.add_worksheet(name=name) + self.set_worksheet_defaults(description_worksheet) + + # Set the test names and static data before doing the autofit. + self.add_static_data_description_worksheet(description_worksheet) + self.add_test_name_description_worksheet(description_worksheet) + + description_worksheet.autofit() + + # Add the title of the test after doing the autofit + self.add_title_description_worksheet(description_worksheet) + def add_formats(self): """Add the specific format""" self.__formats['title'] = self.workbook.add_format({ @@ -373,6 +392,8 @@ def add_formats(self): 'valign': 'vcenter' }) + self.__formats['bold'] = self.workbook.add_format(properties={'bold': True}) + self.__formats['bold_w_border'] = self.workbook.add_format( properties={'bold': True, 'border': 1}) @@ -718,16 +739,71 @@ def add_static_data_summary_worksheet(self, filename=dds_logo_path, options={'x_scale': 0.4, 'y_scale': 0.4, 'decorative': True, 'object_position': 2}) + # Add date + current_row += 1 + worksheet.write(current_row, starting_column + 1, 'Date') + date_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + worksheet.write(current_row, starting_column + 2, date_time) + # Add repo link current_row += 1 worksheet.write(current_row, starting_column + 1,'Repo') worksheet.write(current_row, starting_column + 2, self.REPO_LINK) - # Add date + # Add repo doc link current_row += 1 - worksheet.write(current_row, starting_column + 1, 'Date') - date_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - worksheet.write(current_row, starting_column + 2, date_time) + worksheet.write(current_row, starting_column + 1,'Documentation') + worksheet.write(current_row, starting_column + 2, self.REPO_DOC) + + def add_static_data_description_worksheet(self, + worksheet: xlsxwriter.Workbook.worksheet_class, + name: str = 'Test Descriptions', + starting_row: int = 0, # row 1 + starting_column: int = 1): # B column + """ + Add header to the test descriptions worksheet, it includes the headers + of the columns. + """ + + current_row = starting_row + + # Add column headers + worksheet.write( + current_row, starting_column, + 'Test Name', self.__formats['product_subtitle']) + worksheet.write( + current_row, starting_column + 1, + 'Test Title', self.__formats['product_subtitle']) + worksheet.write_url( + current_row, starting_column + 2, + 'https://omg-dds.github.io/dds-rtps/test_description.html', + string="Click here for full test descriptions") + + def add_test_name_description_worksheet(self, + worksheet: xlsxwriter.Workbook.worksheet_class, + starting_row: int = 1, # row 2 + col: int = 1): # B column + """Add test names to the test descriptions worksheet.""" + + current_row = starting_row + + # Add test name + for test_name in test_suite.rtps_test_suite_1.keys(): + worksheet.write(current_row, col, test_name, self.__formats['bold']) + current_row += 1 + + def add_title_description_worksheet(self, + worksheet: xlsxwriter.Workbook.worksheet_class, + starting_row: int = 1, # row 2 + col: int = 2): # C column + """Add short test description (aka title) to the test descriptions worksheet.""" + + current_row = starting_row + + # Add test title + for value in test_suite.rtps_test_suite_1.values(): + worksheet.write(current_row, col, value['title']) + current_row += 1 def get_file_extension(input) -> str: """Get file extension from the input as Path or str""" diff --git a/interoperability_report.py b/interoperability_report.py index 61469c8..7893340 100644 --- a/interoperability_report.py +++ b/interoperability_report.py @@ -665,8 +665,7 @@ def main(): 'test_suite': args.suite, 'test_cases': args.test, 'test_cases_disabled': args.disable_test, - 'data_representation': args.data_representation - + 'data_representation': args.data_representation, } # The executables's names are supposed to follow the pattern: name_shape_main @@ -740,20 +739,16 @@ def main(): continue else: # if the test case is processed - parameters = test_case_parameters[0] - expected_codes = test_case_parameters[1] - if len(test_case_parameters) == 3: - if callable(test_case_parameters[2]): - check_function = test_case_parameters[2] + parameters = test_case_parameters['apps'] + expected_codes = test_case_parameters['expected_codes'] + if ('check_function' in test_case_parameters): + if callable(test_case_parameters['check_function']): + check_function = test_case_parameters['check_function'] else: raise RuntimeError('Cannot process function of ' f'test case: {test_case_name}') - elif len(test_case_parameters) == 2: - check_function = no_check else: - print('Error in the definition of the Test Suite. ' - 'Number of arguments incorrect.') - break + check_function = no_check assert(len(parameters) == len(expected_codes)) diff --git a/test_suite.py b/test_suite.py index 5c098ad..ba32d16 100644 --- a/test_suite.py +++ b/test_suite.py @@ -17,22 +17,36 @@ # the dictionary is a Test Case that the interoperability_report.py # executes. # The dictionary has the following structure: -# 'name' : [[parameter_list], [expected_return_code_list], checking_function] +# 'name' : { +# 'apps' : [parameter_list], +# 'expected_codes' : [expected_return_code_list], +# 'check_function' : checking_function, +# 'title' : 'This is the title of the test', +# 'description' : 'This is a long description of the test' +# }, # where: # * name: TestCase's name -# * parameter_list: list in which each element is the parameters that -# the shape_main application will use. -# * expected_return_code_list: list with expected ReturnCodes +# * apps: list in which each element contains the parameters that +# the shape_main application will use. Each element of the list +# will run a new app. +# * expected_codes: list with expected ReturnCodes # for a succeed test execution. -# * checking_function [OPTIONAL]: function to check how the Subscribers receive -# the samples from the Publishers. By default, it just checks that +# * check_function [OPTIONAL]: function to check how the subscribers receive +# the samples from the publishers. By default, it just checks that # the data is received. In case that it has a different behavior, that # function must be implemented in the test_suite file and the test case # should reference it in this parameter. -# The number of elements in parameter_list defines how many shape_main +# * title: human-readable short description of the test +# * description: description of the test behavior and parameters +# +# The number of elements in 'apps' list defines how many shape_main # applications the interoperability_report will run. It should be the same as -# the number of elements in expected_return_code_list. +# the number of elements in expected_codes. +# This constant is used to limit the maximum number of samples that tests that +# check the behavior needs to read. For example, checking that the data +# is received in order, or that OWNERSHIP works properly, etc... +MAX_SAMPLES_READ = 500 def test_ownership_receivers(child_sub, samples_sent, timeout): @@ -47,8 +61,8 @@ def test_ownership_receivers(child_sub, samples_sent, timeout): child_sub: child program generated with pexpect samples_sent: list of multiprocessing Queues with the samples - the Publishers send. Element 1 of the list is for - Publisher 1, etc. + the publishers send. Element 1 of the list is for + publisher 1, etc. timeout: time pexpect waits until it matches a pattern. This functions assumes that the subscriber has already received samples @@ -60,7 +74,7 @@ def test_ownership_receivers(child_sub, samples_sent, timeout): second_received = False list_data_received_second = [] list_data_received_first = [] - max_samples_received = 500 + max_samples_received = MAX_SAMPLES_READ samples_read = 0 while(samples_read < max_samples_received): @@ -155,18 +169,111 @@ def test_ownership_receivers(child_sub, samples_sent, timeout): if second_received == True and first_received == True: return ReturnCode.RECEIVING_FROM_BOTH + print(f'Samples read: {samples_read}') return ReturnCode.RECEIVING_FROM_ONE +def test_color_receivers(child_sub, samples_sent, timeout): + + """ + This function is used by test cases that have two publishers and one + subscriber. This tests that only one of the color is received by the + subscriber application because it contains a filter that only allows to + receive data from one color. + + child_sub: child program generated with pexpect + samples_sent: not used + timeout: time pexpect waits until it matches a pattern. + """ + sub_string = re.search('\w\s+(\w+)\s+[0-9]+ [0-9]+ \[[0-9]+\]', + child_sub.before + child_sub.after) + last_color = current_color = sub_string.group(1) + + max_samples_received = MAX_SAMPLES_READ + samples_read = 0 + + while sub_string is not None and samples_read < max_samples_received: + # Check that all received samples have the same color + if last_color != current_color: + return ReturnCode.RECEIVING_FROM_BOTH + + index = child_sub.expect( + [ + '\[[0-9]+\]', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + + if index == 1: + break + + samples_read += 1 + + sub_string = re.search('\w\s+(\w+)\s+[0-9]+ [0-9]+ \[[0-9]+\]', + child_sub.before + child_sub.after) + + print(f'Samples read: {samples_read}') + return ReturnCode.RECEIVING_FROM_ONE -def test_reliability_4(child_sub, samples_sent, timeout): +def test_reliability_order(child_sub, samples_sent, timeout): """ - This function tests reliability, it checks whether the Subscriber receives + This function tests reliability, it checks whether the subscriber receives the samples in order. + child_sub: child program generated with pexpect + samples_sent: not used + timeout: not used + """ + + produced_code = ReturnCode.OK + + # Read the first sample printed by the subscriber + sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', + child_sub.before + child_sub.after) + last_size = 0 + + max_samples_received = MAX_SAMPLES_READ + samples_read = 0 + + while sub_string is not None and samples_read < max_samples_received: + current_size = int(sub_string.group(1)) + if (current_size > last_size): + last_size = current_size + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + + # Get the next sample the subscriber is receiving + index = child_sub.expect( + [ + '\[[0-9]+\]', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + if index == 1: + # no more data to process + break + + samples_read += 1 + + # search the next received sample by the subscriber app + sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', + child_sub.before + child_sub.after) + + print(f'Samples read: {samples_read}') + return produced_code + + +def test_reliability_no_losses(child_sub, samples_sent, timeout): + """ + This function tests RELIABLE reliability, it checks whether the subscriber + receives the samples in order and with no losses. + child_sub: child program generated with pexpect samples_sent: list of multiprocessing Queues with the samples - the Publishers send. Element 1 of the list is for - Publisher 1, etc. + the publishers send. Element 1 of the list is for + publisher 1, etc. timeout: time pexpect waits until it matches a pattern. """ @@ -197,7 +304,11 @@ def test_reliability_4(child_sub, samples_sent, timeout): # checked before that the samples are the same, we need to skip that part # and get the next received sample first_execution = True - while sub_string is not None: + + max_samples_received = MAX_SAMPLES_READ + samples_read = 0 + + while sub_string is not None and samples_read < max_samples_received: # check that all the samples received by the DataReader are in order # and matches the samples sent by the DataWriter try: @@ -230,19 +341,21 @@ def test_reliability_4(child_sub, samples_sent, timeout): if index == 1: # no more data to process break + samples_read += 1 # search the next received sample by the subscriber app sub_string = re.search('[0-9]+ [0-9]+ \[[0-9]+\]', child_sub.before + child_sub.after) + print(f'Samples read: {samples_read}') return produced_code def test_durability_volatile(child_sub, samples_sent, timeout): """ This function tests the volatile durability, it checks that the sample the - Subscriber receives is not the first one. The Publisher application sends + subscriber receives is not the first one. The publisher application sends samples increasing the value of the size, so if the first sample that the - Subscriber app doesn't have the size > 5, the test is correct. + subscriber app doesn't have the size > 5, the test is correct. Note: size > 5 to avoid checking only the first sample, that may be an edge case where the DataReader hasn't matched with the DataWriter yet and @@ -259,7 +372,8 @@ def test_durability_volatile(child_sub, samples_sent, timeout): child_sub.before + child_sub.after) # Check if the element received is not the first 5 samples (aka size >= 5) - # which should not be the case. Checking 5 samples instead of just one to + # which should not be the case because the subscriber application waits some + # seconds after the publisher. Checking 5 samples instead of just one to # make sure that there is not the case in which the DataReader hasn't # matched with the DataWriter yet and the first samples may not be received. # The group(1) contains the matching element for the parameter between @@ -274,9 +388,9 @@ def test_durability_volatile(child_sub, samples_sent, timeout): def test_durability_transient_local(child_sub, samples_sent, timeout): """ This function tests the TRANSIENT_LOCAL durability, it checks that the - sample the Subscriber receives is the first one. The Publisher application + sample the subscriber receives is the first one. The publisher application sends samples increasing the value of the size, so if the first sample that - the Subscriber app does have the size == 1, the test is correct. + the subscriber app does have the size == 1, the test is correct. child_sub: child program generated with pexpect samples_sent: not used @@ -333,145 +447,665 @@ def test_deadline_missed(child_sub, samples_sent, timeout): rtps_test_suite_1 = { - # DATA REPRESENTATION - 'Test_DataRepresentation_0' : [['-P -t Square -x 1', '-S -t Square -x 1'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_DataRepresentation_1' : [['-P -t Square -x 1', '-S -t Square -x 2'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - 'Test_DataRepresentation_2' : [['-P -t Square -x 2', '-S -t Square -x 1'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - 'Test_DataRepresentation_3' : [['-P -t Square -x 2', '-S -t Square -x 2'], [ReturnCode.OK, ReturnCode.OK]], - # DOMAIN - 'Test_Domain_0' : [['-P -t Square', '-S -t Square'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Domain_1' : [['-P -t Square', '-S -t Square -d 1'], [ReturnCode.READER_NOT_MATCHED, ReturnCode.DATA_NOT_RECEIVED]], - 'Test_Domain_2' : [['-P -t Square -d 1', '-S -t Square'], [ReturnCode.READER_NOT_MATCHED, ReturnCode.DATA_NOT_RECEIVED]], - 'Test_Domain_3' : [['-P -t Square -d 1', '-S -t Square -d 1'], [ReturnCode.OK, ReturnCode.OK]], + 'Test_Domain_0' : { + 'apps' : ['-P -t Square -d 0', '-S -t Square -d 0 -b'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication using Domain ID 0', + 'description' : 'This test covers the most basic interoperability scenario:\n' + ' * Configures the publisher / subscriber with Domain ID 0\n' + ' * Configures the subscriber with BEST_EFFORT reliability\n' + 'The tests passes if the publisher and subscriber discover and match each other and the subscriber ' + 'receives the data from the publisher\n' + }, + + 'Test_Domain_1' : { + 'apps' : ['-P -t Square -d 0', '-S -t Square -d 1'], + 'expected_codes' : [ReturnCode.READER_NOT_MATCHED, ReturnCode.DATA_NOT_RECEIVED], + 'title' : 'No communication between publisher and subscriber in a different Domain IDs', + 'description' : 'Verifies that there is no communication between a publisher configured with Domain ID 0 ' + 'and a subscriber configured with Domain ID 1:\n' + ' * Configures the publisher with Domain ID 0\n' + ' * Configures the subscriber with Domain ID 1\n' + 'The tests passes if the publisher and subscriber do not discover each other\n' + }, + + 'Test_Domain_2' : { + 'apps' : ['-P -t Square -d 1', '-S -t Square -d 1 -b'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication using Domain ID 1', + 'description' : 'This test covers interoperability in a non-default Domain ID:\n' + ' * Configures the publisher / subscriber with Domain ID 1\n' + ' * Configures the subscriber with BEST_EFFORT reliability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The tests passes if the subscriber receives samples from the publisher\n' + }, + + # DATA REPRESENTATION + 'Test_DataRepresentation_0' : { + 'apps' : ['-P -t Square -x 1', '-S -t Square -x 1'], + 'expected_codes' : [ ReturnCode.OK, ReturnCode.OK], + 'title' : 'Default communication using XCDR1', + 'description' : 'This test covers the interoperability scenario with XCDR1:\n' + ' * Configures the publisher / subscriber with DATA_REPRESENTATION XCDR version 1\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The tests passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_DataRepresentation_1' : { + 'apps' : ['-P -t Square -x 1', '-S -t Square -x 2'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'Incompatibility publishing XCDR1 and subscribing XCDR2', + 'description' : 'Verifies an XDCR1 publisher does not match with an XCDR2 subscriber and report an ' + 'IncompatibleQos notification\n' + ' * Configures the publisher with XCDR1 data representation\n' + ' * Configures the subscriber with XCDR2 data representation\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_DataRepresentation_2' : { + 'apps' : ['-P -t Square -x 2', '-S -t Square -x 1'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'Incompatibility publishing XCDR2 and subscribing XCDR1', + 'description' : 'Verifies an XDCR2 publisher does not match with an XCDR1 subscriber and report an ' + 'IncompatibleQos notification\n' + ' * Configures the publisher with XCDR2 data representation\n' + ' * Configures the subscriber with XCDR1 data representation\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_DataRepresentation_3' : { + 'apps' : ['-P -t Square -x 2', '-S -t Square -x 2 -b'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Default communication using XCDR2', + 'description' : 'This test covers the interoperability scenario with XCDR2:\n' + ' * Configures publisher / subscriber with DATA_REPRESENTATION XCDR version 2\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The tests passes if the subscriber receives samples from the publisher\n' + }, # RELIABILITY - 'Test_Reliability_0' : [['-P -t Square -b', '-S -t Square -b'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Reliability_1' : [['-P -t Square -b', '-S -t Square -r'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - 'Test_Reliability_2' : [['-P -t Square -r', '-S -t Square -b'], [ReturnCode.OK, ReturnCode.OK]], + 'Test_Reliability_0' : { + 'apps' : ['-P -t Square -b -z 0', '-S -t Square -b -z 0'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : test_reliability_order, + 'title' : 'Communication between BEST_EFFORT publisher and subscriber', + 'description' : 'Verifies a best effort publisher communicates with a best effort subscriber with no out-of-order ' + 'or duplicate samples\n' + ' * Configures the publisher and subscriber with a BEST_EFFORT reliability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the subscriber application receives samples and the value of the "size" member is always increasing\n' + 'The test passes if the value of the "size" is always increasing in ' + f'{MAX_SAMPLES_READ} samples, even if there are missed samples (since reliability ' + 'is BEST_EFFORT) as long as there are no out-of-order or duplicated samples\n' + }, + + 'Test_Reliability_1' : { + 'apps' : ['-P -t Square -b', '-S -t Square -r'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'BEST_EFFORT publisher do not match RELIABLE subscribers', + 'description' : 'Verifies a best effort publisher does not match with a reliable subscriber and report an ' + 'IncompatibleQos notification\n' + ' * Configures the publisher with BEST_EFFORT reliability\n' + ' * Configures the publisher with RELIABLE reliability\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_Reliability_2' : { + 'apps' : ['-P -t Square -r', '-S -t Square -b'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between RELIABLE publisher and BEST_EFFORT subscriber', + 'description' : 'Verifies a reliable publisher communicates with a best effort subscriber\n' + ' * Configures the publisher with a RELIABLE reliability\n' + ' * Configures the subscriber with a BEST_EFFORT reliability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + # This test only checks that data is received correctly - 'Test_Reliability_3' : [['-P -t Square -r -k 3', '-S -t Square -r'], [ReturnCode.OK, ReturnCode.OK]], + 'Test_Reliability_3' : { + 'apps' : ['-P -t Square -r', '-S -t Square -r'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication using reliability RELIABLE', + 'description' : 'Verifies a reliable publisher communicates with a reliable subscriber\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + # This test checks that data is received in the right order - 'Test_Reliability_4' : [['-P -t Square -r -k 0 -w', '-S -t Square -r -k 0'], [ReturnCode.OK, ReturnCode.OK], test_reliability_4], + 'Test_Reliability_4' : { + 'apps' : ['-P -t Square -r -k 0 -w', '-S -t Square -r -k 0'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : test_reliability_no_losses, + 'title' : 'Behavior of RELIABLE reliability', + 'description' : 'Verifies a RELIABLE publisher communicates with a RELIABLE subscriber and samples are received ' + 'in order without any losses or duplicates\n' + ' * Configures the publisher and subscriber with a RELIABLE reliability\n' + ' * Configures the publisher and subscriber with history KEEP_ALL\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber, after receiving a (first) sample from the publisher, it ' + f'receives the next {MAX_SAMPLES_READ} subsequent samples, without losses or duplicates, in ' + 'the same order as sent\n' + }, + + # OWNERSHIP + 'Test_Ownership_0' : { + 'apps' : ['-P -t Square -s -1', '-S -t Square -s -1'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between SHARED OWNERSHIP publisher and subscriber', + 'description' : 'Verifies a shared ownership publisher communicates with a shared ' + 'ownership subscriber\n' + ' * Configures the publisher / subscriber with SHARED ownership\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Ownership_1' : { + 'apps' : ['-P -t Square -s -1', '-S -t Square -s 1'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'Incompatibility between SHARED OWNERSHIP publisher and EXCLUSIVE OWNERSHIP subscriber', + 'description' : 'Verifies a shared ownership publisher does not match with an exclusive ' + 'ownership subscriber and report an IncompatibleQos notification\n' + ' * Configures the publisher with SHARED ownership\n' + ' * Configures the subscriber with EXCLUSIVE ownership\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_Ownership_2' : { + 'apps' : ['-P -t Square -s 3', '-S -t Square -s -1'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'Incompatibility between EXCLUSIVE OWNERSHIP publisher and SHARED OWNERSHIP subscriber', + 'description' : 'Verifies a exclusive ownership publisher does not match with an shared ' + 'ownership subscriber and report an IncompatibleQos notification\n' + ' * Configures the publisher with EXCLUSIVE ownership\n' + ' * Configures the subscriber with SHARED ownership\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + # For Test_Ownership_[3|4|5|6]: each publisher application publishes samples + # with a different shapesize to allow the subscriber app to recognize from + # which publisher is receiving the samples. + + # The DataReader should only receive samples from the DataWriter with higher + # ownership. There may be the situation that the DataReader starts receiving + # samples from one DataWriter until another DataWriter with higher ownership + # strength is created. This should be handled by test_ownership_receivers(). + 'Test_Ownership_3': { + 'apps' : [ + '-P -t Square -s 3 -r -k 0 -c BLUE -w -z 20', + '-P -t Square -s 4 -r -k 0 -c BLUE -w -z 30', + '-S -t Square -s 1 -r -k 0'], + 'expected_codes' :[ + ReturnCode.OK, + ReturnCode.OK, + ReturnCode.RECEIVING_FROM_ONE], + 'check_function' : test_ownership_receivers, + 'title' : 'Behavior of EXCLUSIVE OWNERSHIP QoS with publishers of the same instance', + 'description' : 'Verifies an exclusive ownership subscriber receives samples only from ' + 'the highest ownership strength publisher of the same instance\n' + ' * Use RELIABLE Qos in all publishers and subscriber to avoid samples losses\n' + ' * Use KEEP_ALL HISTORY Qos in all publishers and subscriber\n' + ' * Configures a first publisher with EXCLUSIVE ownership with strength of 3\n' + ' * Configures the first publisher to publish samples with "color" equal to "BLUE" ' + 'and "size" equal to 20\n' + ' * Configures a second publisher with EXCLUSIVE ownership and strength of 4\n' + ' * Configures the second publisher to publish samples with "color" equal to "BLUE" ' + ' and "size" equal to 30\n' + ' * Configures a subscriber with EXCLUSIVE ownership\n' + ' * Verifies that both publishers discover and match the subscriber and vice-versa\n' + ' * Note that the subscriber may start receiving samples from the lower ownership strength ' + 'publisher if it is created before the highest strength ownership publisher. This behavior ' + 'is expected and those samples are ignored\n' + 'The test passes if the subscriber receives samples from the highest strength publisher only ' + '(after receiving the first sample of that publisher. The subscriber reads ' + f'{MAX_SAMPLES_READ} samples in total\n' + }, + + # The DataReader should receive from both publisher apps because they + # publish different instances and the ownership is applied per instance. + 'Test_Ownership_4': { + 'apps' :[ + '-P -t Square -s 3 -r -k 0 -c BLUE -w -z 20', + '-P -t Square -s 4 -r -k 0 -c RED -w -z 30', + '-S -t Square -s 1 -r -k 0'], + 'expected_codes' : [ + ReturnCode.OK, + ReturnCode.OK, + ReturnCode.RECEIVING_FROM_BOTH], + 'check_function' : test_ownership_receivers, + 'title' : 'Behavior of EXCLUSIVE OWNERSHIP QoS with publishers with different instances', + 'description' : 'Verifies an exclusive ownership subscriber receives samples from different ' + 'publishers that publish different instances (ShapeType with different color)\n' + ' * Use RELIABLE Qos in all publishers and subscriber to avoid samples losses\n' + ' * Use KEEP_ALL HISTORY Qos in all publishers and subscriber\n' + ' * Configures a first publisher with EXCLUSIVE ownership with strength of 3\n' + ' * Configures the first publisher to publish samples with "color" equal to "BLUE" ' + ' and "size" equal to 20\n' + ' * Configures a second publisher with EXCLUSIVE ownership and strength of 4\n' + ' * Configures the second publisher to publish samples with "color" equal to "RED" ' + 'and "size" equal to 30\n' + ' * Configures a subscriber with EXCLUSIVE ownership\n' + ' * Verifies that both publishers discover and match the subscriber and vice-versa\n' + 'The test passes if the subscriber receives samples from both publishers in the first ' + f'{MAX_SAMPLES_READ} samples\n' + }, + + # The DataReader should receive from both publisher apps because they have + # shared ownership. + 'Test_Ownership_5': { + 'apps' : [ + '-P -t Square -s -1 -r -k 0 -c BLUE -w -z 20', + '-P -t Square -s -1 -r -k 0 -c BLUE -w -z 30', + '-S -t Square -s -1 -r -k 0'], + 'expected_codes' :[ + ReturnCode.OK, + ReturnCode.OK, + ReturnCode.RECEIVING_FROM_BOTH], + 'check_function' : test_ownership_receivers, + 'title' : 'Behavior of SHARED OWNERSHIP QoS with publishers with the same instance', + 'description' : 'Verifies a shared ownership subscriber receives samples from all ' + 'shared ownership publishers of the different instances\n' + ' * Use RELIABLE Qos in all publishers and subscriber to avoid samples losses\n' + ' * Use KEEP_ALL HISTORY Qos in all publishers and subscriber\n' + ' * Configures a first publisher with SHARED ownership\n' + ' * Configures the first publisher to publish samples with "color" equal to "BLUE" ' + 'and "size" equal to 20\n' + ' * Configures a second publisher with SHARED ownership\n' + ' * Configures the second publisher to publish samples with "color" equal to "BLUE" ' + 'and "size" equal to 30\n' + ' * Configures a subscriber with SHARED ownership\n' + ' * Verifies that both publishers discover and match the subscriber and vice-versa\n' + 'The test passes if the subscriber receives samples from both publishers in the first ' + f'{MAX_SAMPLES_READ} samples\n' + }, + + # The DataReader should receive from both publisher apps because they have + # shared ownership. + 'Test_Ownership_6': { + 'apps' : [ + '-P -t Square -s -1 -r -k 0 -c BLUE -w -z 20', + '-P -t Square -s -1 -r -k 0 -c RED -w -z 30', + '-S -t Square -s -1 -r -k 0'], + 'expected_codes' : [ + ReturnCode.OK, + ReturnCode.OK, + ReturnCode.RECEIVING_FROM_BOTH], + 'check_function' : test_ownership_receivers, + 'title' : 'Behavior of SHARED OWNERSHIP QoS with different instances', + 'description' : 'Verifies a shared ownership subscriber receives samples from all ' + 'shared ownership publishers of different instances\n' + ' * Use RELIABLE Qos in all publishers and subscriber to avoid samples losses\n' + ' * Use KEEP_ALL HISTORY Qos in all publishers and subscriber\n' + ' * Configures a first publisher with SHARED ownership with strength of 3\n' + ' * Configures the first publisher to publish samples with "color" equal to "BLUE" ' + 'and "size" equal to 20\n' + ' * Configures a second publisher with SHARED ownership and strength of 4\n' + ' * Configures the second publisher to publish samples with "color" equal to "RED" ' + 'and "size" equal to 30\n' + ' * Configures a subscriber with SHARED ownership\n' + ' * Verifies that both publishers discover and match the subscriber and vice-versa\n' + 'The test passes if the subscriber receives samples from both publishers in the first ' + f'{MAX_SAMPLES_READ} samples\n' + }, # DEADLINE - 'Test_Deadline_0' : [['-P -t Square -f 3', '-S -t Square -f 5'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Deadline_1' : [['-P -t Square -f 5', '-S -t Square -f 5'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Deadline_2' : [['-P -t Square -f 7', '-S -t Square -f 5'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], + 'Test_Deadline_0' : { + 'apps' : ['-P -t Square -f 3', '-S -t Square -f 5'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication with publisher deadline smaller than subscriber deadline', + 'description' : 'Verifies there is communication between a publisher with a deadline smaller than the subscriber\n' + ' * Configures the publisher with DEADLINE of 3 seconds\n' + ' * Configures the subscriber with DEADLINE of 5 seconds\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Deadline_1' : { + 'apps' : ['-P -t Square -f 5', '-S -t Square -f 5'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication with the same publisher and subscriber deadlines', + 'description' : 'Verifies there is communication between a publisher with the same deadline as the subscriber\n' + ' * Configures the publisher with DEADLINE of 5 seconds\n' + ' * Configures the subscriber with DEADLINE of 5 seconds\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Deadline_2' : { + 'apps' : ['-P -t Square -f 7', '-S -t Square -f 5'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility with publisher deadline higher than subscriber deadline', + 'description' : 'Verifies there is no communication between a publisher with a higher deadline than the ' + 'subscriber and both report an IncompatibleQos notification\n' + ' * Configures the publisher with DEADLINE of 7 seconds\n' + ' * Configures the publisher with DEADLINE of 5 seconds\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + # This test checks that the deadline is missed in both, publisher and subscriber # because the write-period is higher than the deadline period, that means - # that the samples won't be send and received on time - 'Test_Deadline_4' : [['-P -t Square -w -f 2 --write-period 3000', '-S -t Square -f 2'], - [ReturnCode.DEADLINE_MISSED, ReturnCode.DEADLINE_MISSED], - test_deadline_missed], + # that the samples won't be sent and received on time + 'Test_Deadline_3' : { + 'apps' : ['-P -t Square -f 2 --write-period 3000', '-S -t Square -f 2'], + 'expected_codes' : [ReturnCode.DEADLINE_MISSED, ReturnCode.DEADLINE_MISSED], + 'check_function' : test_deadline_missed, + 'title' : 'Deadline is missed in both, publisher and subscriber', + 'description' : 'Verifies that publisher and subscriber miss the deadline\n' + ' * Configures the publisher with DEADLINE of 2 seconds\n' + ' * Configures the subscriber with DEADLINE of 2 seconds\n' + ' * Configures the write period to 3 seconds\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the listeners trigger the DeadlineMissed notification in the publisher ' + 'and the subscriber\n' + }, # TOPIC - 'Test_Topic_0' : [['-P -t Square', '-S -t Square'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Topic_1' : [['-P -t Square', '-S -t Circle'], [ReturnCode.READER_NOT_MATCHED, ReturnCode.DATA_NOT_RECEIVED]], - 'Test_Topic_2' : [['-P -t Circle', '-S -t Square'], [ReturnCode.READER_NOT_MATCHED, ReturnCode.DATA_NOT_RECEIVED]], - 'Test_Topic_3' : [['-P -t Circle', '-S -t Circle'], [ReturnCode.OK, ReturnCode.OK]], + 'Test_Topic_0' : { + 'apps' : ['-P -t Circle', '-S -t Circle'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication using the same topic: Circle', + 'description' : 'Verifies communication between a publisher and a subscriber using a specific topic ("Circle")\n' + ' * Configures the publisher and subscriber to use the topic name "Circle"\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Topic_1' : { + 'apps' : ['-P -t Square', '-S -t Circle'], + 'expected_codes' : [ReturnCode.READER_NOT_MATCHED, ReturnCode.DATA_NOT_RECEIVED], + 'title' : 'No communication when publisher and subscriber are using different topics', + 'description' : 'Verifies that there is no communication between a publisher using topic "Square" ' + 'and a subscriber using topic "Circle"\n' + ' * Configures the publisher to use the topic name "Square"\n' + ' * Configures the subscriber to use the topic name "Circle"\n' + 'The test passes if the publisher and subscriber do not discover each other\n' + }, # COLOR - 'Test_Color_0' : [['-P -t Square -c BLUE', '-S -t Square -c BLUE'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Color_1' : [['-P -t Square -c BLUE', '-S -t Square -c RED'], [ReturnCode.OK, ReturnCode.DATA_NOT_RECEIVED]], - 'Test_Color_2' : [['-P -t Square -c BLUE', '-S -t Square'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Color_3' : [['-P -t Square -c RED', '-S -t Square -c BLUE'], [ReturnCode.OK, ReturnCode.DATA_NOT_RECEIVED]], - 'Test_Color_4' : [['-P -t Square -c RED', '-S -t Square -c RED'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Color_5' : [['-P -t Square -c RED', '-S -t Square'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Color_6' : [['-P -t Square', '-S -t Square -c BLUE'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Color_7' : [['-P -t Square', '-S -t Square -c RED'], [ReturnCode.OK, ReturnCode.DATA_NOT_RECEIVED]], - 'Test_Color_8' : [['-P -t Square', '-S -t Square'], [ReturnCode.OK, ReturnCode.OK]], + 'Test_Color_0' : { + 'apps' : ['-P -t Square -r -c BLUE', '-P -t Square -r -c RED', '-S -t Square -r -c RED'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK, ReturnCode.RECEIVING_FROM_ONE], + 'check_function' : test_color_receivers, + 'title' : 'Use of Content filter to avoid receiving undesired data', + 'description' : 'Verifies a subscription using a ContentFilteredTopic does not receive data that does not pass the filter\n' + ' * Configures a subscriber with a ContentFilteredTopic that selects only the shapes that ' + 'have "color" equal to "RED"\n' + ' * Configures a first publisher to publish samples with "color" equal to "BLUE"\n' + ' * Configures a second publisher to publish samples with "color" equal to "RED"\n' + ' * Use RELIABLE Qos in all publishers and subscriber to ensure any samples that are not ' + 'received are due to filtering\n' + ' * Verifies that both publishers discover and match the subscriber and vice-versa\n' + ' * Note that this test does not check whether the filtering happens in the publisher side or ' + 'the subscriber side. It only checks the middleware filters the samples somewhere.\n' + f'The test passes if the subscriber receives {MAX_SAMPLES_READ} samples of one color\n' + }, # PARTITION - 'Test_Partition_0' : [['-P -t Square -p "p1"', '-S -t Square -p "p1"'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Partition_1' : [['-P -t Square -p "p1"', '-S -t Square -p "p2"'], [ReturnCode.READER_NOT_MATCHED, ReturnCode.DATA_NOT_RECEIVED]], - 'Test_Partition_2' : [['-P -t Square -p "p2"', '-S -t Square -p "p1"'], [ReturnCode.READER_NOT_MATCHED, ReturnCode.DATA_NOT_RECEIVED]], + 'Test_Partition_0' : { + 'apps' : ['-P -t Square -p "p1"', '-S -t Square -p "p1"'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between publisher and subscriber using the same partition', + 'description' : 'Verifies communication between a publisher and a subscriber using the same partition\n' + ' * Configures the publisher and subscriber to use the PARTITION "p1"\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Partition_1' : { + 'apps' : ['-P -t Square -p "p1"', '-S -t Square -p "p2"'], + 'expected_codes' : [ReturnCode.READER_NOT_MATCHED, ReturnCode.DATA_NOT_RECEIVED], + 'title' : 'No communication between publisher and subscriber using different partitions', + 'description' : 'Verifies that there is no communication between a publisher using partition "p1" ' + 'and a subscriber using partition "p2"\n' + ' * Configures the publisher to use the PARTITION "p1"\n' + ' * Configures the subscriber to use the PARTITION "p2"\n' + 'The test passes if the publisher and subscriber do not discover each other\n' + }, + + 'Test_Partition_2' : { + 'apps' : ['-P -t Square -p "p1" -c BLUE', '-P -t Square -p "x1" -c RED', '-S -t Square -p "p*"'], + 'check_function' : test_color_receivers, + 'expected_codes' : [ReturnCode.OK, ReturnCode.READER_NOT_MATCHED, ReturnCode.RECEIVING_FROM_ONE], + 'title' : 'Usage of a partition expression to receive data only from the corresponding publishers', + 'description' : 'Verifies a subscription using a partition expression only receives data from the corresponding ' + 'publishers\n' + ' * Configures a subscriber with a PARTITION expression "p*" that allows only matching ' + 'publishers whose partition starts with "p"\n' + ' * Configures a first publisher to use PARTITION "p1" and "color" equal to "BLUE"\n' + ' * Configures a second publisher to use PARTITION "x1" and "color" equal to "RED"\n' + ' * Verifies that only the first publisher (PARTITION "p1") discovers and matches subscriber\n' + ' * Verifies that the second publisher (PARTITION "x1") does not match the subscriber\n' + f'The test passes if the subscriber receives {MAX_SAMPLES_READ} samples of one color (first publisher)\n' + }, # DURABILITY - 'Test_Durability_0' : [['-P -t Square -D v', '-S -t Square -D v'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Durability_1' : [['-P -t Square -D v', '-S -t Square -D l'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - 'Test_Durability_2' : [['-P -t Square -D v', '-S -t Square -D t'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - 'Test_Durability_3' : [['-P -t Square -D v', '-S -t Square -D p'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - - 'Test_Durability_4' : [['-P -t Square -D l', '-S -t Square -D v'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Durability_5' : [['-P -t Square -D l', '-S -t Square -D l'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Durability_6' : [['-P -t Square -D l', '-S -t Square -D t'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - 'Test_Durability_7' : [['-P -t Square -D l', '-S -t Square -D p'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - - 'Test_Durability_8' : [['-P -t Square -D t', '-S -t Square -D v'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Durability_9' : [['-P -t Square -D t', '-S -t Square -D l'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Durability_10': [['-P -t Square -D t', '-S -t Square -D t'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Durability_11': [['-P -t Square -D t', '-S -t Square -D p'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - - 'Test_Durability_12' : [['-P -t Square -D p', '-S -t Square -D v'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Durability_13' : [['-P -t Square -D p', '-S -t Square -D l'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Durability_14' : [['-P -t Square -D p', '-S -t Square -D t'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Durability_15' : [['-P -t Square -D p', '-S -t Square -D p'], [ReturnCode.OK, ReturnCode.OK]], + 'Test_Durability_0' : { + 'apps' : ['-P -t Square -D v', '-S -t Square -D v'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between VOLATILE publisher and VOLATILE subscriber', + 'description' : 'Verifies a volatile publisher communicates with a volatile subscriber\n' + ' * Configures the publisher with a VOLATILE durability\n' + ' * Configures the subscriber with a VOLATILE durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Durability_1' : { + 'apps' : ['-P -t Square -D v', '-S -t Square -D l'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility between VOLATILE publisher and TRANSIENT_LOCAL subscriber', + 'description' : 'Verifies a volatile publisher does not match with a transient local subscriber and report an ' + 'IncompatibleQos notification\n' + ' * Configures the publisher with VOLATILE durability\n' + ' * Configures the subscriber with TRANSIENT_LOCAL durability\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_Durability_2' : { + 'apps' : ['-P -t Square -D v', '-S -t Square -D t'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility between VOLATILE publisher and TRANSIENT subscriber', + 'description' : 'Verifies a volatile publisher does not match with a transient subscriber and report an ' + 'IncompatibleQos notification\n' + ' * Configures the publisher with VOLATILE durability\n' + ' * Configures the subscriber with TRANSIENT durability\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_Durability_3' : { + 'apps' : ['-P -t Square -D v', '-S -t Square -D p'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility between VOLATILE publisher and PERSISTENT subscriber', + 'description' : 'Verifies a volatile publisher does not match with a persistent subscriber and report an ' + 'IncompatibleQos notification\n' + ' * Configures the publisher with VOLATILE durability\n' + ' * Configures the subscriber with PERSISTENT durability\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_Durability_4' : { + 'apps' : ['-P -t Square -D l', '-S -t Square -D v'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between TRANSIENT_LOCAL publisher and VOLATILE subscriber', + 'description' : 'Verifies a transient local publisher communicates with a volatile subscriber\n' + ' * Configures the publisher with a TRANSIENT_LOCAL durability\n' + ' * Configures the subscriber with a VOLATILE durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Durability_5' : { + 'apps' : ['-P -t Square -D l', '-S -t Square -D l'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between TRANSIENT_LOCAL publisher and TRANSIENT_LOCAL subscriber', + 'description' : 'Verifies a transient local publisher communicates with a transient local subscriber\n' + ' * Configures the publisher with a TRANSIENT_LOCAL durability\n' + ' * Configures the subscriber with a TRANSIENT_LOCAL durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Durability_6' : { + 'apps' : ['-P -t Square -D l', '-S -t Square -D t'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility between TRANSIENT_LOCAL publisher and TRANSIENT subscriber', + 'description' : 'Verifies a transient local publisher does not match with a transient ' + 'subscriber and report an IncompatibleQos notification\n' + ' * Configures the publisher with TRANSIENT_LOCAL durability\n' + ' * Configures the subscriber with TRANSIENT durability\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_Durability_7' : { + 'apps' : ['-P -t Square -D l', '-S -t Square -D p'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility between TRANSIENT_LOCAL publisher and PERSISTENT subscriber', + 'description' : 'Verifies a transient local publisher does not match with a persistent ' + 'subscriber and report an IncompatibleQos notification\n' + ' * Configures the publisher with TRANSIENT_LOCAL durability\n' + ' * Configures the subscriber with PERSISTENT durability\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_Durability_8' : { + 'apps' : ['-P -t Square -D t', '-S -t Square -D v'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between TRANSIENT publisher and VOLATILE subscriber', + 'description' : 'Verifies a transient publisher communicates with a volatile subscriber\n' + ' * Configures the publisher with a TRANSIENT durability\n' + ' * Configures the subscriber with a VOLATILE durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Durability_9' : { + 'apps' : ['-P -t Square -D t', '-S -t Square -D l'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between TRANSIENT publisher and TRANSIENT_LOCAL subscriber', + 'description' : 'Verifies a transient publisher communicates with a transient local subscriber\n' + ' * Configures the publisher with a TRANSIENT durability\n' + ' * Configures the subscriber with a TRANSIENT_LOCAL durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Durability_10' : { + 'apps' : ['-P -t Square -D t', '-S -t Square -D t'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between TRANSIENT publisher and TRANSIENT subscriber', + 'description' : 'Verifies a transient publisher communicates with a transient subscriber\n' + ' * Configures the publisher with a TRANSIENT durability\n' + ' * Configures the subscriber with a TRANSIENT durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Durability_11' : { + 'apps' : ['-P -t Square -D t', '-S -t Square -D p'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility between TRANSIENT publisher and PERSISTENT subscriber', + 'description' : 'Verifies a transient publisher does not match with a persistent ' + 'subscriber and report an IncompatibleQos notification\n' + ' * Configures the publisher with TRANSIENT durability\n' + ' * Configures the subscriber with PERSISTENT durability\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_Durability_12' : { + 'apps' : ['-P -t Square -D p', '-S -t Square -D v'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between PERSISTENT publisher and VOLATILE subscriber', + 'description' : 'Verifies a persistent publisher communicates with a volatile subscriber\n' + ' * Configures the publisher with a PERSISTENT durability\n' + ' * Configures the subscriber with a VOLATILE durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Durability_13' : { + 'apps' : ['-P -t Square -D p', '-S -t Square -D l'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between PERSISTENT publisher and TRANSIENT_LOCAL subscriber', + 'description' : 'Verifies a persistent publisher communicates with a transient local subscriber\n' + ' * Configures the publisher with a PERSISTENT durability\n' + ' * Configures the subscriber with a TRANSIENT_LOCAL durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Durability_14' : { + 'apps' : ['-P -t Square -D p', '-S -t Square -D t'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between PERSISTENT publisher and TRANSIENT subscriber', + 'description' : 'Verifies a persistent publisher communicates with a transient subscriber\n' + ' * Configures the publisher with a PERSISTENT durability\n' + ' * Configures the subscriber with a TRANSIENT durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_Durability_15' : { + 'apps' : ['-P -t Square -D p', '-S -t Square -D p'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Communication between PERSISTENT publisher and PERSISTENT subscriber', + 'description' : 'Verifies a persistent publisher communicates with a persistent subscriber\n' + ' * Configures the publisher with a PERSISTENT durability\n' + ' * Configures the subscriber with a PERSISTENT durability\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, # Test durability behavior # This test sends all samples with reliable reliability and check that the - # first sample that the subscriber app reads are not the first one + # first sample that the subscriber app reads is not the first one # (the interoperability_test waits 1 second before creating the second # entity, the subscriber, if durability is set) - 'Test_Durability_16' : [['-P -t Square -z 0 -r -k 0 -D v -w', '-S -t Square -r -k 0 -D v'], [ReturnCode.OK, ReturnCode.OK], test_durability_volatile], + 'Test_Durability_16' : { + 'apps' : ['-P -t Square -z 0 -r -k 0 -D v -w', '-S -t Square -r -k 0 -D v'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : test_durability_volatile, + 'title' : 'Test the behavior of the VOLATILE durability', + 'description' : 'Verifies a volatile publisher and subscriber communicates and work as expected\n' + ' * Configures the publisher / subscriber with a VOLATILE durability\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + ' * Note that there is at least 1 second delay between the creation of each entity\n' + 'The test passes if the first sample the subscriber receives is not ' + 'the first sample that the publisher sent (by checking the "size" value).\n' + }, + # This test checks that the subscriber application reads the first sample that # have been sent by the publisher before creating the subscriber - 'Test_Durability_17' : [['-P -t Square -z 0 -r -k 0 -D t -w', '-S -t Square -r -k 0 -D t'], [ReturnCode.OK, ReturnCode.OK], test_durability_transient_local], + 'Test_Durability_17' : { + 'apps' : ['-P -t Square -z 0 -r -k 0 -D l -w', '-S -t Square -r -k 0 -D l'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : test_durability_transient_local, + 'title' : 'Test the behavior of the TRANSIENT_LOCAL durability', + 'description' : 'Verifies a transient local publisher and subscriber communicates and work as expected\n' + ' * Configures the publisher / subscriber with a TRANSIENT_LOCAL durability\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n' + ' * Note that there is at least 1 second delay between the creation of each entity\n' + 'The test passes if the first sample the subscriber receives is the first sample ' + 'that the publisher sent (by checking the "size" value is equal to 1).\n' + }, - # HISTORY - 'Test_History_0' : [['-P -t Square -k 3', '-S -t Square -k 3'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_History_1' : [['-P -t Square -k 3', '-S -t Square -k 0'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_History_2' : [['-P -t Square -k 0', '-S -t Square -k 3'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_History_3' : [['-P -t Square -k 0', '-S -t Square -k 0'], [ReturnCode.OK, ReturnCode.OK]], - - # OWNERSHIP - 'Test_Ownership_0': [['-P -t Square -s -1', '-S -t Square -s -1'], [ReturnCode.OK, ReturnCode.OK]], - 'Test_Ownership_1': [['-P -t Square -s -1', '-S -t Square -s 1'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - 'Test_Ownership_2': [['-P -t Square -s 3', '-S -t Square -s -1'], [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS]], - - # For Test_Ownership_[3|4|5|6]: each publisher application publishes samples - # with a different shapesize to allow the subscriber app to recognize from - # which Publisher is receiving the samples. - - # The DataReader should receive from both publisher apps because they - # publish different instances and the ownership is applied per instance. - 'Test_Ownership_3': [['-P -t Square -s 3 -r -k 0 -c BLUE -w -z 20', - '-P -t Square -s 4 -r -k 0 -c RED -w -z 30', - '-S -t Square -s 1 -r -k 0'], - [ReturnCode.OK, - ReturnCode.OK, - ReturnCode.RECEIVING_FROM_BOTH], - test_ownership_receivers], - - # The DataReader should only receive samples from the DataWriter with higher - # ownership. There may be the situation that the DataReader starts receiving - # samples from one DataWriter until another DataWriter with higher ownership - # strength is created. This should be handled by test_ownership_receivers(). - 'Test_Ownership_4': [['-P -t Square -s 3 -r -k 0 -c BLUE -w -z 20', - '-P -t Square -s 4 -r -k 0 -c BLUE -w -z 30', - '-S -t Square -s 1 -r -k 0'], - [ReturnCode.OK, - ReturnCode.OK, - ReturnCode.RECEIVING_FROM_ONE], - test_ownership_receivers], - - # The DataReader should receive from both publisher apps because they have - # shared ownership. - 'Test_Ownership_5': [['-P -t Square -s -1 -r -k 0 -c BLUE -w -z 20', - '-P -t Square -s -1 -r -k 0 -c RED -w -z 30', - '-S -t Square -s -1 -r -k 0'], - [ReturnCode.OK, - ReturnCode.OK, - ReturnCode.RECEIVING_FROM_BOTH], - test_ownership_receivers], - - # The DataReader should receive from both publisher apps because they have - # shared ownership. - 'Test_Ownership_6': [['-P -t Square -s -1 -r -k 0 -c BLUE -w -z 20', - '-P -t Square -s -1 -r -k 0 -c BLUE -w -z 30', - '-S -t Square -s -1 -r -k 0'], - [ReturnCode.OK, - ReturnCode.OK, - ReturnCode.RECEIVING_FROM_BOTH], - test_ownership_receivers], }