[#165] wait_for_empty_queue in big migration tests

Adds calls to assert certain things only after the delay queue has been
emptied and then allow iqdel -a to perform cleanup just in case. This
prevents some concurrency errors which led to things like failures to
delete delay rules from the catalog after successful completion of the
delayed rules (in spite of the test passing).

packaging/test_plugin_unified_storage_tiering.py

@@ -997,6 +997,8 @@ def test_put_gt_max_sql_rows(self):
                 sleep(5)
                 delay_assert_icommand(admin_session, ['ils', '-l', dirname], 'STDOUT_SINGLELINE', 'ufs1')
                 delay_assert_icommand(admin_session, ['ils', '-l', dirname], 'STDOUT_SINGLELINE', 'ufs0')
+                # Wait for the queue to be emptied and ensure that everything has tiered out from ufs0
+                wait_for_empty_queue(lambda: admin_session.assert_icommand_fail(['ils', '-l', dirname], 'STDOUT', 'ufs0'))
 
                 delay_assert_icommand(admin_session, 'iqdel -a')
 
@@ -1021,6 +1023,8 @@ def test_put_max_sql_rows(self):
                 sleep(5)
                 delay_assert_icommand(admin_session, ['ils', '-l', dirname], 'STDOUT_SINGLELINE', 'ufs1')
                 delay_assert_icommand(admin_session, ['ils', '-l', dirname], 'STDOUT_SINGLELINE', 'ufs0')
+                # Wait for the queue to be emptied and ensure that everything has tiered out from ufs0
+                wait_for_empty_queue(lambda: admin_session.assert_icommand_fail(['ils', '-l', dirname], 'STDOUT', 'ufs0'))
 
                 delay_assert_icommand(admin_session, 'iqdel -a')
 
@@ -1046,8 +1050,10 @@ def test_put_object_limit_lt(self):
                 sleep(5)
                 admin_session.assert_icommand('irule -r irods_rule_engine_plugin-unified_storage_tiering-instance -F /var/lib/irods/example_unified_tiering_invocation.r')
                 delay_assert_icommand(admin_session, ['ils', '-l', dirname], 'STDOUT_SINGLELINE', 'ufs1')
-                delay_assert_icommand(admin_session, ['ils', '-l', last_item_path], 'STDOUT_SINGLELINE', 'ufs0')
-                delay_assert_icommand(admin_session, ['ils', '-l', next_to_last_item_path], 'STDOUT_SINGLELINE', 'ufs0')
+                # Wait for the queue to be emptied and ensure that everything has tiered out from ufs0
+                wait_for_empty_queue(lambda: admin_session.assert_icommand(['ils', '-l', next_to_last_item_path], 'STDOUT', 'ufs1'))
+                # Ensure that the item which is 1 past the object_limit did not tier out from ufs0
+                admin_session.assert_icommand(['ils', '-l', last_item_path], 'STDOUT_SINGLELINE', 'ufs0')
 
                 delay_assert_icommand(admin_session, 'iqdel -a')