Fix/ Tests: check status before break in await_queue_edit() (#2411)

* check status before break

* fix AAAI test

* fix CVPR test

* fix emnlp and iclr tests

* fix ICML test

* more fixes to aaai and emnlp

* more fixes

* fix aaai test again

* fix EMNLP test again

* fix ICLR test again

* fix NeurIPS test

* fix venue request test

* fix venue submission test

* fix single blind test

* fix matching test

* fix ICML

* more fixes

* fix aaai tests

* fix workshop tests

* fix request form v2 test

* create deepcopy of submission content

* fix

* create deep copies instead

* Await process logs for ARR (#2417)

* Await process logs for ARR

* Temporary print logs

* Log id

* Reverse if statement

* Use equality

* Revert logging

* search profile by first and last

* remove venue configuration files

---------

Co-authored-by: Harold Rubio <[email protected]>

openreview/arr/helpers.py

@@ -1,4 +1,5 @@
 import openreview
+import time
 from enum import Enum
 from datetime import datetime, timedelta
 from openreview.venue import matching
@@ -820,8 +821,8 @@ def __init__(self, client_v2, venue, configuration_note, request_form_id, suppor
                 due_date=self.configuration_note.content.get('reviewer_checklist_due_date'),
                 exp_date=self.configuration_note.content.get('reviewer_checklist_exp_date'),
                 #start_date=self.venue.submission_stage.exp_date.strftime('%Y/%m/%d %H:%M'), Discuss with Harold
-                process='process/checklist_process.py',
-                preprocess='process/checklist_preprocess.py',
+                process='../arr/process/checklist_process.py',
+                preprocess='../arr/process/checklist_preprocess.py',
                 extend=ARRWorkflow._extend_reviewer_checklist
             ),
             ARRStage(
@@ -844,8 +845,8 @@ def __init__(self, client_v2, venue, configuration_note, request_form_id, suppor
                 due_date=self.configuration_note.content.get('ae_checklist_due_date'),
                 exp_date=self.configuration_note.content.get('ae_checklist_exp_date'),
                 #start_date=self.venue.submission_stage.exp_date.strftime('%Y/%m/%d %H:%M'), Discuss with Harold
-                process='process/checklist_process.py',
-                preprocess='process/checklist_preprocess.py',
+                process='../arr/process/checklist_process.py',
+                preprocess='../arr/process/checklist_preprocess.py',
                 extend=ARRWorkflow._extend_ae_checklist
             ),
             ARRStage(
@@ -864,7 +865,7 @@ def __init__(self, client_v2, venue, configuration_note, request_form_id, suppor
                 },
                 exp_date=self.configuration_note.content.get('form_expiration_date'),
                 #start_date=self.venue.submission_stage.exp_date.strftime('%Y/%m/%d %H:%M'), Discuss with Harold
-                process='process/verification_process.py',
+                process='../arr/process/verification_process.py',
                 extend=ARRWorkflow._extend_desk_reject_verification
             ),
             ARRStage(
@@ -1197,6 +1198,7 @@ class Participants(Enum):
     FIELD_READERS = {
     }
     UPDATE_WAIT_TIME = 5
+    PROCESS_LOG_TIMEOUT = 360 # 360 iterations for 30 minutes total
 
     def __init__(self,
         type = None,
@@ -1446,6 +1448,10 @@ def set_stage(self, client_v1, client, venue, invitation_builder, request_form_n
             self._post_new_dates(client, venue, current_invitation)
         else:
             self._set_field_readers(venue)
+            expected_statuses = ['error', 'ok']
+            current_log_count = len(
+                [log for log in client.get_process_logs(invitation=self.super_invitation_id) if log['status'] in expected_statuses]
+            )
 
             if self.type == ARRStage.Type.REGISTRATION_STAGE:
                 venue.registration_stages = [openreview.stages.RegistrationStage(**self.stage_arguments)]
@@ -1482,6 +1488,21 @@ def set_stage(self, client_v1, client, venue, invitation_builder, request_form_n
                 invitation_builder.set_process_invitation(self)
 
             if self.extend:
+                # Wait until previous changes are done
+                times_polled = 0
+                completed_logs = len(
+                    [log for log in client.get_process_logs(invitation=self.super_invitation_id) if log['status'] in expected_statuses]
+                )
+                print(f"check for {self.super_invitation_id} to be updated | original={current_log_count} current={completed_logs}")
+                while times_polled <= ARRStage.PROCESS_LOG_TIMEOUT and completed_logs <= current_log_count:
+                    print(f"waiting for {self.super_invitation_id} to be updated | {current_log_count}")
+                    time.sleep(ARRStage.UPDATE_WAIT_TIME)
+                    completed_logs = len(
+                        [log for log in client.get_process_logs(invitation=self.super_invitation_id) if log['status'] in expected_statuses]
+                    )
+                    times_polled += 1
+                print(f"finished waiting {completed_logs} > {current_log_count}")
+
                 self.extend(
                     client, venue, invitation_builder, request_form_note
                 )

openreview/arr/invitation.py

@@ -56,16 +56,9 @@ def __init__(self, venue, update_wait_time=5000):
 
     # Non-blocking custom stage with process/pre-process arguments
     def set_custom_stage_invitation(self, process_script = None, preprocess_script = None):
+        self.venue.custom_stage.process_path = process_script
+        self.venue.custom_stage.preprocess_path = preprocess_script
         invitation = self.venue_invitation_builder.set_custom_stage_invitation()
-        if not process_script and not preprocess_script:
-            return invitation
-
-        if process_script:
-            invitation.content['custom_stage_process_script'] = { 'value': self.get_process_content(process_script)}
-        if preprocess_script:
-            invitation.edit['invitation']['preprocess'] = self.get_process_content(preprocess_script)
-
-        self.save_invitation(invitation, replacement=False)
         return invitation
 
     def get_process_content(self, file_path):

openreview/stages/venue_stages.py

@@ -2,6 +2,7 @@
 import datetime
 from enum import Enum
 from . import default_content
+from copy import deepcopy
 
 SHORT_BUFFER_MIN = 30
 
@@ -225,7 +226,7 @@ def get_desk_rejected_submission_id(self, conference):
     def get_content(self, api_version='1', conference=None, venue_id=None):
 
         if api_version == '1':
-            content = default_content.submission.copy()
+            content = deepcopy(default_content.submission)
 
             if self.subject_areas:
                 content['subject_areas'] = {
@@ -255,7 +256,7 @@ def get_content(self, api_version='1', conference=None, venue_id=None):
                 }
 
         elif api_version == '2':
-            content = default_content.submission_v2.copy()
+            content = deepcopy(default_content.submission_v2)
 
             if self.subject_areas:
                 content['subject_areas'] = {
@@ -505,7 +506,7 @@ def __init__(self, name='Revision', start_date=None, due_date=None, additional_f
 
     def get_content(self, api_version='2', conference=None):
 
-        content = conference.submission_stage.get_content(api_version, conference).copy()
+        content = deepcopy(conference.submission_stage.get_content(api_version, conference))
 
         for field in self.remove_fields:
             if field in content:
@@ -655,7 +656,7 @@ def get_signatures(self, conference, number):
 
     def get_content(self, api_version='2', conference=None):
 
-        content = default_content.review_v2.copy()
+        content = deepcopy(default_content.review_v2)
 
         for field in self.remove_fields:
             if field in content:
@@ -798,7 +799,7 @@ def get_signatures(self, conference, number):
 
     def get_content(self, api_version='2', conference=None):
 
-        content = default_content.ethics_review_v2.copy()
+        content = deepcopy(default_content.ethics_review_v2)
 
         for field in self.remove_fields:
             if field in content:
@@ -881,7 +882,7 @@ def get_invitation_readers(self, conference, number):
 
     def get_content(self, api_version='2', conference=None):
 
-        content = default_content.rebuttal_v2.copy()
+        content = deepcopy(default_content.rebuttal_v2)
 
         for field in self.remove_fields:
             if field in content:
@@ -1246,7 +1247,7 @@ def get_content(self, api_version='2', conference=None):
         if self.content:
             return self.content
 
-        content = default_content.meta_review_v2.copy()
+        content = deepcopy(default_content.meta_review_v2)
 
         for field in self.remove_fields:
             if field in content:
@@ -1334,7 +1335,7 @@ def get_content(self, api_version='2', conference=None):
                 self.content['decision']['value']['param']['enum'] = self.options
             return self.content
 
-        content = default_content.decision_v2.copy()
+        content = deepcopy(default_content.decision_v2)
 
         for field in self.remove_fields:
             if field in content:
@@ -1471,6 +1472,8 @@ def __init__(self, name, reply_to, source, reply_type=ReplyType.REPLY, start_dat
         self.notify_readers = notify_readers
         self.email_template = email_template
         self.allow_de_anonymization = allow_de_anonymization
+        self.process_path = None
+        self.preprocess_path = None
 
     def get_invitees(self, conference, number):
         invitees = [conference.id]
@@ -1618,7 +1621,7 @@ def get_reply_type(self):
 
     def get_content(self, api_version='2', conference=None):
 
-        content = self.content.copy()
+        content = deepcopy(self.content)
 
         if conference:
             invitation_id = conference.get_invitation_id(self.name)

openreview/venue/invitation.py

@@ -1097,7 +1097,7 @@ def set_recruitment_invitation(self, committee_name, options):
                 invitation_content['overlap_committee_name'] = { 'delete': True }
                 invitation_content['overlap_committee_id'] = { 'delete': True  }
 
-        content = default_content.recruitment_v2.copy()
+        content = deepcopy(default_content.recruitment_v2)
 
         reduced_load = options.get('reduced_load_on_decline', None)
         reduced_load_dict = {}
@@ -1322,7 +1322,7 @@ def set_official_comment_invitation(self):
         comment_cdate = tools.datetime_millis(comment_stage.start_date if comment_stage.start_date else datetime.datetime.utcnow())
         comment_expdate = tools.datetime_millis(comment_stage.end_date) if comment_stage.end_date else None
 
-        content = default_content.comment_v2.copy()
+        content = deepcopy(default_content.comment_v2)
         invitees = comment_stage.get_invitees(self.venue, number='${3/content/noteNumber/value}')
 
         comment_readers = comment_stage.get_readers(self.venue, '${5/content/noteNumber/value}')
@@ -1483,7 +1483,7 @@ def set_public_comment_invitation(self):
         comment_cdate = tools.datetime_millis(comment_stage.start_date if comment_stage.start_date else datetime.datetime.utcnow())
         comment_expdate = tools.datetime_millis(comment_stage.end_date) if comment_stage.end_date else None
 
-        content = default_content.comment_v2.copy()
+        content = deepcopy(default_content.comment_v2)
 
         invitation = Invitation(id=public_comment_invitation,
             invitees=[venue_id],
@@ -2297,7 +2297,7 @@ def set_desk_rejection_invitation(self):
         cdate = tools.datetime_millis(expdate) if expdate else None
         exp_date = tools.datetime_millis(self.venue.submission_stage.due_date + datetime.timedelta(days = 90)) if self.venue.submission_stage.due_date else None
 
-        content = default_content.desk_reject_v2.copy()
+        content = deepcopy(default_content.desk_reject_v2)
 
         invitation = Invitation(id=self.venue.get_invitation_id(submission_stage.desk_rejection_name),
             invitees=[venue_id],
@@ -2755,14 +2755,15 @@ def set_custom_stage_invitation(self):
                 note_nonreaders = []
                 all_signatures = ['${7/content/replytoSignatures/value}']
 
+        process_path = 'process/custom_stage_process.py' if custom_stage.process_path is None else custom_stage.process_path
         invitation_content = {
             'source': { 'value': custom_stage_source },
             'reply_to': { 'value': custom_stage_replyto },
             'email_pcs': { 'value': custom_stage.email_pcs },
             'email_sacs': { 'value': custom_stage.email_sacs },
             'notify_readers': { 'value': custom_stage.notify_readers },
             'email_template': { 'value': custom_stage.email_template if custom_stage.email_template else '' },
-            'custom_stage_process_script': { 'value': self.get_process_content('process/custom_stage_process.py')}
+            'custom_stage_process_script': { 'value': self.get_process_content(process_path)}
         }
 
         invitation = Invitation(id=custom_stage_invitation_id,
@@ -2889,6 +2890,8 @@ def set_custom_stage_invitation(self):
             invitation.edit['invitation']['expdate'] = custom_stage_expdate
         if not custom_stage.multi_reply:
             invitation.edit['invitation']['maxReplies'] = 1
+        if custom_stage.preprocess_path:
+            invitation.edit['invitation']['preprocess'] = self.get_process_content(custom_stage.preprocess_path)
 
         self.save_invitation(invitation, replacement=False)
         return invitation
@@ -3328,7 +3331,7 @@ def set_paper_recruitment_invitation(self, invitation_id, committee_id, invited_
             'external_paper_committee_id': {'value': venue.get_committee_id(name=invited_committee_name, number='{number}') if assignment_title else ''}
         }
 
-        content = default_content.paper_recruitment_v2.copy()
+        content = deepcopy(default_content.paper_recruitment_v2)
 
         with open(os.path.join(os.path.dirname(__file__), 'webfield/paperRecruitResponseWebfield.js')) as webfield_reader:
             webfield_content = webfield_reader.read()

tests/conftest.py

@@ -79,14 +79,15 @@ def await_queue():
 
     @staticmethod
     def await_queue_edit(super_client, edit_id=None, invitation=None, count=1, error=False):
+        expected_status = 'error' if error else 'ok'
         while True:
             process_logs = super_client.get_process_logs(id=edit_id, invitation=invitation)
-            if len(process_logs) >= count:
+            if len(process_logs) >= count and all(process_log['status'] == expected_status for process_log in process_logs):
                 break
 
             time.sleep(0.5)
 
-        assert process_logs[0]['status'] == ('error' if error else 'ok'), process_logs[0]['log']
+        assert process_logs[0]['status'] == (expected_status), process_logs[0]['log']
 
 
     @staticmethod

tests/test_aaai_conference.py

@@ -402,6 +402,8 @@ def test_post_submission(self, client, openreview_client, test_client, helpers,
 
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, 'AAAI.org/2025/Conference/-/Post_Submission-0-1', count=3)
+
         submissions = openreview_client.get_notes(invitation='AAAI.org/2025/Conference/-/Submission', sort='number:asc')
         assert len(submissions) == 10
         assert ['AAAI.org/2025/Conference',
@@ -589,6 +591,8 @@ def test_ac_bidding(self, client, openreview_client, helpers, test_client, reque
 
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, 'AAAI.org/2025/Conference/-/Post_Submission-0-1', count=4)
+
         ac_client = openreview.api.OpenReviewClient(username = '[email protected]', password=helpers.strong_password)
         submissions = ac_client.get_notes(invitation='AAAI.org/2025/Conference/-/Submission', sort='number:asc')
         assert len(submissions) == 10
@@ -724,6 +728,8 @@ def test_review_stage(self, client, openreview_client, helpers, selenium, reques
         ))
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, 'AAAI.org/2025/Conference/-/First_Round_Review-0-1', count=1)
+
         invitation = openreview_client.get_invitation('AAAI.org/2025/Conference/Submission1/-/First_Round_Review')
 
         assert len(openreview_client.get_invitations(invitation='AAAI.org/2025/Conference/-/First_Round_Review')) == 10
@@ -817,6 +823,8 @@ def test_review_stage(self, client, openreview_client, helpers, selenium, reques
         ))
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, 'AAAI.org/2025/Conference/-/Second_Round_Review-0-1', count=1)
+
         assert len(openreview_client.get_invitations(invitation='AAAI.org/2025/Conference/-/Second_Round_Review')) == 9
         assert openreview_client.get_invitation('AAAI.org/2025/Conference/Submission1/-/Second_Round_Review')
 
@@ -877,6 +885,8 @@ def test_release_reviews(self, client, openreview_client, helpers, selenium, req
         ))
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, edit_id='AAAI.org/2025/Conference/-/Second_Round_Review-0-1', count=2)
+
         review_note = openreview_client.get_notes(invitation='AAAI.org/2025/Conference/Submission1/-/Second_Round_Review', sort='number:asc')[0]
         assert 'AAAI.org/2025/Conference/Submission1/Authors' in review_note.readers
 
@@ -922,6 +932,9 @@ def test_meta_review_stage(self, client, openreview_client, helpers, selenium, r
         ))
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, edit_id='AAAI.org/2025/Conference/-/Meta_Review-0-1', count=1)
+        helpers.await_queue_edit(openreview_client, edit_id='AAAI.org/2025/Conference/-/Meta_Review_AC_Revision-0-1', count=1)
+
         invitations = openreview_client.get_invitations(invitation='AAAI.org/2025/Conference/-/Meta_Review')
         assert len(invitations) == 9
         assert invitations[0].edit['note']['id']['param']['withInvitation'] == invitations[0].id
@@ -1030,6 +1043,8 @@ def test_comment_emails(self, client, openreview_client, helpers, request_page,
         ))
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, 'AAAI.org/2025/Conference/-/Official_Comment-0-1', count=1)
+
         assert comment_stage_note
 
         # Post comment as reviewer
@@ -1095,6 +1110,8 @@ def test_rebuttal_stage(self, client, openreview_client, helpers, selenium, requ
         ))
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, 'AAAI.org/2025/Conference/-/Rebuttal-0-1', count=1)
+
         assert len(openreview_client.get_invitations(invitation='AAAI.org/2025/Conference/-/Rebuttal')) == 9
 
         submissions = openreview_client.get_notes(invitation='AAAI.org/2025/Conference/-/Submission', sort='number:asc')
@@ -1152,6 +1169,8 @@ def test_release_rebuttals(self, client, openreview_client, helpers, selenium, r
         ))
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, 'AAAI.org/2025/Conference/-/Rebuttal-0-1', count=2)
+
         rebuttals = pc_client_v2.get_notes(invitation='AAAI.org/2025/Conference/Submission1/-/Rebuttal')
         assert len(rebuttals) == 1
         assert rebuttals[0].readers == [

tests/test_client.py

@@ -164,7 +164,7 @@ def test_search_profiles(self, client, helpers):
 
         assert '~Melisa_Bokk1' == client.search_profiles(ids = ['~Melisa_Bokk1'])[0].id
         assert '~Melisa_Bokk1' == client.search_profiles(confirmedEmails = ['[email protected]'])['[email protected]'].id
-        assert '~Melisa_Bokk1' == client.search_profiles(first = 'Melisa')[0].id
+        assert '~Melisa_Bokk1' == client.search_profiles(first = 'Melisa', last = 'Bokk')[0].id
         assert len(client.search_profiles(ids = ['~Melisa_Bok2'])) == 0
         assert len(client.search_profiles(emails = ['[email protected]'])) == 0
         assert len(client.search_profiles(first = 'Anna')) == 0

tests/test_cvpr_conference_v2.py

@@ -528,6 +528,8 @@ def test_review_rating_stage(self, client, openreview_client, helpers, test_clie
 
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, 'thecvf.com/CVPR/2024/Conference/-/Official_Review-0-1', count=1)
+
         comment_invitation = f'openreview.net/Support/-/Request{request_form.number}/Stage_Error_Status'
         error_comments = client.get_notes(invitation=comment_invitation, sort='tmdate')
         assert not error_comments or len(error_comments) == 0        
@@ -875,6 +877,8 @@ def test_secondary_ac_assignment(self, openreview_client, helpers, client):
 
         helpers.await_queue()
 
+        helpers.await_queue_edit(openreview_client, 'thecvf.com/CVPR/2024/Conference/-/Official_Comment-0-1', count=1)
+
         ## post a comment as a Secondary AC
         submission = openreview_client.get_notes(invitation='thecvf.com/CVPR/2024/Conference/-/Submission', number=4)[0]   
         anon_reviewers_group_id = ac1_client.get_groups(prefix=f'thecvf.com/CVPR/2024/Conference/Submission4/Secondary_Area_Chair_', signatory='[email protected]')[0].id