diff --git a/openreview/conference/invitation.py b/openreview/conference/invitation.py index b772b7872..9c24b798a 100644 --- a/openreview/conference/invitation.py +++ b/openreview/conference/invitation.py @@ -870,7 +870,7 @@ def __init__(self, conference, note): ] }, 'signatures': { - 'values-regex': comment_stage.get_signatures_regex(conference, note.number), + 'values-regex': '|'.join(comment_stage.get_signatures(conference, note.number)), 'description': 'How your identity will be displayed.' } } @@ -931,7 +931,7 @@ class PaperReviewInvitation(openreview.Invitation): def __init__(self, conference, note): review_stage = conference.review_stage - signature_regex = review_stage.get_signatures(conference, note.number) + signature_regex = '|'.join(review_stage.get_signatures(conference, note.number)) readers = review_stage.get_readers(conference, note.number) nonreaders = review_stage.get_nonreaders(conference, note.number) @@ -1013,7 +1013,7 @@ class PaperEthicsReviewInvitation(openreview.Invitation): def __init__(self, conference, note): ethics_review_stage = conference.ethics_review_stage - signature_regex = ethics_review_stage.get_signatures(conference, note.number) + signature_regex = '|'.join(ethics_review_stage.get_signatures(conference, note.number)) readers = ethics_review_stage.get_readers(conference, note.number) nonreaders = ethics_review_stage.get_nonreaders(conference, note.number) @@ -1370,7 +1370,7 @@ def __init__(self, conference, note): 'description': 'Who can edit this meta-review.' }, 'signatures': { - 'values-regex': meta_review_stage.get_signatures_regex(conference, note.number), + 'values-regex': '|'.join(meta_review_stage.get_signatures(conference, note.number)), 'description': 'How your identity will be displayed.' } } diff --git a/openreview/journal/invitation.py b/openreview/journal/invitation.py index 53570312b..dbc80975d 100644 --- a/openreview/journal/invitation.py +++ b/openreview/journal/invitation.py @@ -526,7 +526,11 @@ def set_reviewer_responsibility_invitation(self): 'duedate': '${2/content/duedate/value}', 'dateprocesses': [self.reviewer_reminder_process], 'edit': { - 'signatures': { 'param': { 'regex': '~.*' }}, + 'signatures': { + 'param': { + 'items': [{ 'prefix': '~.*' }] + } + }, 'readers': [venue_id, '${2/signatures}'], 'note': { 'forum': forum_note_id, @@ -667,7 +671,11 @@ def set_reviewer_assignment_acknowledgement_invitation(self): 'process': self.process_script, 'dateprocesses': [self.reviewer_ack_reminder_process], 'edit': { - 'signatures': { 'param': { 'regex': self.journal.get_reviewers_id(number='${5/content/noteNumber/value}', anon=True) }}, + 'signatures': { + 'param': { + 'items': [{ 'prefix': self.journal.get_reviewers_id(number='${7/content/noteNumber/value}', anon=True) }] + } + }, 'readers': [venue_id, '${2/signatures}'], 'note': { 'forum': '${4/content/noteId/value}', @@ -739,7 +747,13 @@ def set_reviewer_report_invitation(self): writers=[venue_id], signatures=[venue_id], edit={ - 'signatures': { 'param': { 'regex': '~.*|' + editors_in_chief_id }}, + 'signatures': { + 'param': { + 'items': [ + { 'prefix': '~.*', 'optional': True }, + { 'value': editors_in_chief_id, 'optional': True }] + } + }, 'readers': [venue_id, '${2/signatures}'], 'note': { 'id': { @@ -793,7 +807,8 @@ def set_reviewer_report_invitation(self): 'type': 'string', 'maxLength': 200000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -823,7 +838,11 @@ def set_submission_invitation(self): writers=[venue_id], signatures=[editor_in_chief_id], edit={ - 'signatures': { 'param': { 'regex': '~.*' }}, + 'signatures': { + 'param': { + 'items': [{ 'prefix': '~.*' }] + } + }, 'readers': [ venue_id, self.journal.get_action_editors_id(number='${2/note/number}'), self.journal.get_authors_id(number='${2/note/number}')], 'writers': [ venue_id ], 'note': { @@ -890,7 +909,8 @@ def set_submission_invitation(self): 'type': 'file', 'extensions': ['zip', 'pdf'], 'maxSize': 100, - "optional": True + 'optional': True, + 'deletable': True } }, "description": "All supplementary material must be self-contained and zipped into a single file. Note that supplementary material will be visible to reviewers and the public throughout and after the review period, and ensure all material is anonymized. The maximum file size is 100MB.", @@ -902,7 +922,8 @@ def set_submission_invitation(self): 'param': { 'type': "string", 'regex': 'https:\/\/openreview\.net\/forum\?id=.*', - 'optional': True + 'optional': True, + 'deletable': True } }, 'description': f'If a version of this submission was previously rejected by {short_name}, give the OpenReview link to the original {short_name} submission (which must still be anonymous) and describe the changes below.', @@ -915,6 +936,7 @@ def set_submission_invitation(self): 'maxLength': 5000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } }, @@ -1052,6 +1074,7 @@ def set_ae_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1113,6 +1136,7 @@ def set_ae_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1173,6 +1197,7 @@ def set_ae_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1235,6 +1260,7 @@ def set_ae_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1297,6 +1323,7 @@ def set_ae_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1364,6 +1391,7 @@ def set_ae_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1424,6 +1452,7 @@ def set_ae_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1734,6 +1763,7 @@ def set_reviewer_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1795,6 +1825,7 @@ def set_reviewer_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1862,6 +1893,7 @@ def set_reviewer_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -1934,6 +1966,7 @@ def set_reviewer_assignment(self, assignment_delay): 'label': { 'param': { 'optional': True, + 'deletable': True, 'minLength': 1 } } @@ -2168,7 +2201,14 @@ def set_review_approval_invitation(self): 'process': self.process_script, 'dateprocesses': [self.ae_reminder_process], 'edit': { - 'signatures': { 'param': { 'regex': f"{editors_in_chief_id}|{self.journal.get_action_editors_id(number='${5/content/noteNumber/value}')}" }}, + 'signatures': { + 'param': { + 'items': [ + { 'value': editors_in_chief_id, 'optional': True }, + { 'value': self.journal.get_action_editors_id(number='${7/content/noteNumber/value}'), 'optional': True } + ] + } + }, 'readers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}')], 'writers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}')], 'note': { @@ -2204,6 +2244,7 @@ def set_review_approval_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -2312,6 +2353,7 @@ def set_desk_rejection_approval_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -2372,7 +2414,11 @@ def set_withdrawal_invitation(self): 'maxReplies': 1, 'process': self.process_script, 'edit': { - 'signatures': { 'param': { 'regex': self.journal.get_authors_id(number='${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [{ 'value': self.journal.get_authors_id(number='${7/content/noteNumber/value}') }] + } + }, 'readers': [ editors_in_chief_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), self.journal.get_reviewers_id(number='${4/content/noteNumber/value}'), self.journal.get_authors_id(number='${4/content/noteNumber/value}') ], 'writers': [ venue_id, self.journal.get_authors_id(number='${4/content/noteNumber/value}')], 'note': { @@ -2404,6 +2450,7 @@ def set_withdrawal_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -2477,6 +2524,7 @@ def set_desk_rejection_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -2535,7 +2583,11 @@ def set_retraction_invitation(self): 'maxReplies': 1, 'process': self.process_script, 'edit': { - 'signatures': { 'param': { 'regex': self.journal.get_authors_id(number='${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [ { 'value': self.journal.get_authors_id(number='${7/content/noteNumber/value}') }] + } + }, 'readers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), self.journal.get_authors_id(number='${4/content/noteNumber/value}') ], 'writers': [ venue_id, self.journal.get_authors_id(number='${4/content/noteNumber/value}')], 'note': { @@ -3108,6 +3160,7 @@ def set_accepted_invitation(self): 'type': 'string[]', 'enum': self.journal.get_certifications() + ([self.journal.get_expert_reviewer_certification()] if self.journal.has_expert_reviewers() else []), 'optional': True, + 'deletable': True, 'input': 'select' } } @@ -3121,7 +3174,8 @@ def set_accepted_invitation(self): 'param': { 'fieldName': f'Authors that are also {self.journal.short_name} Expert Reviewers', 'type': 'string[]', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -3397,7 +3451,14 @@ def set_review_invitation(self): 'process': self.process_script, 'dateprocesses': [self.reviewer_reminder_process_with_EIC], 'edit': { - 'signatures': { 'param': { 'regex': f"{self.journal.get_reviewers_id(number='${5/content/noteNumber/value}', anon=True)}.*|{self.journal.get_action_editors_id(number='${5/content/noteNumber/value}')}" }}, + 'signatures': { + 'param': { + 'items': [ + { 'prefix': self.journal.get_reviewers_id(number='${7/content/noteNumber/value}', anon=True), 'optional': True }, + { 'value': self.journal.get_action_editors_id(number='${7/content/noteNumber/value}'), 'optional': True } + ] + } + }, 'readers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), '${2/signatures}'], 'writers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), '${2/signatures}'], 'note': { @@ -3618,7 +3679,14 @@ def set_official_recommendation_invitation(self): 'script': self.get_super_dateprocess_content('cdate_script') }, self.reviewer_reminder_process_with_EIC], 'edit': { - 'signatures': { 'param': { 'regex': f"{self.journal.get_reviewers_id(number='${5/content/noteNumber/value}', anon=True)}.*|{self.journal.get_action_editors_id(number='${5/content/noteNumber/value}')}" }}, + 'signatures': { + 'param': { + 'items': [ + { 'prefix': self.journal.get_reviewers_id(number='${7/content/noteNumber/value}', anon=True), 'optional': True }, + { 'value': self.journal.get_action_editors_id(number='${7/content/noteNumber/value}'), 'optional': True } + ] + } + }, 'readers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), '${2/signatures}'], 'nonreaders': [ self.journal.get_authors_id(number='${4/content/noteNumber/value}') ], 'writers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), '${2/signatures}'], @@ -3690,6 +3758,7 @@ def set_official_recommendation_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -3708,6 +3777,7 @@ def set_official_recommendation_invitation(self): 'type': 'string[]', 'enum': self.journal.get_certifications(), 'optional': True, + 'deletable': True, 'input': 'checkbox' } } @@ -3768,7 +3838,11 @@ def set_solicit_review_invitation(self): 'process': self.process_script, 'preprocess': self.preprocess_script, 'edit': { - 'signatures': { 'param': { 'regex': f'~.*' }}, + 'signatures': { + 'param': { + 'items': [{ 'prefix': '~.*' }] + } + }, 'readers': [ editors_in_chief_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), '${2/signatures}'], 'nonreaders': [ self.journal.get_authors_id(number='${4/content/noteNumber/value}') ], 'writers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), '${2/signatures}'], @@ -3814,6 +3888,7 @@ def set_solicit_review_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -3934,6 +4009,7 @@ def set_solicit_review_approval_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -4000,7 +4076,13 @@ def set_revision_invitation(self): 'deletable': True } }, - 'signatures': { 'param': { 'regex': f"{self.journal.get_authors_id(number='${5/content/noteNumber/value}')}|{editors_in_chief_id}" }}, + 'signatures': { + 'param': { + 'items': [ + { 'value': self.journal.get_authors_id(number='${7/content/noteNumber/value}'), 'optional': True }, + { 'value': editors_in_chief_id, 'optional': True }] + } + }, 'readers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), self.journal.get_reviewers_id(number='${4/content/noteNumber/value}'), self.journal.get_authors_id(number='${4/content/noteNumber/value}')], 'writers': [ venue_id, self.journal.get_authors_id(number='${4/content/noteNumber/value}')], 'note': { @@ -4022,7 +4104,8 @@ def set_revision_invitation(self): 'type': "string", 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } }, 'description': 'Abstract of paper. Add TeX formulas using the following formats: $In-line Formula$ or $$Block Formula$$.', @@ -4057,7 +4140,8 @@ def set_revision_invitation(self): 'param': { 'type': "string", 'regex': 'https:\/\/openreview\.net\/forum\?id=.*', - 'optional': True + 'optional': True, + 'deletable': True } }, 'description': f'If a version of this submission was previously rejected by {short_name}, give the OpenReview link to the original {short_name} submission (which must still be anonymous) and describe the changes below.', @@ -4070,6 +4154,7 @@ def set_revision_invitation(self): 'maxLength': 5000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } }, @@ -4202,7 +4287,11 @@ def set_comment_invitation(self): 'writers': [venue_id], 'signatures': [venue_id], 'edit': { - 'signatures': { 'param': { 'regex': f'~.*' }}, + 'signatures': { + 'param': { + 'items': [{ 'prefix': '~.*' }] + } + }, 'readers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), '${2/signatures}'], 'writers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}'), '${2/signatures}'], 'note': { @@ -4237,7 +4326,8 @@ def set_comment_invitation(self): 'type': 'string', 'maxLength': 500, 'input': 'text', - 'optional': True + 'optional': True, + 'deletable': True, } } }, @@ -4278,7 +4368,15 @@ def set_comment_invitation(self): 'writers': [venue_id], 'signatures': [venue_id], 'edit': { - 'signatures': { 'param': { 'regex': f"{editors_in_chief_id}|{self.journal.get_action_editors_id(number='${5/content/noteNumber/value}')}|{self.journal.get_reviewers_id(number='${5/content/noteNumber/value}', anon=True)}.*|{self.journal.get_authors_id(number='${5/content/noteNumber/value}')}" }}, + 'signatures': { + 'param': { + 'items': [ + { 'value': editors_in_chief_id, 'optional': True }, + { 'value': self.journal.get_action_editors_id(number='${7/content/noteNumber/value}'), 'optional': True }, + { 'prefix': self.journal.get_reviewers_id(number='${7/content/noteNumber/value}', anon=True), 'optional': True }, + { 'value': self.journal.get_authors_id(number='${7/content/noteNumber/value}'), 'optional': True } ] + } + }, 'readers': [ venue_id, '${2/signatures}' ], 'writers': [ venue_id, '${2/signatures}' ], 'note': { @@ -4317,7 +4415,8 @@ def set_comment_invitation(self): 'type': 'string', 'maxLength': 500, 'input': 'text', - 'optional': True + 'optional': True, + 'deletable': True, } } }, @@ -4349,7 +4448,14 @@ def set_comment_invitation(self): 'writers': [venue_id], 'signatures': [venue_id], 'edit': { - 'signatures': { 'param': { 'regex': f"{editors_in_chief_id}|{self.journal.get_action_editors_id(number='${5/content/noteNumber/value}')}" }}, + 'signatures': { + 'param': { + 'items': [ + { 'value': editors_in_chief_id, 'optional': True }, + { 'value': self.journal.get_action_editors_id(number='${7/content/noteNumber/value}'), 'optional': True }, + ] + } + }, 'readers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}')], 'writers': [ venue_id, self.journal.get_action_editors_id(number='${4/content/noteNumber/value}')], 'note': { @@ -4361,11 +4467,6 @@ def set_comment_invitation(self): 'forum': '${4/content/noteId/value}', 'readers': ['everyone'], 'writers': [venue_id, self.journal.get_action_editors_id(number='${5/content/noteNumber/value}')], - # 'signatures': { - # 'param': { - # 'regex': '~.*' - # } - # }, 'content': { 'title': { 'order': 1, @@ -4375,7 +4476,8 @@ def set_comment_invitation(self): 'type': 'string', 'maxLength': 500, 'input': 'text', - 'optional': True + 'optional': True, + 'deletable': True } }, 'readers': [ venue_id, self.journal.get_action_editors_id(number='${7/content/noteNumber/value}'), '${5/signatures}'] @@ -4615,7 +4717,8 @@ def set_decision_invitation(self): 'The authors may consider submitting a major revision at a later time.' ], 'input': 'checkbox', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -4636,6 +4739,7 @@ def set_decision_invitation(self): 'type': 'string[]', 'enum': self.journal.get_certifications(), 'optional': True, + 'deletable': True, 'input': 'checkbox' } } @@ -4738,6 +4842,7 @@ def set_decision_approval_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -5114,7 +5219,8 @@ def set_camera_ready_revision_invitation(self): 'type': 'file', 'extensions': ['zip', 'pdf'], 'maxSize': 100, - "optional": True + 'optional': True, + 'deletable': True } }, "description": "All supplementary material must be self-contained and zipped into a single file. Note that supplementary material will be visible to reviewers and the public throughout and after the review period, and ensure all material is anonymized. The maximum file size is 100MB.", @@ -5126,7 +5232,8 @@ def set_camera_ready_revision_invitation(self): 'param': { 'type': 'string', 'regex': 'https:\/\/openreview\.net\/forum\?id=.*', - 'optional': True + 'optional': True, + 'deletable': True } }, 'description': f'If a version of this submission was previously rejected by {short_name}, give the OpenReview link to the original {short_name} submission (which must still be anonymous) and describe the changes below.', @@ -5139,6 +5246,7 @@ def set_camera_ready_revision_invitation(self): 'maxLength': 5000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } @@ -5177,7 +5285,8 @@ def set_camera_ready_revision_invitation(self): 'param': { 'type': 'string', "regex": 'https?://.+', - 'optional': True + 'optional': True, + 'deletable': True } } }, @@ -5188,7 +5297,8 @@ def set_camera_ready_revision_invitation(self): 'param': { 'type': 'string', "regex": 'https?://.+', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -5401,7 +5511,8 @@ def set_eic_revision_invitation(self): 'type': 'file', 'extensions': ['zip', 'pdf'], 'maxSize': 100, - "optional": True + 'optional': True, + 'deletable': True } }, "description": "All supplementary material must be self-contained and zipped into a single file. Note that supplementary material will be visible to reviewers and the public throughout and after the review period, and ensure all material is anonymized. The maximum file size is 100MB.", @@ -5412,7 +5523,8 @@ def set_eic_revision_invitation(self): 'param': { 'type': 'string', 'regex': 'https:\/\/openreview\.net\/forum\?id=.*', - 'optional': True + 'optional': True, + 'deletable': True } }, 'description': f'If a version of this submission was previously rejected by {short_name}, give the OpenReview link to the original {short_name} submission (which must still be anonymous) and describe the changes below.', @@ -5425,6 +5537,7 @@ def set_eic_revision_invitation(self): 'maxLength': 5000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } @@ -5463,7 +5576,8 @@ def set_eic_revision_invitation(self): 'param': { 'type': 'string', "regex": 'https?://.+', - 'optional': True + 'optional': True, + 'deletable': True } } }, @@ -5474,7 +5588,8 @@ def set_eic_revision_invitation(self): 'param': { 'type': 'string', "regex": 'https?://.+', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -5742,7 +5857,8 @@ def set_assignment_configuration_invitation(self): 'param': { 'type': 'string', #'default': '{}/-/Custom_User_Demands'.format(self.match_group.id), - 'optional': True + 'optional': True, + 'deletable': True } } }, @@ -5795,6 +5911,7 @@ def set_assignment_configuration_invitation(self): 'type': 'string', 'regex': '.*', 'optional': True, + 'deletable': True, 'hidden': True } } @@ -5808,6 +5925,7 @@ def set_assignment_configuration_invitation(self): 'enum': ['Yes', 'No'], 'input': 'radio', 'optional': True, + 'deletable': True, 'default': 'Yes' } } @@ -5820,6 +5938,7 @@ def set_assignment_configuration_invitation(self): 'type': 'string', 'regex': r'[-+]?[0-9]*\.?[0-9]*', 'optional': True, + 'deletable': True, 'default': '1' } } @@ -5832,6 +5951,7 @@ def set_assignment_configuration_invitation(self): 'type': 'string', 'regex': r'[-+]?[0-9]*\.?[0-9]*', 'optional': True, + 'deletable': True, 'default': '', 'hidden': True } diff --git a/openreview/journal/journal.py b/openreview/journal/journal.py index b8873f58e..cbea7ad90 100644 --- a/openreview/journal/journal.py +++ b/openreview/journal/journal.py @@ -979,6 +979,7 @@ def update_solicit_review(self, note_number, invitation): "maxLength": 200000, "input": "textarea", "optional": True, + 'deletable': True, "markdown": True } diff --git a/openreview/stages/default_content.py b/openreview/stages/default_content.py index 467d67ab9..8be49d99c 100644 --- a/openreview/stages/default_content.py +++ b/openreview/stages/default_content.py @@ -38,7 +38,8 @@ 'param': { 'type': 'string', 'maxLength': 500, - 'optional': True + 'optional': True, + 'deletable': True } } }, @@ -131,6 +132,7 @@ 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -221,19 +223,20 @@ 'order': 3, 'value': { 'param': { - 'type': 'string', + 'type': 'integer', 'enum': [ - '10: Top 5% of accepted papers, seminal paper', - '9: Top 15% of accepted papers, strong accept', - '8: Top 50% of accepted papers, clear accept', - '7: Good paper, accept', - '6: Marginally above acceptance threshold', - '5: Marginally below acceptance threshold', - '4: Ok but not good enough - rejection', - '3: Clear rejection', - '2: Strong rejection', - '1: Trivial or wrong' - ] + { 'value': 10, 'description': '10: Top 5% of accepted papers, seminal paper' }, + { 'value': 9, 'description': '9: Top 15% of accepted papers, strong accept' }, + { 'value': 8, 'description': '8: Top 50% of accepted papers, clear accept' }, + { 'value': 7, 'description': '7: Good paper, accept' }, + { 'value': 6, 'description': '6: Marginally above acceptance threshold' }, + { 'value': 5, 'description': '5: Marginally below acceptance threshold' }, + { 'value': 4, 'description': '4: Ok but not good enough - rejection' }, + { 'value': 3, 'description': '3: Clear rejection' }, + { 'value': 2, 'description': '2: Strong rejection' }, + { 'value': 1, 'description': '1: Trivial or wrong' } + ], + 'input': 'radio' } } }, @@ -241,14 +244,15 @@ 'order': 4, 'value': { 'param': { - 'type': 'string', + 'type': 'integer', 'enum': [ - '5: The reviewer is absolutely certain that the evaluation is correct and very familiar with the relevant literature', - '4: The reviewer is confident but not absolutely certain that the evaluation is correct', - '3: The reviewer is fairly confident that the evaluation is correct', - '2: The reviewer is willing to defend the evaluation, but it is quite likely that the reviewer did not understand central parts of the paper', - '1: The reviewer\'s evaluation is an educated guess' - ] + { 'value': 5, 'description': '5: The reviewer is absolutely certain that the evaluation is correct and very familiar with the relevant literature' }, + { 'value': 4, 'description': '4: The reviewer is confident but not absolutely certain that the evaluation is correct' }, + { 'value': 3, 'description': '3: The reviewer is fairly confident that the evaluation is correct' }, + { 'value': 2, 'description': '2: The reviewer is willing to defend the evaluation, but it is quite likely that the reviewer did not understand central parts of the paper' }, + { 'value': 1, 'description': '1: The reviewer\'s evaluation is an educated guess' } + ], + 'input': 'radio' } } } @@ -298,7 +302,8 @@ 'maxLength': 200000, 'markdown': True, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -356,7 +361,8 @@ 'Accept (Oral)', 'Accept (Poster)', 'Reject' - ] + ], + 'input': 'radio' } } }, @@ -477,7 +483,8 @@ 'fieldName': 'TL;DR', 'type': 'string', 'maxLength': 250, - 'optional': True + 'optional': True, + 'deletable': True } } }, @@ -590,6 +597,7 @@ 'type': 'string', 'maxLength': 5000, 'optional': True, + 'deletable': True, 'input': 'textarea' } } @@ -692,6 +700,7 @@ 'type': 'string', 'maxLength': 5000, 'optional': True, + 'deletable': True, 'input': 'textarea' } } @@ -733,7 +742,8 @@ 'Accept (Oral)', 'Accept (Poster)', 'Reject' - ] + ], + 'input': 'radio' } } }, @@ -744,7 +754,8 @@ 'type': 'string', 'markdown': True, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } diff --git a/openreview/stages/venue_stages.py b/openreview/stages/venue_stages.py index 0d8ed980f..ab46ea95d 100644 --- a/openreview/stages/venue_stages.py +++ b/openreview/stages/venue_stages.py @@ -607,9 +607,9 @@ def get_nonreaders(self, conference, number): def get_signatures(self, conference, number): if self.allow_de_anonymization: - return '~.*' + return ['~.*'] - return conference.get_anon_reviewer_id(number=number, anon_id='.*') + return [conference.get_anon_reviewer_id(number=number, anon_id='.*')] def get_content(self, api_version='2', conference=None): @@ -743,7 +743,7 @@ def get_nonreaders(self, conference, number): return [conference.get_authors_id(number = number)] def get_signatures(self, conference, number): - return conference.get_anon_reviewer_id(number=number, anon_id='.*', name=conference.ethics_reviewers_name) + '|' + conference.get_program_chairs_id() + return [conference.get_anon_reviewer_id(number=number, anon_id='.*', name=conference.ethics_reviewers_name), conference.get_program_chairs_id()] def get_content(self, api_version='2', conference=None): @@ -940,7 +940,33 @@ def __init__(self, self.readers = readers self.invitees = invitees - def get_readers(self, conference, number): + def get_readers(self, conference, number, api_version='1'): + + readers = [{ 'value': conference.get_program_chairs_id(), 'optional': False }] + if api_version == '2' and self.reader_selection: + if conference.use_senior_area_chairs and self.Readers.SENIOR_AREA_CHAIRS_ASSIGNED in self.readers: + readers.append({ 'value': conference.get_senior_area_chairs_id(number), 'optional': False }) + + if self.allow_public_comments or self.Readers.EVERYONE in self.readers: + readers.append({ 'value': 'everyone', 'optional': True }) + + if self.reader_selection: + readers.append({ 'prefix': conference.get_anon_reviewer_id(number=number, anon_id='.*'), 'optional': True }) + + if conference.use_area_chairs and self.Readers.AREA_CHAIRS_ASSIGNED in self.readers: + readers.append({ 'value': conference.get_area_chairs_id(number), 'optional': True }) + + if self.Readers.REVIEWERS_ASSIGNED in self.readers: + readers.append({ 'value': conference.get_reviewers_id(number), 'optional': True }) + + if self.Readers.REVIEWERS_SUBMITTED in self.readers: + readers.append({ 'value': conference.get_reviewers_id(number) + '/Submitted', 'optional': True }) + + if self.Readers.AUTHORS in self.readers: + readers.append({ 'value': conference.get_authors_id(number), 'optional': True }) + + return readers + readers = [conference.get_program_chairs_id()] if self.allow_public_comments or self.Readers.EVERYONE in self.readers: @@ -966,7 +992,7 @@ def get_readers(self, conference, number): return readers - def get_signatures_regex(self, conference, number): + def get_signatures(self, conference, number): committee = [conference.get_program_chairs_id()] @@ -985,7 +1011,7 @@ def get_signatures_regex(self, conference, number): if self.Readers.AUTHORS in self.invitees: committee.append(conference.get_authors_id(number)) - return '|'.join(committee) + return committee def get_invitees(self, conference, number): invitees = [conference.get_id(), conference.support_user] @@ -1088,14 +1114,14 @@ def get_nonreaders(self, conference, number): return [conference.get_authors_id(number = number)] - def get_signatures_regex(self, conference, number): + def get_signatures(self, conference, number): committee = [conference.get_program_chairs_id()] if conference.use_area_chairs: committee.append(conference.get_anon_area_chair_id(number=number, anon_id='.*')) - return '|'.join(committee) + return committee def get_content(self, api_version='2', conference=None): @@ -1364,7 +1390,7 @@ def get_readers(self, conference, number): return readers - def get_signatures_regex(self, conference, number): + def get_signatures(self, conference, number): committee = [conference.get_program_chairs_id()] if conference.use_senior_area_chairs and self.Participants.SENIOR_AREA_CHAIRS_ASSIGNED in self.invitees: @@ -1385,7 +1411,7 @@ def get_signatures_regex(self, conference, number): if conference.use_ethics_reviewers and self.Participants.ETHICS_REVIEWERS_ASSIGNED in self.invitees: committee.append(conference.get_anon_reviewer_id(number=number, anon_id='.*', name=conference.ethics_reviewers_name)) - return '|'.join(committee) + return committee def get_source_submissions(self): diff --git a/openreview/venue/invitation.py b/openreview/venue/invitation.py index dfe09f930..2d4bfc41f 100644 --- a/openreview/venue/invitation.py +++ b/openreview/venue/invitation.py @@ -134,7 +134,14 @@ def set_submission_invitation(self): duedate=tools.datetime_millis(submission_stage.due_date) if submission_stage.due_date else None, expdate = tools.datetime_millis(submission_stage.exp_date) if submission_stage.exp_date else None, edit = { - 'signatures': { 'param': { 'regex': f'~.*|{self.venue.get_program_chairs_id()}' } }, + 'signatures': { + 'param': { + 'items': [ + { 'prefix': '~.*', 'optional': True }, + { 'value': self.venue.get_program_chairs_id(), 'optional': True } + ] + } + }, 'readers': edit_readers, 'writers': [venue_id, '${2/note/content/authorids/value}'], 'ddate': { @@ -235,7 +242,8 @@ def set_post_submission_invitation(self): 'type': 'string', 'maxLength': 200000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -355,7 +363,11 @@ def set_review_invitation(self): funcs['process'](client, edit, invitation) ''', 'edit': { - 'signatures': { 'param': { 'regex': review_stage.get_signatures(self.venue, '${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [ { 'prefix': s, 'optional': True } if '.*' in s else { 'value': s, 'optional': True } for s in review_stage.get_signatures(self.venue, '${7/content/noteNumber/value}')] + } + }, 'readers': ['${2/note/readers}'], 'nonreaders': review_stage.get_nonreaders(self.venue, '${4/content/noteNumber/value}'), 'writers': [venue_id], @@ -510,7 +522,11 @@ def set_review_rebuttal_invitation(self): funcs['process'](client, edit, invitation) ''', 'edit': { - 'signatures': { 'param': { 'regex': self.venue.get_authors_id(number='${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [{ 'value': self.venue.get_authors_id(number='${7/content/noteNumber/value}') }] + } + }, 'readers': review_rebuttal_stage.get_invitation_readers(self.venue, '${4/content/noteNumber/value}'), 'writers': [venue_id], 'note': { @@ -618,7 +634,11 @@ def set_meta_review_invitation(self): funcs['process'](client, edit, invitation) ''', 'edit': { - 'signatures': { 'param': { 'regex': meta_review_stage.get_signatures_regex(self.venue, '${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [ { 'prefix': s, 'optional': True } if '.*' in s else { 'value': s, 'optional': True } for s in meta_review_stage.get_signatures(self.venue, '${7/content/noteNumber/value}')] + } + }, 'readers': meta_review_stage.get_readers(self.venue, '${4/content/noteNumber/value}'), 'nonreaders': meta_review_stage.get_nonreaders(self.venue, '${4/content/noteNumber/value}'), 'writers': [venue_id], @@ -702,7 +722,13 @@ def set_meta_review_invitation(self): 'maxReplies': 1, 'cdate': meta_review_cdate, 'edit': { - 'signatures': { 'param': { 'regex': self.venue.get_senior_area_chairs_id(number='${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [ + { 'value': self.venue.get_senior_area_chairs_id(number='${7/content/noteNumber/value}') } + ] + } + }, 'readers': meta_review_stage.get_readers(self.venue, '${4/content/noteNumber/value}'), 'nonreaders': meta_review_stage.get_nonreaders(self.venue, '${4/content/noteNumber/value}'), 'writers': [venue_id], @@ -764,7 +790,8 @@ def set_recruitment_invitation(self, committee_name, options): 'type': 'string', 'enum': reduced_load, 'input': 'select', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -943,7 +970,7 @@ def set_official_comment_invitation(self): if comment_stage.reader_selection: comment_readers = { 'param': { - 'enum': comment_stage.get_readers(self.venue, '${7/content/noteNumber/value}') + 'items': comment_stage.get_readers(self.venue, '${8/content/noteNumber/value}', api_version='2'), } } @@ -1011,7 +1038,11 @@ def set_official_comment_invitation(self): funcs['process'](client, edit, invitation) ''', 'edit': { - 'signatures': { 'param': { 'regex': comment_stage.get_signatures_regex(self.venue, '${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [ { 'prefix': s, 'optional': True } if '.*' in s else { 'value': s, 'optional': True } for s in comment_stage.get_signatures(self.venue, '${7/content/noteNumber/value}')] + } + }, 'readers': ['${2/note/readers}'], # 'nonreaders': [], 'writers': [venue_id], @@ -1074,8 +1105,8 @@ def set_official_comment_invitation(self): invitation.edit['invitation']['edit']['note']['readers'] = comment_readers invitation.edit['invitation']['invitees'].append(self.venue.get_ethics_reviewers_id('${3/content/noteNumber/value}')) - invitation.edit['invitation']['edit']['signatures']['param']['regex'] += '|' + self.venue.get_ethics_reviewers_id('${5/content/noteNumber/value}', anon=True) - invitation.edit['invitation']['edit']['signatures']['param']['regex'] += '|' + self.venue.get_ethics_chairs_id() + invitation.edit['invitation']['edit']['signatures']['param']['items'].append({ 'prefix': self.venue.get_ethics_reviewers_id('${7/content/noteNumber/value}', anon=True), 'optional': True }) + invitation.edit['invitation']['edit']['signatures']['param']['items'].append({ 'value': self.venue.get_ethics_chairs_id(), 'optional': True }) self.save_invitation(invitation, replacement=False) return invitation @@ -1146,7 +1177,11 @@ def set_public_comment_invitation(self): funcs['process'](client, edit, invitation) ''', 'edit': { - 'signatures': { 'param': { 'regex': '~.*' }}, + 'signatures': { + 'param': { + 'items': [{ 'prefix': '~.*' }] + } + }, 'readers': ['${2/note/readers}'], 'writers': [venue_id], 'note': { @@ -1350,7 +1385,11 @@ def set_withdrawal_invitation(self): exec(script, funcs) funcs['process'](client, edit, invitation)''', 'edit': { - 'signatures': { 'param': { 'regex': self.venue.get_authors_id(number='${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [{ 'value': self.venue.get_authors_id(number='${7/content/noteNumber/value}') }] + } + }, 'readers': submission_stage.get_withdrawal_readers(self.venue, '${4/content/noteNumber/value}'), 'writers': [ venue_id, self.venue.get_authors_id(number='${4/content/noteNumber/value}')], 'note': { @@ -1382,6 +1421,7 @@ def set_withdrawal_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -1419,7 +1459,8 @@ def set_withdrawal_invitation(self): 'type': 'string', 'maxLength': 200000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -1583,6 +1624,7 @@ def set_withdrawal_invitation(self): 'maxLength': 200000, 'input': 'textarea', 'optional': True, + 'deletable': True, 'markdown': True } } @@ -1700,7 +1742,8 @@ def set_desk_rejection_invitation(self): 'type': 'string', 'maxLength': 200000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -1927,7 +1970,14 @@ def set_submission_revision_invitation(self, submission_revision_stage=None): } } , - 'signatures': { 'param': { 'regex': f"{self.venue.get_authors_id(number='${5/content/noteNumber/value}')}|{self.venue.get_program_chairs_id()}" }}, + 'signatures': { + 'param': { + 'items': [ + { 'value': self.venue.get_authors_id(number='${7/content/noteNumber/value}'), 'optional': True }, + { 'value': self.venue.get_program_chairs_id(), 'optional': True } + ] + } + }, 'readers': [ venue_id, self.venue.get_authors_id(number='${4/content/noteNumber/value}')], 'writers': [ venue_id, self.venue.get_authors_id(number='${4/content/noteNumber/value}')], 'note': { @@ -2039,7 +2089,11 @@ def set_custom_stage_invitation(self): funcs['process'](client, edit, invitation) ''', 'edit': { - 'signatures': { 'param': { 'regex': custom_stage.get_signatures_regex(self.venue, '${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [ { 'prefix': s, 'optional': True } if '.*' in s else { 'value': s, 'optional': True } for s in custom_stage.get_signatures(self.venue, '${7/content/noteNumber/value}')] + } + }, 'readers': custom_stage.get_readers(self.venue, '${4/content/noteNumber/value}'), 'writers': [venue_id], 'note': { @@ -2470,7 +2524,7 @@ def set_registration_invitations(self): maxReplies = 1, minReplies = 1, edit={ - 'signatures': { 'param': { 'regex': '~.*' }}, + 'signatures': { 'param': { 'items': [ { 'prefix': '~.*' } ] }}, 'readers': [venue_id, '${2/signatures}'], 'note': { 'id': { @@ -2852,7 +2906,11 @@ def set_ethics_review_invitation(self): funcs['process'](client, edit, invitation) ''', 'edit': { - 'signatures': { 'param': { 'regex': ethics_review_stage.get_signatures(self.venue, '${5/content/noteNumber/value}') }}, + 'signatures': { + 'param': { + 'items': [ { 'prefix': s, 'optional': True } if '.*' in s else { 'value': s, 'optional': True } for s in ethics_review_stage.get_signatures(self.venue, '${7/content/noteNumber/value}')] + } + }, 'readers': ethics_review_stage.get_readers(self.venue, '${4/content/noteNumber/value}', '${2/signatures}'), 'nonreaders': ethics_review_stage.get_nonreaders(self.venue, '${4/content/noteNumber/value}'), 'writers': [venue_id], @@ -2937,7 +2995,8 @@ def set_ethics_stage_invitation(self): 'value': { 'param': { 'type': 'boolean', - 'enum': [True, False] + 'enum': [True, False], + 'input': 'radio' } }, 'readers': flag_readers @@ -2949,7 +3008,8 @@ def set_ethics_stage_invitation(self): 'maxLength': 5000, 'markdown': True, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } }, 'readers': [venue_id, self.venue.get_ethics_chairs_id(), self.venue.get_ethics_reviewers_id('${{4/id}/number}')] @@ -3044,7 +3104,14 @@ def set_SAC_ethics_flag_invitation(self, sac_ethics_flag_duedate=None): funcs['process'](client, edit, invitation) ''', 'edit': { - 'signatures': { 'param': { 'regex': f"{venue.get_senior_area_chairs_id(number='${5/content/noteNumber/value}')}|{venue.get_program_chairs_id()}" }}, + 'signatures': { + 'param': { + 'items': [ + { 'value': venue.get_senior_area_chairs_id(number='${7/content/noteNumber/value}'), 'optional': True }, + { 'value': venue.get_program_chairs_id(), 'optional': True } + ] + } + }, 'readers': ['${2/note/readers}'], 'nonreaders': ['${2/note/nonreaders}'], 'writers': [venue_id], @@ -3085,7 +3152,8 @@ def set_SAC_ethics_flag_invitation(self, sac_ethics_flag_duedate=None): 'maxLength': 5000, 'markdown': True, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } }, 'description': 'Optional comment to Program Chairs.' diff --git a/openreview/venue/matching.py b/openreview/venue/matching.py index 8a70df6ac..776a7821d 100644 --- a/openreview/venue/matching.py +++ b/openreview/venue/matching.py @@ -175,7 +175,8 @@ def _create_edge_invitation(self, edge_id, any_tail=False, default_label=None): edge_label = { 'param': { 'regex': '.*', - 'optional': True + 'optional': True, + 'deletable': True } } @@ -818,7 +819,8 @@ def _build_config_invitation(self, scores_specification): 'type': 'string', 'regex': '{}/.*/-/Custom_User_Demands$'.format(venue.id), 'default': '{}/-/Custom_User_Demands'.format(self.match_group.id), - 'optional': True + 'optional': True, + 'deletable': True } } }, @@ -830,7 +832,8 @@ def _build_config_invitation(self, scores_specification): 'type': 'string', 'regex': '{}/.*/-/Custom_Max_Papers$'.format(venue.id), 'default': venue.get_custom_max_papers_id(self.match_group.id), - 'optional': True + 'optional': True, + 'deletable': True } } }, @@ -883,6 +886,7 @@ def _build_config_invitation(self, scores_specification): 'type': 'string', 'regex': '.*', 'optional': True, + 'deletable': True, 'hidden': True } } @@ -896,6 +900,7 @@ def _build_config_invitation(self, scores_specification): 'enum': ['Yes', 'No'], 'input': 'radio', 'optional': True, + 'deletable': True, 'default': 'Yes' } } @@ -908,6 +913,7 @@ def _build_config_invitation(self, scores_specification): 'type': 'string', 'regex': r'[-+]?[0-9]*\.?[0-9]*', 'optional': True, + 'deletable': True, 'default': '1' } } @@ -920,6 +926,7 @@ def _build_config_invitation(self, scores_specification): 'type': 'string', 'regex': r'[-+]?[0-9]*\.?[0-9]*', 'optional': True, + 'deletable': True, 'default': '', 'hidden': True } @@ -1098,6 +1105,7 @@ def setup_invite_assignment(self, hash_seed, assignment_title=None, due_date=Non 'Conflict Detected' ], 'optional': True, + 'deletable': True, 'default': invited_label } } diff --git a/tests/test_emnlp_conference.py b/tests/test_emnlp_conference.py index ddcadf4f5..94e44bea2 100644 --- a/tests/test_emnlp_conference.py +++ b/tests/test_emnlp_conference.py @@ -616,9 +616,13 @@ def test_desk_rejection_by_SAC(self, test_client, client, openreview_client, hel ], 'edit': { 'signatures': { - 'param': { - 'regex': 'EMNLP/2023/Conference/Submission${5/content/noteNumber/value}/Senior_Area_Chairs|EMNLP/2023/Conference/Submission${5/content/noteNumber/value}/Area_Chair_.*|EMNLP/2023/Conference/Program_Chairs' - } + 'param': { + 'items': [ + { 'value': 'EMNLP/2023/Conference/Submission${7/content/noteNumber/value}/Senior_Area_Chairs', 'optional': True }, + { 'prefix': 'EMNLP/2023/Conference/Submission${7/content/noteNumber/value}/Area_Chair_.*', 'optional': True }, + { 'value': 'EMNLP/2023/Conference/Program_Chairs', 'optional': True } + ] + } }, 'note': { 'content': { @@ -638,7 +642,11 @@ def test_desk_rejection_by_SAC(self, test_client, client, openreview_client, hel 'EMNLP/2023/Conference/Submission1/Senior_Area_Chairs', 'EMNLP/2023/Conference/Submission1/Area_Chairs' ] - assert invitation.edit['signatures']['param']['regex'] == 'EMNLP/2023/Conference/Submission1/Senior_Area_Chairs|EMNLP/2023/Conference/Submission1/Area_Chair_.*|EMNLP/2023/Conference/Program_Chairs' + assert invitation.edit['signatures']['param']['items'] == [ + { 'value': 'EMNLP/2023/Conference/Submission1/Senior_Area_Chairs', 'optional': True }, + { 'prefix': 'EMNLP/2023/Conference/Submission1/Area_Chair_.*', 'optional': True }, + { 'value': 'EMNLP/2023/Conference/Program_Chairs', 'optional': True } + ] pc_client=openreview.Client(username='pc@emnlp.org', password=helpers.strong_password) pc_client_v2=openreview.api.OpenReviewClient(username='pc@emnlp.org', password=helpers.strong_password) diff --git a/tests/test_icml_conference.py b/tests/test_icml_conference.py index 2baef90dd..c6373e371 100644 --- a/tests/test_icml_conference.py +++ b/tests/test_icml_conference.py @@ -2015,7 +2015,7 @@ def test_reviewer_reassignment(self, client, openreview_client, helpers, seleniu sac_group = pc_client.get_group('ICML.cc/2023/Conference/Submission11/Senior_Area_Chairs') assert [] == sac_group.members - def test_review_stage(self, client, openreview_client, helpers): + def test_review_stage(self, client, openreview_client, helpers, selenium, request_page): pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] @@ -2205,18 +2205,18 @@ def test_review_stage(self, client, openreview_client, helpers): "description": "Please provide an \"overall score\" for this submission.", "value": { "param": { - "type": "string", + "type": 'integer', "enum": [ - "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.", - "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations.", - "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations.", - "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations.", - "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations.", - "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly.", - "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly.", - "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations.", - "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations.", - "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" + { 'value': 10, 'description': "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations." }, + { 'value': 9, 'description': "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, + { 'value': 8, 'description': "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, + { 'value': 7, 'description': "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations." }, + { 'value': 6, 'description': "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations." }, + { 'value': 5, 'description': "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly." }, + { 'value': 4, 'description': "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly." }, + { 'value': 3, 'description': "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations." }, + { 'value': 2, 'description': "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations." }, + { 'value': 1, 'description': "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" } ], "input": "radio" @@ -2228,13 +2228,13 @@ def test_review_stage(self, client, openreview_client, helpers): "description": "Please provide a \"confidence score\" for your assessment of this submission to indicate how confident you are in your evaluation.", "value": { "param": { - "type": "string", + "type": 'integer', "enum": [ - "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.", - "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.", - "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.", - "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.", - "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." + { 'value': 5, 'description': "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully." }, + { 'value': 4, 'description': "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work." }, + { 'value': 3, 'description': "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, + { 'value': 2, 'description': "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, + { 'value': 1, 'description': "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." } ], "input": "radio" } @@ -2424,18 +2424,18 @@ def test_review_stage(self, client, openreview_client, helpers): "description": "Please provide an \"overall score\" for this submission.", "value": { "param": { - "type": "string", + "type": 'integer', "enum": [ - "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.", - "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations.", - "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations.", - "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations.", - "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations.", - "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly.", - "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly.", - "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations.", - "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations.", - "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" + { 'value': 10, 'description': "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations." }, + { 'value': 9, 'description': "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, + { 'value': 8, 'description': "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, + { 'value': 7, 'description': "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations." }, + { 'value': 6, 'description': "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations." }, + { 'value': 5, 'description': "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly." }, + { 'value': 4, 'description': "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly." }, + { 'value': 3, 'description': "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations." }, + { 'value': 2, 'description': "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations." }, + { 'value': 1, 'description': "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" } ], "input": "radio" @@ -2447,13 +2447,13 @@ def test_review_stage(self, client, openreview_client, helpers): "description": "Please provide a \"confidence score\" for your assessment of this submission to indicate how confident you are in your evaluation.", "value": { "param": { - "type": "string", + "type": 'integer', "enum": [ - "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.", - "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.", - "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.", - "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.", - "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." + { 'value': 5, 'description': "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully." }, + { 'value': 4, 'description': "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work." }, + { 'value': 3, 'description': "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, + { 'value': 2, 'description': "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, + { 'value': 1, 'description': "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." } ], "input": "radio" } @@ -2512,8 +2512,8 @@ def test_review_stage(self, client, openreview_client, helpers): 'soundness': { 'value': '3 good'}, 'presentation': { 'value': '3 good'}, 'contribution': { 'value': '3 good'}, - 'rating': { 'value': '10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.'}, - 'confidence': { 'value': '5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.'}, + 'rating': { 'value': 10 }, + 'confidence': { 'value': 5 }, 'code_of_conduct': { 'value': 'Yes'}, } ) @@ -2527,6 +2527,20 @@ def test_review_stage(self, client, openreview_client, helpers): messages = openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] Your official review has been received on your assigned Paper number: 1, Paper title: "Paper title 1 Version 2"') assert messages and len(messages) == 1 + ## check how the description is rendered + note = review_edit['note'] + review_id = note['id'] + request_page(selenium, "http://localhost:3030/forum?id=" + review_edit['note']['forum'], openreview_client.token, by=By.ID, wait_for_element='forum-replies') + note_panel = selenium.find_element(By.XPATH, f'//div[@data-id="{review_id}"]') + fields = note_panel.find_elements(By.CLASS_NAME, 'note-content-field') + assert len(fields) == 11 + assert fields[8].text == 'Rating:' + assert fields[9].text == 'Confidence:' + values = note_panel.find_elements(By.CLASS_NAME, 'note-content-value') + assert len(values) == 11 + assert values[8].text == '10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.' + assert values[9].text == '5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.' + review_edit = reviewer_client.post_note_edit( invitation='ICML.cc/2023/Conference/Submission1/-/Official_Review', signatures=[anon_group_id], @@ -2541,8 +2555,8 @@ def test_review_stage(self, client, openreview_client, helpers): 'soundness': { 'value': '3 good'}, 'presentation': { 'value': '3 good'}, 'contribution': { 'value': '3 good'}, - 'rating': { 'value': '10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.'}, - 'confidence': { 'value': '5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.'}, + 'rating': { 'value': 10 }, + 'confidence': { 'value': 5 }, 'code_of_conduct': { 'value': 'Yes'}, } ) @@ -2576,8 +2590,8 @@ def test_review_stage(self, client, openreview_client, helpers): 'soundness': { 'value': '1 poor'}, 'presentation': { 'value': '1 poor'}, 'contribution': { 'value': '1 poor'}, - 'rating': { 'value': '1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations'}, - 'confidence': { 'value': '5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.'}, + 'rating': { 'value': 1 }, + 'confidence': { 'value': 5 }, 'code_of_conduct': { 'value': 'Yes'}, } ) @@ -2601,8 +2615,8 @@ def test_review_stage(self, client, openreview_client, helpers): 'soundness': { 'value': '3 good'}, 'presentation': { 'value': '3 good'}, 'contribution': { 'value': '3 good'}, - 'rating': { 'value': '10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.'}, - 'confidence': { 'value': '5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.'}, + 'rating': { 'value': 10 }, + 'confidence': { 'value': 5 }, 'code_of_conduct': { 'value': 'Yes'}, } ) @@ -2627,13 +2641,13 @@ def test_review_stage(self, client, openreview_client, helpers): 'soundness': { 'value': '1 poor'}, 'presentation': { 'value': '1 poor'}, 'contribution': { 'value': '1 poor'}, - 'rating': { 'value': '10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.'}, - 'confidence': { 'value': '1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked.'}, + 'rating': { 'value': 10 }, + 'confidence': { 'value': 1 }, 'code_of_conduct': { 'value': 'Yes'}, } ) ) - assert openReviewError.value.args[0].get('name') == 'ValidationError' + assert openReviewError.value.args[0].get('name') == 'ItemsError' ## Extend deadline start_date = now - datetime.timedelta(days=20) @@ -2784,18 +2798,18 @@ def test_review_stage(self, client, openreview_client, helpers): "description": "Please provide an \"overall score\" for this submission.", "value": { "param": { - "type": "string", + "type": 'integer', "enum": [ - "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.", - "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations.", - "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations.", - "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations.", - "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations.", - "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly.", - "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly.", - "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations.", - "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations.", - "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" + { 'value': 10, 'description': "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations." }, + { 'value': 9, 'description': "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, + { 'value': 8, 'description': "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, + { 'value': 7, 'description': "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations." }, + { 'value': 6, 'description': "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations." }, + { 'value': 5, 'description': "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly." }, + { 'value': 4, 'description': "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly." }, + { 'value': 3, 'description': "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations." }, + { 'value': 2, 'description': "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations." }, + { 'value': 1, 'description': "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" } ], "input": "radio" @@ -2807,13 +2821,13 @@ def test_review_stage(self, client, openreview_client, helpers): "description": "Please provide a \"confidence score\" for your assessment of this submission to indicate how confident you are in your evaluation.", "value": { "param": { - "type": "string", + "type": 'integer', "enum": [ - "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.", - "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.", - "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.", - "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.", - "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." + { 'value': 5, 'description': "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully." }, + { 'value': 4, 'description': "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work." }, + { 'value': 3, 'description': "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, + { 'value': 2, 'description': "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, + { 'value': 1, 'description': "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." } ], "input": "radio" } @@ -3051,8 +3065,8 @@ def test_review_rating(self, client, openreview_client, helpers): 'soundness': { 'value': '3 good'}, 'presentation': { 'value': '3 good'}, 'contribution': { 'value': '3 good'}, - 'rating': { 'value': '10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.'}, - 'confidence': { 'value': '5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.'}, + 'rating': { 'value': 10 }, + 'confidence': { 'value': 5 }, 'code_of_conduct': { 'value': 'Yes'}, } ) @@ -3457,8 +3471,32 @@ def test_comment_stage(self, openreview_client, helpers): assert invitation assert 'ICML.cc/2023/Conference/Submission1/Ethics_Reviewers' in invitation.invitees assert 'ICML.cc/2023/Conference/Submission1/Ethics_Reviewers' in invitation.edit['note']['readers']['param']['enum'] - assert 'ICML.cc/2023/Conference/Submission1/Ethics_Reviewer_.*' in invitation.edit['signatures']['param']['regex'] - assert 'ICML.cc/2023/Conference/Ethics_Chairs' in invitation.edit['signatures']['param']['regex'] + assert invitation.edit['signatures']['param']['items'] == [ + { + "value": "ICML.cc/2023/Conference/Program_Chairs", + "optional": True + }, + { + "value": "ICML.cc/2023/Conference/Submission1/Senior_Area_Chairs", + "optional": True + }, + { + "prefix": "ICML.cc/2023/Conference/Submission1/Area_Chair_.*", + "optional": True + }, + { + "prefix": "ICML.cc/2023/Conference/Submission1/Reviewer_.*", + "optional": True + }, + { + "prefix": "ICML.cc/2023/Conference/Submission1/Ethics_Reviewer_.*", + "optional": True + }, + { + "value": "ICML.cc/2023/Conference/Ethics_Chairs", + "optional": True + } + ] invitation = openreview_client.get_invitation('ICML.cc/2023/Conference/Submission2/-/Official_Comment') assert invitation assert 'ICML.cc/2023/Conference/Submission2/Ethics_Reviewers' not in invitation.edit['note']['readers']['param']['enum'] @@ -3468,9 +3506,34 @@ def test_comment_stage(self, openreview_client, helpers): assert invitation assert 'ICML.cc/2023/Conference/Submission5/Ethics_Reviewers' in invitation.invitees assert 'ICML.cc/2023/Conference/Submission5/Ethics_Reviewers' in invitation.edit['note']['readers']['param']['enum'] - assert 'ICML.cc/2023/Conference/Submission5/Ethics_Reviewer_.*' in invitation.edit['signatures']['param']['regex'] - assert 'ICML.cc/2023/Conference/Ethics_Chairs' in invitation.edit['signatures']['param']['regex'] - + + assert invitation.edit['signatures']['param']['items'] == [ + { + "value": "ICML.cc/2023/Conference/Program_Chairs", + "optional": True + }, + { + "value": "ICML.cc/2023/Conference/Submission5/Senior_Area_Chairs", + "optional": True + }, + { + "prefix": "ICML.cc/2023/Conference/Submission5/Area_Chair_.*", + "optional": True + }, + { + "prefix": "ICML.cc/2023/Conference/Submission5/Reviewer_.*", + "optional": True + }, + { + "prefix": "ICML.cc/2023/Conference/Submission5/Ethics_Reviewer_.*", + "optional": True + }, + { + "value": "ICML.cc/2023/Conference/Ethics_Chairs", + "optional": True + } + ] + pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) # unflag a paper note = openreview_client.get_notes(invitation='ICML.cc/2023/Conference/-/Submission', number=[5])[0] @@ -3492,8 +3555,32 @@ def test_comment_stage(self, openreview_client, helpers): assert invitation assert 'ICML.cc/2023/Conference/Submission5/Ethics_Reviewers' not in invitation.invitees assert 'ICML.cc/2023/Conference/Submission5/Ethics_Reviewers' in invitation.edit['note']['readers']['param']['enum'] - assert 'ICML.cc/2023/Conference/Submission5/Ethics_Reviewer_.*' in invitation.edit['signatures']['param']['regex'] - + assert invitation.edit['signatures']['param']['items'] == [ + { + "value": "ICML.cc/2023/Conference/Program_Chairs", + "optional": True + }, + { + "value": "ICML.cc/2023/Conference/Submission5/Senior_Area_Chairs", + "optional": True + }, + { + "prefix": "ICML.cc/2023/Conference/Submission5/Area_Chair_.*", + "optional": True + }, + { + "prefix": "ICML.cc/2023/Conference/Submission5/Reviewer_.*", + "optional": True + }, + { + "prefix": "ICML.cc/2023/Conference/Submission5/Ethics_Reviewer_.*", + "optional": True + }, + { + "value": "ICML.cc/2023/Conference/Ethics_Chairs", + "optional": True + } + ] submissions = openreview_client.get_notes(content= { 'venueid': 'ICML.cc/2023/Conference/Submission'}, sort='number:asc') assert submissions and len(submissions) == 100 assert 'flagged_for_ethics_review' in submissions[4].content and not submissions[4].content['flagged_for_ethics_review']['value'] @@ -3747,18 +3834,18 @@ def test_rebuttal_stage(self, client, openreview_client, helpers): "description": "Please provide an \"overall score\" for this submission.", "value": { "param": { - "type": "string", + "type": 'integer', "enum": [ - "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.", - "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations.", - "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations.", - "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations.", - "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations.", - "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly.", - "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly.", - "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations.", - "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations.", - "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" + { 'value': 10, 'description': "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations." }, + { 'value': 9, 'description': "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, + { 'value': 8, 'description': "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, + { 'value': 7, 'description': "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations." }, + { 'value': 6, 'description': "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations." }, + { 'value': 5, 'description': "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly." }, + { 'value': 4, 'description': "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly." }, + { 'value': 3, 'description': "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations." }, + { 'value': 2, 'description': "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations." }, + { 'value': 1, 'description': "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" } ], "input": "radio" @@ -3770,13 +3857,13 @@ def test_rebuttal_stage(self, client, openreview_client, helpers): "description": "Please provide a \"confidence score\" for your assessment of this submission to indicate how confident you are in your evaluation.", "value": { "param": { - "type": "string", + "type": 'integer', "enum": [ - "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.", - "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.", - "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.", - "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.", - "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." + { 'value': 5, 'description': "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully." }, + { 'value': 4, 'description': "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work." }, + { 'value': 3, 'description': "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, + { 'value': 2, 'description': "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, + { 'value': 1, 'description': "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." } ], "input": "radio" } @@ -4070,7 +4157,8 @@ def test_meta_review_stage(self, client, openreview_client, helpers): 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -4308,7 +4396,8 @@ def test_decision_stage(self, client, openreview_client, helpers): 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -4373,7 +4462,8 @@ def test_decision_stage(self, client, openreview_client, helpers): 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -4422,7 +4512,8 @@ def test_decision_stage(self, client, openreview_client, helpers): 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } diff --git a/tests/test_neurips_conference_v2.py b/tests/test_neurips_conference_v2.py index 6a2f4eaed..01d1cc3d1 100644 --- a/tests/test_neurips_conference_v2.py +++ b/tests/test_neurips_conference_v2.py @@ -774,7 +774,8 @@ def test_update_submission_invitation(self, client, helpers, openreview_client): 'param': { 'type': 'string', 'regex': '~.*|([a-z0-9_\\-\\.]{1,}@[a-z0-9_\\-\\.]{2,}\\.[a-z]{2,},){0,}([a-z0-9_\\-\\.]{1,}@[a-z0-9_\\-\\.]{2,}\\.[a-z]{2,})', - 'optional': True + 'optional': True, + 'deletable': True } }, 'description': 'Select which author should be the primary corresponding author for this submission. Please enter an email address or an OpenReview ID that exactly matches one of the authors.', diff --git a/tests/test_non_anonnymous_venue.py b/tests/test_non_anonnymous_venue.py index cdafea7d1..006e32d96 100644 --- a/tests/test_non_anonnymous_venue.py +++ b/tests/test_non_anonnymous_venue.py @@ -142,5 +142,10 @@ def test_review_stage(self, venue, openreview_client, helpers): assert len(invitations) == 1 invitation = openreview_client.get_invitation('TestNonAnonymousVenue.cc/Submission1/-/Official_Review') - assert '~.*' == invitation.edit['signatures']['param']['regex'] + assert invitation.edit['signatures']['param']['items'] == [ + { + "prefix": "~.*", + "optional": True + } + ] diff --git a/tests/test_react_note_editor.py b/tests/test_react_note_editor.py new file mode 100644 index 000000000..9f61498a9 --- /dev/null +++ b/tests/test_react_note_editor.py @@ -0,0 +1,234 @@ +import csv +import openreview +import pytest +import time +import json +import datetime +import random +import os +import re +from openreview.api import OpenReviewClient +from openreview.api import Note +from openreview.api import Group +from openreview.api import Invitation +from openreview.api import Edge + +from openreview.venue import Venue +from openreview.stages import SubmissionStage, BidStage + +class TestReactNoteEditor(): + + @pytest.fixture(scope="class") + def venue(self, openreview_client): + conference_id = 'ReactVenue.cc' + + venue = Venue(openreview_client, conference_id, 'openreview.net/Support') + venue.invitation_builder.update_wait_time = 2000 + venue.invitation_builder.update_date_string = "#{4/mdate} + 2000" + venue.automatic_reviewer_assignment = True + venue.use_area_chairs = True + venue.name = 'Test Venue V2' + venue.short_name = 'TV 22' + venue.website = 'testvenue.org' + venue.contact = 'testvenue@contact.com' + venue.reviewer_identity_readers = [openreview.stages.IdentityReaders.PROGRAM_CHAIRS, openreview.stages.IdentityReaders.AREA_CHAIRS_ASSIGNED] + + now = datetime.datetime.utcnow() + venue.submission_stage = SubmissionStage( + double_blind=True, + due_date=now + datetime.timedelta(minutes = 30), + readers=[SubmissionStage.Readers.EVERYONE], + withdrawn_submission_public=True, + withdrawn_submission_reveal_authors=True, + desk_rejected_submission_public=True, + force_profiles=True, + additional_fields={ + 'checkbox_mandatory': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'enum': ['Yes'], + 'type': 'string', + 'input': 'checkbox', + 'default': 'Yes', + } + } + }, + 'checkbox_optional': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'enum': ['Yes'], + 'type': 'string', + 'input': 'checkbox', + 'optional': True, + 'deletable': True + } + } + }, + 'radio_button': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'fieldName': 'Paper color', + 'enum': ['Black', 'White'], + 'type': 'string', + 'input': 'radio' + } + } + }, + 'radio_button_number': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'type': 'integer', + 'enum': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'input': 'radio' + } + } + }, + 'radio_button_number_with_description': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'type': 'integer', + 'enum': [ + { 'value': 1, 'description': 'This is a description for 1' }, + { 'value': 2, 'description': 'This is a description for 2' }, + { 'value': 3, 'description': 'This is a description for 3'}, + { 'value': 4, 'description': 'This is a description for 4'}, + { 'value': 5, 'description': 'This is a description for 5' }, + { 'value': 6, 'description': 'This is a description for 6' }, + { 'value': 7, 'description': 'This is a description for 7' }, + { 'value': 8, 'description': 'This is a description for 8' }, + { 'value': 9, 'description': 'This is a description for 9' }, + ], + 'input': 'radio' + } + } + }, + 'dropdown_field': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'type': 'string', + 'enum': ['A', 'B', 'C'], + 'input': 'select' + } + } + }, + 'multi_dropdown_field': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'type': 'string[]', + 'enum': ['A', 'B', 'C', 'D', 'E', 'F', 'G'], + 'input': 'select' + } + } + }, + 'default_dropdown_field': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'type': 'string', + 'enum': ['A', 'B', 'C'] + } + } + }, + 'default_dropdown_field_two': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'type': 'string', + 'enum': [{ 'value': 'A', 'description': 'Letter A' }, { 'value': 'B', 'description': 'Letter B' }, { 'value': 'C', 'description': 'Letter C' }] + } + } + }, + 'dropdown_number_with_description': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'type': 'integer[]', + 'items': [ + { 'value': 1, 'description': 'This is a description for 1', 'optional': True }, + { 'value': 2, 'description': 'This is a description for 2', 'optional': True }, + { 'value': 3, 'description': 'This is a description for 3', 'optional': True }, + { 'value': 4, 'description': 'This is a description for 4', 'optional': True }, + { 'value': 5, 'description': 'This is a description for 5', 'optional': True }, + { 'value': 6, 'description': 'This is a description for 6', 'optional': True }, + { 'value': 7, 'description': 'This is a description for 7', 'optional': True }, + { 'value': 8, 'description': 'This is a description for 8', 'optional': True }, + { 'value': 9, 'description': 'This is a description for 9', 'optional': True }, + ], + 'input': 'select' + } + } + }, + 'dropdown_number_with_mandatory': { + 'description': 'This is a description', + 'order': 1, + 'value': { + 'param': { + 'type': 'string[]', + 'items': [ + { 'value': 'ICLR.cc/Program_Chairs', 'description': 'Program Chairs', 'optional': False }, + { 'value': 'ICLR.cc/Senior_Area_Chairs', 'description': 'Senior Area Chairs', 'optional': False }, + { 'value': 'ICLR.cc/Area_Chairs', 'description': 'All Ara Chairs', 'optional': True }, + { 'value': 'ICLR.cc/Reviewers', 'description': 'All Reviewers', 'optional': True }, + { 'value': 'ICLR.cc/Authors', 'description': 'Submission authors', 'optional': True } + ], + 'input': 'select' + } + } + }, + 'json_field': { + 'description': 'Settings for the venue', + 'order': 1, + 'value': { + 'param': { + 'type': 'json', + 'optional': True, + 'deletable': True + } + } + } + }, + remove_fields=[], + ) + return venue + + + def test_setup(self, venue, openreview_client, helpers): + + venue.setup(program_chair_ids=['venue_pc@mail.com']) + venue.create_submission_stage() + assert openreview_client.get_group('ReactVenue.cc') + assert openreview_client.get_group('ReactVenue.cc/Authors') + + + invitations = openreview_client.get_invitations(prefix = 'ReactVenue.cc', type = 'all') + for invitation in invitations: + if 'ReactVenue.cc/-/Edit' not in invitation.id: + openreview_client.post_invitation_edit( + invitations='ReactVenue.cc/-/Edit', + signatures=['ReactVenue.cc'], + invitation=openreview.api.Invitation( + id=invitation.id, + expdate=openreview.tools.datetime_millis(datetime.datetime.utcnow()) + ) + ) + + helpers.await_queue(openreview_client) + + diff --git a/tests/test_venue_request_v2.py b/tests/test_venue_request_v2.py index 62cc0a636..3d27395b3 100644 --- a/tests/test_venue_request_v2.py +++ b/tests/test_venue_request_v2.py @@ -303,7 +303,8 @@ def test_venue_deployment(self, client, selenium, request_page, helpers, openrev client.post_note(venue_revision_note) venue_revision_note.content['desk_rejected_submissions_author_anonymity'] = 'Yes, author identities of desk rejected submissions should be revealed.' - venue_revision_note.content['Submission Deadline'] = request_form_note.content['Submission Deadline'] + ' ' + venue_revision_note.content['abstract_registration_deadline'] = (now - datetime.timedelta(days=1)).strftime('%Y/%m/%d %H:%M') + venue_revision_note.content['Submission Deadline'] = (now - datetime.timedelta(days=1)).strftime('%Y/%m/%d %H:%M') venue_revision_note=client.post_note(venue_revision_note) helpers.await_queue() @@ -1503,9 +1504,32 @@ def test_venue_review_stage(self, client, test_client, selenium, request_page, h 'make_reviews_public': 'Yes, reviews should be revealed publicly when they are posted', 'release_reviews_to_authors': 'No, reviews should NOT be revealed when they are posted to the paper\'s authors', 'release_reviews_to_reviewers': 'Reviews should be immediately revealed to the paper\'s reviewers who have already submitted their review', - 'remove_review_form_options': 'title', + 'remove_review_form_options': 'rating,title', 'email_program_chairs_about_reviews': 'Yes, email program chairs for each review received', - 'review_rating_field_name': 'review_rating' + 'review_rating_field_name': 'review_rating', + 'additional_review_form_options': { + 'review_rating': { + 'order': 3, + 'value': { + 'param': { + 'type': 'integer', + 'enum': [ + { 'value': 10, 'description': '10: Top 5% of accepted papers, seminal paper' }, + { 'value': 9, 'description': '9: Top 15% of accepted papers, strong accept' }, + { 'value': 8, 'description': '8: Top 50% of accepted papers, clear accept' }, + { 'value': 7, 'description': '7: Good paper, accept' }, + { 'value': 6, 'description': '6: Marginally above acceptance threshold' }, + { 'value': 5, 'description': '5: Marginally below acceptance threshold' }, + { 'value': 4, 'description': '4: Ok but not good enough - rejection' }, + { 'value': 3, 'description': '3: Clear rejection' }, + { 'value': 2, 'description': '2: Strong rejection' }, + { 'value': 1, 'description': '1: Trivial or wrong' } + ], + 'input': 'radio' + } + } + } + }, }, forum=venue['request_form_note'].forum, invitation='{}/-/Request{}/Review_Stage'.format(venue['support_group_id'], venue['request_form_note'].number), @@ -1577,8 +1601,8 @@ def test_venue_review_stage(self, client, test_client, selenium, request_page, h note=Note( content={ 'review': { 'value': 'great paper!' }, - 'rating': { 'value': '10: Top 5% of accepted papers, seminal paper' }, - 'confidence': { 'value': '3: The reviewer is fairly confident that the evaluation is correct' } + 'review_rating': { 'value': 10 }, + 'confidence': { 'value': 3 } } ) ) @@ -1609,8 +1633,32 @@ def test_release_reviews_to_authors(self, client, test_client, selenium, request 'make_reviews_public': 'No, reviews should NOT be revealed publicly when they are posted', 'release_reviews_to_authors': 'Yes, reviews should be revealed when they are posted to the paper\'s authors', 'release_reviews_to_reviewers': 'Reviews should be immediately revealed to the paper\'s reviewers', - 'remove_review_form_options': 'title', - 'email_program_chairs_about_reviews': 'Yes, email program chairs for each review received' + 'remove_review_form_options': 'rating,title', + 'email_program_chairs_about_reviews': 'Yes, email program chairs for each review received', + 'review_rating_field_name': 'review_rating', + 'additional_review_form_options': { + 'review_rating': { + 'order': 3, + 'value': { + 'param': { + 'type': 'integer', + 'enum': [ + { 'value': 10, 'description': '10: Top 5% of accepted papers, seminal paper' }, + { 'value': 9, 'description': '9: Top 15% of accepted papers, strong accept' }, + { 'value': 8, 'description': '8: Top 50% of accepted papers, clear accept' }, + { 'value': 7, 'description': '7: Good paper, accept' }, + { 'value': 6, 'description': '6: Marginally above acceptance threshold' }, + { 'value': 5, 'description': '5: Marginally below acceptance threshold' }, + { 'value': 4, 'description': '4: Ok but not good enough - rejection' }, + { 'value': 3, 'description': '3: Clear rejection' }, + { 'value': 2, 'description': '2: Strong rejection' }, + { 'value': 1, 'description': '1: Trivial or wrong' } + ], + 'input': 'radio' + } + } + } + }, }, forum=venue['request_form_note'].forum, invitation='{}/-/Request{}/Review_Stage'.format(venue['support_group_id'], venue['request_form_note'].number), @@ -1636,7 +1684,7 @@ def test_release_reviews_to_authors(self, client, test_client, selenium, request assert 'V2.cc/2030/Conference/Submission1/Reviewers' in reviews[0].readers assert 'V2.cc/2030/Conference/Submission1/Authors' in reviews[0].readers assert len(reviews[0].nonreaders) == 0 - assert 'readers' not in reviews[0].content['rating'] + assert 'readers' not in reviews[0].content['review_rating'] #hide ratings from authors without changing readers of reviews now = datetime.datetime.utcnow() @@ -1651,24 +1699,28 @@ def test_release_reviews_to_authors(self, client, test_client, selenium, request 'make_reviews_public': 'No, reviews should NOT be revealed publicly when they are posted', 'release_reviews_to_authors': 'Yes, reviews should be revealed when they are posted to the paper\'s authors', 'release_reviews_to_reviewers': 'Reviews should be immediately revealed to the paper\'s reviewers', + 'remove_review_form_options': 'rating,title', + 'email_program_chairs_about_reviews': 'Yes, email program chairs for each review received', + 'review_rating_field_name': 'review_rating', 'additional_review_form_options': { - 'rating': { + 'review_rating': { 'order': 3, 'value': { 'param': { - 'type': 'string', + 'type': 'integer', 'enum': [ - '10: Top 5% of accepted papers, seminal paper', - '9: Top 15% of accepted papers, strong accept', - '8: Top 50% of accepted papers, clear accept', - '7: Good paper, accept', - '6: Marginally above acceptance threshold', - '5: Marginally below acceptance threshold', - '4: Ok but not good enough - rejection', - '3: Clear rejection', - '2: Strong rejection', - '1: Trivial or wrong' - ] + { 'value': 10, 'description': '10: Top 5% of accepted papers, seminal paper' }, + { 'value': 9, 'description': '9: Top 15% of accepted papers, strong accept' }, + { 'value': 8, 'description': '8: Top 50% of accepted papers, clear accept' }, + { 'value': 7, 'description': '7: Good paper, accept' }, + { 'value': 6, 'description': '6: Marginally above acceptance threshold' }, + { 'value': 5, 'description': '5: Marginally below acceptance threshold' }, + { 'value': 4, 'description': '4: Ok but not good enough - rejection' }, + { 'value': 3, 'description': '3: Clear rejection' }, + { 'value': 2, 'description': '2: Strong rejection' }, + { 'value': 1, 'description': '1: Trivial or wrong' } + ], + 'input': 'radio' } }, 'readers': [ @@ -1677,9 +1729,7 @@ def test_release_reviews_to_authors(self, client, test_client, selenium, request "${5/signatures}" ] } - }, - 'remove_review_form_options': 'title', - 'email_program_chairs_about_reviews': 'Yes, email program chairs for each review received' + }, }, forum=venue['request_form_note'].forum, invitation='{}/-/Request{}/Review_Stage'.format(venue['support_group_id'], venue['request_form_note'].number), @@ -1700,8 +1750,8 @@ def test_release_reviews_to_authors(self, client, test_client, selenium, request assert 'V2.cc/2030/Conference/Submission1/Reviewers' in reviews[0].readers assert 'V2.cc/2030/Conference/Submission1/Authors' in reviews[0].readers assert len(reviews[0].nonreaders) == 0 - assert 'readers' in reviews[0].content['rating'] - assert reviews[0].content['rating']['readers'] == [ + assert 'readers' in reviews[0].content['review_rating'] + assert reviews[0].content['review_rating']['readers'] == [ "V2.cc/2030/Conference", "V2.cc/2030/Conference/Submission1/Area_Chairs", reviews[0].signatures[0] @@ -1771,8 +1821,8 @@ def test_rebuttal_stage(self, client, test_client, venue, openreview_client, hel note=Note( content={ 'review': { 'value': 'very good paper' }, - 'rating': { 'value': '7: Good paper, accept' }, - 'confidence': { 'value': '3: The reviewer is fairly confident that the evaluation is correct' } + 'review_rating': { 'value': 7 }, + 'confidence': { 'value': 3 } } ) ) @@ -1781,8 +1831,8 @@ def test_rebuttal_stage(self, client, test_client, venue, openreview_client, hel reviews = openreview_client.get_notes(invitation='V2.cc/2030/Conference/Submission2/-/Official_Review') assert len(reviews) == 1 - assert 'readers' in reviews[0].content['rating'] - assert reviews[0].content['rating']['readers'] == [ + assert 'readers' in reviews[0].content['review_rating'] + assert reviews[0].content['review_rating']['readers'] == [ "V2.cc/2030/Conference", "V2.cc/2030/Conference/Submission2/Area_Chairs", reviews[0].signatures[0] @@ -1855,7 +1905,8 @@ def test_venue_meta_review_stage(self, client, test_client, selenium, request_pa 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -1965,7 +2016,8 @@ def test_release_meta_reviews_to_authors_and_reviewers(self, test_client, helper 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -2040,12 +2092,13 @@ def test_venue_comment_stage(self, client, test_client, selenium, request_page, paper_official_comment_invitation = openreview.tools.get_invitation(openreview_client, '{}/Submission1/-/Official_Comment'.format(venue['venue_id'])) assert paper_official_comment_invitation - assert 'V2.cc/2030/Conference/Program_Chairs' in paper_official_comment_invitation.edit['note']['readers']['param']['enum'] - assert 'V2.cc/2030/Conference/Submission1/Reviewer_.*' in paper_official_comment_invitation.edit['note']['readers']['param']['enum'] - assert 'V2.cc/2030/Conference/Submission1/Senior_Area_Chairs' in paper_official_comment_invitation.edit['note']['readers']['param']['enum'] - assert 'V2.cc/2030/Conference/Submission1/Area_Chairs' in paper_official_comment_invitation.edit['note']['readers']['param']['enum'] - assert 'V2.cc/2030/Conference/Submission1/Reviewers' in paper_official_comment_invitation.edit['note']['readers']['param']['enum'] - assert 'V2.cc/2030/Conference/Submission1/Authors' in paper_official_comment_invitation.edit['note']['readers']['param']['enum'] + readers = [item.get('value', item.get('prefix')) for item in paper_official_comment_invitation.edit['note']['readers']['param']['items']] + assert 'V2.cc/2030/Conference/Program_Chairs' in readers + assert 'V2.cc/2030/Conference/Submission1/Reviewer_.*' in readers + assert 'V2.cc/2030/Conference/Submission1/Senior_Area_Chairs' in readers + assert 'V2.cc/2030/Conference/Submission1/Area_Chairs' in readers + assert 'V2.cc/2030/Conference/Submission1/Reviewers' in readers + assert 'V2.cc/2030/Conference/Submission1/Authors' in readers author_client = OpenReviewClient(username='venue_author_v2@mail.com', password=helpers.strong_password) # Assert that an official comment can be posted by the paper author @@ -2107,7 +2160,8 @@ def test_venue_decision_stage(self, client, test_client, selenium, request_page, 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -2220,7 +2274,8 @@ def test_venue_decision_stage(self, client, test_client, selenium, request_page, 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -2274,7 +2329,8 @@ def test_venue_decision_stage(self, client, test_client, selenium, request_page, 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -2328,7 +2384,8 @@ def test_venue_decision_stage(self, client, test_client, selenium, request_page, 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -2383,7 +2440,8 @@ def test_venue_decision_stage(self, client, test_client, selenium, request_page, 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -2437,7 +2495,8 @@ def test_venue_decision_stage(self, client, test_client, selenium, request_page, 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -2502,7 +2561,8 @@ def test_venue_decision_stage(self, client, test_client, selenium, request_page, 'type': 'string', 'maxLength': 5000, 'input': 'textarea', - 'optional': True + 'optional': True, + 'deletable': True } } } @@ -2785,10 +2845,8 @@ def test_post_decision_stage(self, client, test_client, selenium, request_page, assert "Dear VenueTwo Author,\n\nThank you for submitting your paper, test submission, to TestVenue@OR'2030V2." in last_message['content']['text'] assert f"https://openreview.net/forum?id={submissions[0].id}" in last_message['content']['text'] - request_page(selenium, 'http://localhost:3030/group?id={}'.format(venue['venue_id']), test_client.token, wait_for_element='header') - notes_panel = selenium.find_element(By.ID, 'notes') - assert notes_panel - tabs = notes_panel.find_element(By.CLASS_NAME, 'tabs-container') + request_page(selenium, 'http://localhost:3030/group?id={}'.format(venue['venue_id']), test_client.token, by=By.CLASS_NAME, wait_for_element='tabs-container') + tabs = selenium.find_element(By.CLASS_NAME, 'tabs-container') assert tabs assert tabs.find_element(By.LINK_TEXT, "Your Consoles") assert tabs.find_element(By.LINK_TEXT, "Accept") diff --git a/tests/test_workshop_no_anonymity.py b/tests/test_workshop_no_anonymity.py index b33782bba..28dbdc37b 100644 --- a/tests/test_workshop_no_anonymity.py +++ b/tests/test_workshop_no_anonymity.py @@ -292,4 +292,9 @@ def test_review_stage(self, client, openreview_client, helpers): helpers.await_queue() invitation = openreview_client.get_invitation('PRL/2024/ICAPS/Submission1/-/Official_Review') - assert invitation.edit['signatures']['param']['regex'] == '~.*' \ No newline at end of file + assert invitation.edit['signatures']['param']['items'] == [ + { + "prefix": "~.*", + "optional": True + } + ] \ No newline at end of file