Skip to content

Add venue profile validation for invite assignment #2572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Add venue profile validation for invite assignment #2572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3616db6
Add venue profile validation for invite assignment
enrubio Apr 24, 2025
fac00fc
Add filter for pdate if requiring publications
enrubio Apr 24, 2025
81bbf7b
Merge branch 'master' into feature/add-venue-profile-check
melisabok May 1, 2025
a5cee13
Remove filter on pubs, impersonate active venues
enrubio May 5, 2025
147daa1
Move profile completion logic to tools.js
enrubio May 6, 2025
ce98253
Merge branch 'master' into feature/add-venue-profile-check
melisabok May 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions openreview/conference/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ def get_conference(client, request_form_id, support_user='OpenReview.net/Support
venue.submission_assignment_max_reviewers = int(note.content.get('submission_assignment_max_reviewers')) if note.content.get('submission_assignment_max_reviewers') is not None else None
venue.comment_notification_threshold = int(note.content.get('comment_notification_threshold')) if note.content.get('comment_notification_threshold') is not None else None
venue.preferred_emails_groups = note.content.get('preferred_emails_groups', [])
venue.invited_reviewer_profile_minimum_requirements = venue_content.get('invited_reviewer_profile_minimum_requirements', {}).get('value', False)
venue.iThenticate_plagiarism_check = note.content.get('iThenticate_plagiarism_check', 'No') == 'Yes'
venue.iThenticate_plagiarism_check_api_key = note.content.get('iThenticate_plagiarism_check_api_key', '')
venue.iThenticate_plagiarism_check_api_base_url = note.content.get('iThenticate_plagiarism_check_api_base_url', '')
Expand Down
3 changes: 3 additions & 0 deletions openreview/venue/group.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,9 @@ def create_venue_group(self):

if self.venue.comment_notification_threshold:
content['comment_notification_threshold'] = { 'value': self.venue.comment_notification_threshold }

if venue_group.content.get('invited_reviewer_profile_minimum_requirements'):
content['invited_reviewer_profile_minimum_requirements'] = venue_group.content.get('invited_reviewer_profile_minimum_requirements')

update_content = self.get_update_content(venue_group.content, content)
if update_content:
Expand Down
6 changes: 6 additions & 0 deletions openreview/venue/process/invite_assignment_pre_process.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ async function process(client, edge, invitation) {
const conflictNYears = domain.content.reviewers_conflict_n_years?.value
const reviewersName = reviewersId.split('/').pop().toLowerCase()
const quota = domain.content?.['submission_assignment_max_' + reviewersName]?.value
const profileReqs = domain.content.invited_reviewer_profile_minimum_requirements?.value

if (edge.ddate && edge.label !== inviteLabel) {
return Promise.reject(new OpenReviewError({ name: 'Error', message: `Cannot cancel the invitation since it has status: "${edge.label}"` }))
Expand All @@ -27,6 +28,11 @@ async function process(client, edge, invitation) {
const profiles = await client.tools.getProfiles([edge.tail], true)
const userProfile = profiles[0]

// Check for complete profile, if no profile then go to pending sign up
if (!client.tools.isProfileComplete(userProfile, profileReqs)) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens if profileReqs is empty?

return Promise.reject(new OpenReviewError({ name: 'Error', message: `Can not invite ${userProfile.id}, the user has an incomplete profile according to venue standards` }))
}

if (userProfile.id !== edge.tail) {
const { edges } = await client.getEdges({ invitation: edge.invitation, head: edge.head, tail: userProfile.id })
if (edges.length) {
Expand Down
98 changes: 79 additions & 19 deletions openreview/venue/venue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ def __init__(self, client, venue_id, support_user):
self.iThenticate_plagiarism_check_exclude_custom_sections = False
self.iThenticate_plagiarism_check_exclude_small_matches = 8
self.comment_notification_threshold = None
self.invited_reviewer_profile_minimum_requirements = {}

def get_id(self):
return self.venue_id
Expand Down Expand Up @@ -1373,6 +1374,19 @@ def poll_ithenticate_for_status(self):
@classmethod
def check_new_profiles(Venue, client):

def send_incomplete_profile_notification(venue_group, edge, submission, user_profile):
## Send email to reviewer
subject=f"[{venue_group.content['subtitle']['value']}] Incomplete profile for paper {submission.number}"
message =f'''Hi {{{{fullname}}}},
You have accepted the invitation to review the paper number: {submission.number}, title: {submission.content['title']['value']}.

However, your profile was found to be incomplete according to {venue_group.content['subtitle']['value']} standards and the assignment is pending your profile completion. Please review your venue's profile requirements and update your profile.

If you have any questions, please contact us as info@openreview.net.

OpenReview Team'''
response = client.post_message(subject, [edge.tail], message, invitation=venue_group.content['meta_invitation_id']['value'], signature=venue_group.id, replyTo=venue_group.content['contact']['value'], sender=venue_group.content['message_sender']['value'])

def mark_as_conflict(venue_group, edge, submission, user_profile):
edge.label='Conflict Detected'
edge.tail=user_profile.id
Expand Down Expand Up @@ -1474,29 +1488,35 @@ def mark_as_accepted(venue_group, edge, submission, user_profile, invite_assignm
active_venues = client.get_group('active_venues').members

for venue_id in active_venues:
# Create new client for each venue
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cool!

venue_client = openreview.api.OpenReviewClient(
baseurl=openreview.tools.get_base_urls(client)[1],
token=client.token
)
venue_client.impersonate(venue_id)

venue_group = client.get_group(venue_id)
venue_group = venue_client.get_group(venue_id)

if hasattr(venue_group, 'domain') and venue_group.content:

print(f'Check active venue {venue_group.id}')

edge_invitations = client.get_all_invitations(prefix=venue_id, type='edge')
edge_invitations = venue_client.get_all_invitations(prefix=venue_id, type='edge')
invite_assignment_invitations = [inv.id for inv in edge_invitations if inv.id.endswith('Invite_Assignment')]

for invite_assignment_invitation_id in invite_assignment_invitations:

## check if it is expired?
invite_assignment_invitation = openreview.tools.get_invitation(client, invite_assignment_invitation_id)
invite_assignment_invitation = openreview.tools.get_invitation(venue_client, invite_assignment_invitation_id)

if invite_assignment_invitation:
grouped_edges = client.get_grouped_edges(invitation=invite_assignment_invitation.id, label='Pending Sign Up', groupby='tail')
grouped_edges = venue_client.get_grouped_edges(invitation=invite_assignment_invitation.id, label='Pending Sign Up', groupby='tail')
print('Pending sign up edges found', len(grouped_edges))

for grouped_edge in grouped_edges:

tail = grouped_edge['id']['tail']
profiles=openreview.tools.get_profiles(client, [tail], with_publications=True, with_relations=True)
profiles=openreview.tools.get_profiles(venue_client, [tail], with_publications=True, with_relations=True)

if profiles and profiles[0].active:

Expand All @@ -1508,33 +1528,73 @@ def mark_as_accepted(venue_group, edge, submission, user_profile, invite_assignm
for edge in edges:

edge = openreview.api.Edge.from_json(edge)
submission=client.get_note(id=edge.head)
submission=venue_client.get_note(id=edge.head)

if submission.content['venueid']['value'] == venue_group.content.get('submission_venue_id', {}).get('value'):

## Check if there is already an accepted edge for that profile id
accepted_edges = client.get_edges(invitation=invite_assignment_invitation.id, label='Accepted', head=submission.id, tail=user_profile.id)
accepted_edges = venue_client.get_edges(invitation=invite_assignment_invitation.id, label='Accepted', head=submission.id, tail=user_profile.id)

if not accepted_edges:
## Check if the user was invited again with a profile id
invitation_edges = client.get_edges(invitation=invite_assignment_invitation.id, label='Invitation Sent', head=submission.id, tail=user_profile.id)
invitation_edges = venue_client.get_edges(invitation=invite_assignment_invitation.id, label='Invitation Sent', head=submission.id, tail=user_profile.id)
if invitation_edges:
invitation_edge = invitation_edges[0]
print(f'User invited twice, remove double invitation edge {invitation_edge.id}')
invitation_edge.ddate = openreview.tools.datetime_millis(datetime.datetime.now())
client.post_edge(invitation_edge)
venue_client.post_edge(invitation_edge)

## Check venue profile requirements
min_requirements = venue_group.content.get('invited_reviewer_profile_minimum_requirements', {}).get('value')
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you move this code to tools.py?

is_incomplete = False

if min_requirements:
# { content.relations: 2, content.dblp: true, active: true }
for profile_path, expected_value in min_requirements.items():
path_items = profile_path.split('.')
actual_value = user_profile

## Resolve actual value from the profile
for item in path_items:
if isinstance(actual_value, openreview.Profile):
actual_value = getattr(actual_value, item, None)
elif isinstance(actual_value, dict):
actual_value = actual_value.get(item)
else:
## Can't traverse, but more items to resolve
actual_value = None

if actual_value is None:
break

## Check against requirement
## Check number of entries
if type(expected_value) == int:
if not actual_value or not isinstance(actual_value, list) or len(actual_value) < expected_value:
is_incomplete = True
break
## Check if field exists in profile (e.g. links)
elif expected_value is True and not actual_value:
is_incomplete = True
break
else:
print(f'Invalid path: {profile_path}')
Copy link
Preview

Copilot AI May 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of simply printing 'Invalid path', consider using a logging framework or raising a controlled warning so that unexpected data types in the profile requirements are properly tracked and can be handled or debugged more systematically.

Copilot uses AI. Check for mistakes.


if is_incomplete:
print(f'Sending messages for incomplete profile {user_profile.id} for paper {edge.head}')
send_incomplete_profile_notification(venue_group, edge, submission, user_profile)
else:
## Check conflicts
author_profiles = openreview.tools.get_profiles(venue_client, submission.content['authorids']['value'], with_publications=True, with_relations=True)
conflicts=openreview.tools.get_conflicts(author_profiles, user_profile, policy=venue_group.content.get('reviewers_conflict_policy', {}).get('value'), n_years=venue_group.content.get('reviewers_conflict_n_years', {}).get('value'))

## Check conflicts
author_profiles = openreview.tools.get_profiles(client, submission.content['authorids']['value'], with_publications=True, with_relations=True)
conflicts=openreview.tools.get_conflicts(author_profiles, user_profile, policy=venue_group.content.get('reviewers_conflict_policy', {}).get('value'), n_years=venue_group.content.get('reviewers_conflict_n_years', {}).get('value'))
if conflicts:
print(f'Conflicts detected for {edge.head} and {user_profile.id}', conflicts)
mark_as_conflict(venue_group, edge, submission, user_profile)
else:
print(f'Mark accepted for {edge.head} and {user_profile.id}')
mark_as_accepted(venue_group, edge, submission, user_profile, invite_assignment_invitation)

if conflicts:
print(f'Conflicts detected for {edge.head} and {user_profile.id}', conflicts)
mark_as_conflict(venue_group, edge, submission, user_profile)
else:
print(f'Mark accepted for {edge.head} and {user_profile.id}')
mark_as_accepted(venue_group, edge, submission, user_profile, invite_assignment_invitation)

else:
print("user already accepted with another invitation edge", submission.id, user_profile.id)

Expand Down
Loading