Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Avoid fetching thousands of pages from GitLab API #377

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Avoid fetching thousands of pages from GitLab API #377

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 41 additions & 6 deletions did/plugins/gitlab.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
# Retry fetching
GITLAB_ATTEMPTS = 5
GITLAB_INTERVAL = 5
GITLAB_MAX_PAGE_LIST = 20

# Identifier padding
PADDING = 3
Expand Down Expand Up @@ -143,6 +144,17 @@ def get_project_mr(self, project_id, mr_id):
def get_project_mrs(self, project_id):
if project_id not in self.project_mrs:
query = 'projects/{0}/merge_requests'.format(project_id)

# Check that this will not return more then 20 pages; if it does,
# skip rather than spending a large amount of time to query all of
# the results.
result = self._get_gitlab_api(query)
result.raise_for_status()
log.debug("Page count for {0}: {1}".format(query, result.headers.get('x-total-pages')))
if int(result.headers.get('x-total-pages', GITLAB_MAX_PAGE_LIST + 1)) > GITLAB_MAX_PAGE_LIST:
self.project_mrs[project_id] = []
return []

self.project_mrs[project_id] = self._get_gitlab_api_list(
query, get_all_results=True)
return self.project_mrs[project_id]
Expand All @@ -155,6 +167,17 @@ def get_project_issue(self, project_id, issue_id):
def get_project_issues(self, project_id):
if project_id not in self.project_issues:
query = 'projects/{0}/issues'.format(project_id)

# Check that this will not return more then 20 pages; if it does,
# skip rather than spending a large amount of time to query all of
# the results.
result = self._get_gitlab_api(query)
result.raise_for_status()
log.debug("Page count for {0}: {1}".format(query, result.headers.get('x-total-pages')))
if int(result.headers.get('x-total-pages', GITLAB_MAX_PAGE_LIST + 1)) > GITLAB_MAX_PAGE_LIST:
self.project_issues[project_id] = []
return []

self.project_issues[project_id] = self._get_gitlab_api_list(
query, get_all_results=True)
return self.project_issues[project_id]
Expand Down Expand Up @@ -200,8 +223,12 @@ def __init__(self, data, parent):
self.title = data['target_title']

def iid(self):
return self.gitlabapi.get_project_issue(
self.data['project_id'], self.data['target_id'])['iid']
issue = self.gitlabapi.get_project_issue(
self.data['project_id'], self.data['target_id'])

if issue is not None:
return issue['iid']
return "unknown"

def __str__(self):
""" String representation """
Expand All @@ -226,8 +253,12 @@ def __str__(self):
class MergeRequest(Issue):

def iid(self):
return self.gitlabapi.get_project_mr(
self.data['project_id'], self.data['target_id'])['iid']
merge_request = self.gitlabapi.get_project_mr(
self.data['project_id'], self.data['target_id'])
if merge_request is not None:
return merge_request['iid']

return "unknown"


class Note(Issue):
Expand All @@ -244,9 +275,13 @@ def iid(self):
return issue['iid']
return 'unknown'
elif self.data['note']['noteable_type'] == 'MergeRequest':
return self.gitlabapi.get_project_mr(
merge_request = self.gitlabapi.get_project_mr(
self.data['project_id'],
self.data['note']['noteable_id'])['iid']
self.data['note']['noteable_id'])

if merge_request is not None:
return merge_request['iid']
return "unknown"
else:
return "unknown"

Expand Down