This repository has been archived by the owner on Aug 4, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Bug 1341214 - Add a small API to handle taskcluster queue and index r…
…equests. r=dustin Various modules under taskcluster are doing ad-hoc url formatting or requests to taskcluster services. While we could use the taskcluster client python module, it's kind of overkill for the simple requests done here. So instead of vendoring that module, create a smaller one with a limited set of functions we need. This changes the behavior of the get_artifact function to return a file-like object when the file is neither a json nor a yaml, but that branch was never used (and was actually returning an unassigned variable, so it was broken anyways). At the same time, make the function that does HTTP requests more error-resistant, using urllib3's Retry with a backoff factor. Also add a function that retrieves the list of artifacts, that while currently unused, will be used by `mach artifact` shortly.
- Loading branch information
Showing
12 changed files
with
116 additions
and
79 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| # -*- coding: utf-8 -*- | ||
|
|
||
| # This Source Code Form is subject to the terms of the Mozilla Public | ||
| # License, v. 2.0. If a copy of the MPL was not distributed with this | ||
| # file, You can obtain one at http://mozilla.org/MPL/2.0/. | ||
|
|
||
| from __future__ import absolute_import, print_function, unicode_literals | ||
|
|
||
| import functools | ||
| import yaml | ||
| import requests | ||
| from mozbuild.util import memoize | ||
| from requests.packages.urllib3.util.retry import Retry | ||
| from requests.adapters import HTTPAdapter | ||
|
|
||
|
|
||
| @memoize | ||
| def get_session(): | ||
| session = requests.Session() | ||
| retry = Retry(total=5, backoff_factor=0.1, | ||
| status_forcelist=[500, 502, 503, 504]) | ||
| session.mount('http://', HTTPAdapter(max_retries=retry)) | ||
| session.mount('https://', HTTPAdapter(max_retries=retry)) | ||
| return session | ||
|
|
||
|
|
||
| def _do_request(url): | ||
| session = get_session() | ||
| return session.get(url, stream=True) | ||
|
|
||
|
|
||
| def get_artifact_url(task_id, path, use_proxy=False): | ||
| if use_proxy: | ||
| ARTIFACT_URL = 'http://taskcluster/queue/v1/task/{}/artifacts/{}' | ||
| else: | ||
| ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' | ||
| return ARTIFACT_URL.format(task_id, path) | ||
|
|
||
|
|
||
| def get_artifact(task_id, path, use_proxy=False): | ||
| """ | ||
| Returns the artifact with the given path for the given task id. | ||
| If the path ends with ".json" or ".yml", the content is deserialized as, | ||
| respectively, json or yaml, and the corresponding python data (usually | ||
| dict) is returned. | ||
| For other types of content, a file-like object is returned. | ||
| """ | ||
| response = _do_request(get_artifact_url(task_id, path, use_proxy)) | ||
| response.raise_for_status() | ||
| if path.endswith('.json'): | ||
| return response.json() | ||
| if path.endswith('.yml'): | ||
| return yaml.load(response.text) | ||
| response.raw.read = functools.partial(response.raw.read, | ||
| decode_content=True) | ||
| return response.raw | ||
|
|
||
|
|
||
| def list_artifacts(task_id, use_proxy=False): | ||
| response = _do_request(get_artifact_url(task_id, '', use_proxy).rstrip('/')) | ||
| response.raise_for_status() | ||
| return response.json()['artifacts'] | ||
|
|
||
|
|
||
| def get_index_url(index_path, use_proxy=False): | ||
| if use_proxy: | ||
| INDEX_URL = 'http://taskcluster/index/v1/task/{}' | ||
| else: | ||
| INDEX_URL = 'https://index.taskcluster.net/v1/task/{}' | ||
| return INDEX_URL.format(index_path) | ||
|
|
||
|
|
||
| def find_task_id(index_path, use_proxy=False): | ||
| response = _do_request(get_index_url(index_path, use_proxy)) | ||
| response.raise_for_status() | ||
| return response.json()['taskId'] |