Skip to content

Commit afecd13

Browse files
julien-langjulien-lang
committed
Tests
1 parent f8b7396 commit afecd13

File tree

2 files changed

+109
-122
lines changed

2 files changed

+109
-122
lines changed

‎azure-pipelines-templates/run-tests.yml‎

Lines changed: 95 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -49,104 +49,103 @@ parameters:
4949

5050

5151
steps:
52-
# Specifies which version of Python we want to use. That's where the strategy comes in.
53-
# Each job will share this set of steps, but each of them will receive a different
54-
# $(python.version)
55-
# TODO: We should provide `githubToken` if we want to download a python release.
56-
# Otherwise we may hit the GitHub anonymous download limit.
57-
- task: UsePythonVersion@0
58-
inputs:
59-
versionSpec: ${{ parameters.python_version }}
52+
# Specifies which version of Python we want to use. That's where the strategy comes in.
53+
# Each job will share this set of steps, but each of them will receive a different
54+
# $(python.version)
55+
# TODO: We should provide `githubToken` if we want to download a python release.
56+
# Otherwise we may hit the GitHub anonymous download limit.
57+
- task: UsePythonVersion@0
58+
inputs:
59+
versionSpec: ${{ parameters.python_version }}
6060

61-
# Install all dependencies needed for running the tests. This command is good
62-
# for all OSes
63-
- task: Bash@3
64-
displayName: Install dependencies
65-
inputs:
66-
targetType: inline
67-
script: |
68-
python --version
61+
# Install all dependencies needed for running the tests. This command is good
62+
# for all OSes
63+
- task: Bash@3
64+
displayName: Install dependencies
65+
inputs:
66+
targetType: inline
67+
script: |
68+
python --version
6969
python -c "import sys; print(sys.platform)"
70+
pip install --upgrade pip
71+
pip install --upgrade setuptools wheel
72+
pip install --upgrade --requirement tests/ci_requirements.txt
7073
71-
# pip install --upgrade pip
72-
# pip install --upgrade setuptools wheel
73-
# pip install --upgrade --requirement tests/ci_requirements.txt
74+
# The {{}} syntax is meant for the the pre-processor of Azure pipeline. Every statement inside
75+
# a {{}} block will be evaluated and substituted before the file is parsed to create the jobs.
76+
# So here we're inserting an extra step if the template is being invoked for Windows.
77+
- ${{ if startsWith(parameters.vm_image, 'windows') }}:
78+
# On Windows, we need to update the certificates, the cert store is missing the newer one
79+
# from Amazon like some clients experienced a while back. Who would have thought Microsoft
80+
# would have been out of date! ;)
81+
- powershell: |
82+
$cert_url = "https://www.amazontrust.com/repository/SFSRootCAG2.cer"
83+
$cert_file = New-TemporaryFile
84+
Invoke-WebRequest -Uri $cert_url -UseBasicParsing -OutFile $cert_file.FullName
85+
Import-Certificate -FilePath $cert_file.FullName -CertStoreLocation Cert:\LocalMachine\Root
86+
displayName: Updating OS Certificates
7487
75-
# # The {{}} syntax is meant for the the pre-processor of Azure pipeline. Every statement inside
76-
# # a {{}} block will be evaluated and substituted before the file is parsed to create the jobs.
77-
# # So here we're inserting an extra step if the template is being invoked for Windows.
78-
# - ${{ if startsWith(parameters.vm_image, 'windows') }}:
79-
# # On Windows, we need to update the certificates, the cert store is missing the newer one
80-
# # from Amazon like some clients experienced a while back. Who would have thought Microsoft
81-
# # would have been out of date! ;)
82-
# - powershell: |
83-
# $cert_url = "https://www.amazontrust.com/repository/SFSRootCAG2.cer"
84-
# $cert_file = New-TemporaryFile
85-
# Invoke-WebRequest -Uri $cert_url -UseBasicParsing -OutFile $cert_file.FullName
86-
# Import-Certificate -FilePath $cert_file.FullName -CertStoreLocation Cert:\LocalMachine\Root
87-
# displayName: Updating OS Certificates
88+
# Runs the tests and generates test coverage. The tests results are uploaded to Azure Pipelines in the
89+
# Tests tab of the build and each test run will be named after the --test-run-title argument to pytest,
90+
# for example 'Windows - 2.7'
91+
- task: Bash@3
92+
displayName: Running tests
93+
inputs:
94+
targetType: inline
95+
script: |
96+
cp ./tests/example_config ./tests/config
97+
pytest --durations=0 -v \
98+
--cov shotgun_api3 --cov-report xml \
99+
--test-run-title="${{parameters.name}}-${{ parameters.python_version }}"
100+
env:
101+
# Pass the values needed to authenticate with the Flow Production Tracking site and create some entities.
102+
# Remember, on a pull request from a client or on forked repos, those variables
103+
# will be empty!
104+
SG_SERVER_URL: $(ci_site)
105+
SG_SCRIPT_NAME: $(ci_site_script_name)
106+
SG_API_KEY: $(ci_site_script_key)
107+
# The unit tests manipulate the user and project during the tests, which can cause collisions,
108+
# so sandbox each build variant.
109+
# Ideally, we would use the agent name here. The problem is that the agent name is in a build
110+
# variable, so we can't edit it's value through a ${{replace(a,b,c)}} expression, which are evaluated before
111+
# build variables are available. Because of this, we need to find another way to generate a
112+
# unique login. So instead, we'll use the the name of the platform and the python version,
113+
# which should make it unique.
114+
SG_HUMAN_LOGIN: $(python_api_human_login)-${{parameters.name}}-$(python.version)
115+
# This will give a user name like 'something macOS 2.7'
116+
SG_HUMAN_NAME: $(python_api_human_name) ${{parameters.name}} $(python.version)
117+
SG_HUMAN_PASSWORD: $(python_api_human_password)
118+
# So, first, we need to make sure that two builds running at the same time do not manipulate
119+
# the same entities, so we're sandboxing build nodes based on their name.
120+
SG_PROJECT_NAME: Python API CI - $(Agent.Name)
121+
# The entities created and then reused between tests assume that the same user is always
122+
# manipulating them. Because different builds will be assigned different agents and therefore
123+
# different projects, it means each project needs to have an entity specific to a given user.
124+
# Again, this would have been a lot simpler if we could simply have had a login based on the
125+
# agent name, but alas, the agent name has a space in it which needs to be replaced to something
126+
# else and string substitution can't be made on build variables, only template parameters.
127+
SG_ASSET_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
128+
SG_VERSION_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
129+
SG_SHOT_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
130+
SG_TASK_CONTENT: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
131+
SG_PLAYLIST_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
88132

89-
# # Runs the tests and generates test coverage. The tests results are uploaded to Azure Pipelines in the
90-
# # Tests tab of the build and each test run will be named after the --test-run-title argument to pytest,
91-
# # for example 'Windows - 2.7'
92-
# - task: Bash@3
93-
# displayName: Running tests
94-
# inputs:
95-
# targetType: inline
96-
# script: |
97-
# cp ./tests/example_config ./tests/config
98-
# pytest --durations=0 -v \
99-
# --cov shotgun_api3 --cov-report xml \
100-
# --test-run-title="${{parameters.name}}-${{ parameters.python_version }}"
101-
# env:
102-
# # Pass the values needed to authenticate with the Flow Production Tracking site and create some entities.
103-
# # Remember, on a pull request from a client or on forked repos, those variables
104-
# # will be empty!
105-
# SG_SERVER_URL: $(ci_site)
106-
# SG_SCRIPT_NAME: $(ci_site_script_name)
107-
# SG_API_KEY: $(ci_site_script_key)
108-
# # The unit tests manipulate the user and project during the tests, which can cause collisions,
109-
# # so sandbox each build variant.
110-
# # Ideally, we would use the agent name here. The problem is that the agent name is in a build
111-
# # variable, so we can't edit it's value through a ${{replace(a,b,c)}} expression, which are evaluated before
112-
# # build variables are available. Because of this, we need to find another way to generate a
113-
# # unique login. So instead, we'll use the the name of the platform and the python version,
114-
# # which should make it unique.
115-
# SG_HUMAN_LOGIN: $(python_api_human_login)-${{parameters.name}}-$(python.version)
116-
# # This will give a user name like 'something macOS 2.7'
117-
# SG_HUMAN_NAME: $(python_api_human_name) ${{parameters.name}} $(python.version)
118-
# SG_HUMAN_PASSWORD: $(python_api_human_password)
119-
# # So, first, we need to make sure that two builds running at the same time do not manipulate
120-
# # the same entities, so we're sandboxing build nodes based on their name.
121-
# SG_PROJECT_NAME: Python API CI - $(Agent.Name)
122-
# # The entities created and then reused between tests assume that the same user is always
123-
# # manipulating them. Because different builds will be assigned different agents and therefore
124-
# # different projects, it means each project needs to have an entity specific to a given user.
125-
# # Again, this would have been a lot simpler if we could simply have had a login based on the
126-
# # agent name, but alas, the agent name has a space in it which needs to be replaced to something
127-
# # else and string substitution can't be made on build variables, only template parameters.
128-
# SG_ASSET_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
129-
# SG_VERSION_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
130-
# SG_SHOT_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
131-
# SG_TASK_CONTENT: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
132-
# SG_PLAYLIST_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
133-
134-
# # Upload the code coverage result to codecov.io.
135-
# - ${{ if startsWith(parameters.vm_image, 'windows') }}:
136-
# - powershell: |
137-
# $ProgressPreference = 'SilentlyContinue'
138-
# Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
139-
# .\codecov.exe -f coverage.xml
140-
# displayName: Uploading code coverage
141-
# - ${{ elseif startsWith(parameters.vm_image, 'macos') }}:
142-
# - script: |
143-
# curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov
144-
# chmod +x codecov
145-
# ./codecov -f coverage.xml
146-
# displayName: Uploading code coverage
147-
# - ${{ else }}:
148-
# - script: |
149-
# curl -Os https://uploader.codecov.io/latest/linux/codecov
150-
# chmod +x codecov
151-
# ./codecov -f coverage.xml
152-
# displayName: Uploading code cover
133+
# Upload the code coverage result to codecov.io.
134+
- ${{ if startsWith(parameters.vm_image, 'windows') }}:
135+
- powershell: |
136+
$ProgressPreference = 'SilentlyContinue'
137+
Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
138+
.\codecov.exe -f coverage.xml
139+
displayName: Uploading code coverage
140+
- ${{ elseif startsWith(parameters.vm_image, 'macos') }}:
141+
- script: |
142+
curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov
143+
chmod +x codecov
144+
./codecov -f coverage.xml
145+
displayName: Uploading code coverage
146+
- ${{ else }}:
147+
- script: |
148+
curl -Os https://uploader.codecov.io/latest/linux/codecov
149+
chmod +x codecov
150+
./codecov -f coverage.xml
151+
displayName: Uploading code cover

‎azure-pipelines.yml‎

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -59,24 +59,28 @@ pr:
5959
- "*"
6060

6161
parameters:
62-
- name: python_versions
62+
- name: test_python_versions
6363
type: object
6464
default:
6565
- '3.9'
6666
- '3.10'
6767
- '3.11'
6868

69-
- name: os_versions
69+
- name: test_os_versions
7070
type: object
7171
default:
72-
- name: Linux
73-
vm_image: ubuntu-latest
72+
- name: Windows
73+
vm_image: windows-latest
74+
# Select Windows first because it's slower
7475

7576
- name: macOS
7677
vm_image: macOS-latest
78+
# then mac because 2nd slower
79+
80+
- name: Linux
81+
vm_image: ubuntu-latest
82+
# finally Linux is the fastest
7783

78-
- name: Windows
79-
vm_image: windows-latest
8084

8185
# This here is the list of jobs we want to run for our build.
8286
# Jobs run in parallel.
@@ -91,20 +95,13 @@ jobs:
9195

9296
- template: azure-pipelines-templates/type_checking.yml
9397

94-
# - ${{ each os_version in parameters.os_versions }}:
95-
# - ${{ each python_version in parameters.python_versions }}:
96-
# - template: azure-pipelines-templates/run-tests.yml
97-
# parameters:
98-
# name: "${{ os_version.name }} Python ${{ python_version }}"
99-
# vm_image: ${{ os_version.vm_image }}
100-
# python_version: ${{ python_version }}
101-
98+
# Run tests for all combinations of Python and OS
10299
- job:
103100
strategy:
104101
matrix:
105-
${{ each python_version in parameters.python_versions }}:
106-
${{ each os_version in parameters.os_versions }}:
107-
"Run Tests ${{ os_version.name }} Python ${{ python_version }}":
102+
${{ each python_version in parameters.test_python_versions }}:
103+
${{ each os_version in parameters.test_os_versions }}:
104+
"Tests ${{ os_version.name }} Python ${{ python_version }}":
108105
imageName: '${{ os_version.vm_image }}'
109106
python.version: '${{ python_version }}'
110107

@@ -121,12 +118,3 @@ jobs:
121118
- template: azure-pipelines-templates/run-tests.yml
122119
parameters:
123120
python_version: $(python.version)
124-
125-
126-
# - task: UsePythonVersion@0
127-
# inputs:
128-
# versionSpec: $(python.version)
129-
130-
# - script: |
131-
# python --version
132-
# python -c "import sys; print(sys.platform)"

0 commit comments

Comments
 (0)