Skip to content

Commit cfa2380

Browse files
julien-langjulien-lang
committed
tests
1 parent 3e0635d commit cfa2380

File tree

2 files changed

+125
-142
lines changed

2 files changed

+125
-142
lines changed

‎azure-pipelines-templates/run-tests.yml‎

Lines changed: 84 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ jobs:
3838
# name of the keys under the strategy.matrix scope. So for each OS we'll have "<OS> Python 3.9" and
3939
# "<OS> Python 3.10".
4040
- job:
41+
strategy:
42+
parallel: 3
4143
displayName: ${{ parameters.name }}
4244
pool:
4345
vmImage: ${{ parameters.vm_image }}
@@ -63,85 +65,88 @@ jobs:
6365
inputs:
6466
targetType: inline
6567
script: |
66-
pip install --upgrade pip
67-
pip install --upgrade setuptools wheel
68-
pip install --upgrade --requirement tests/ci_requirements.txt
68+
python --version
69+
python -c "import sys; print(sys.platform)"
6970
70-
# The {{}} syntax is meant for the the pre-processor of Azure pipeline. Every statement inside
71-
# a {{}} block will be evaluated and substituted before the file is parsed to create the jobs.
72-
# So here we're inserting an extra step if the template is being invoked for Windows.
73-
- ${{ if startsWith(parameters.vm_image, 'windows') }}:
74-
# On Windows, we need to update the certificates, the cert store is missing the newer one
75-
# from Amazon like some clients experienced a while back. Who would have thought Microsoft
76-
# would have been out of date! ;)
77-
- powershell: |
78-
$cert_url = "https://www.amazontrust.com/repository/SFSRootCAG2.cer"
79-
$cert_file = New-TemporaryFile
80-
Invoke-WebRequest -Uri $cert_url -UseBasicParsing -OutFile $cert_file.FullName
81-
Import-Certificate -FilePath $cert_file.FullName -CertStoreLocation Cert:\LocalMachine\Root
82-
displayName: Updating OS Certificates
71+
# pip install --upgrade pip
72+
# pip install --upgrade setuptools wheel
73+
# pip install --upgrade --requirement tests/ci_requirements.txt
8374

84-
# Runs the tests and generates test coverage. The tests results are uploaded to Azure Pipelines in the
85-
# Tests tab of the build and each test run will be named after the --test-run-title argument to pytest,
86-
# for example 'Windows - 2.7'
87-
- task: Bash@3
88-
displayName: Running tests
89-
inputs:
90-
targetType: inline
91-
script: |
92-
cp ./tests/example_config ./tests/config
93-
pytest --durations=0 -v \
94-
--cov shotgun_api3 --cov-report xml \
95-
--test-run-title="${{parameters.name}}-${{ parameters.python_version }}"
96-
env:
97-
# Pass the values needed to authenticate with the Flow Production Tracking site and create some entities.
98-
# Remember, on a pull request from a client or on forked repos, those variables
99-
# will be empty!
100-
SG_SERVER_URL: $(ci_site)
101-
SG_SCRIPT_NAME: $(ci_site_script_name)
102-
SG_API_KEY: $(ci_site_script_key)
103-
# The unit tests manipulate the user and project during the tests, which can cause collisions,
104-
# so sandbox each build variant.
105-
# Ideally, we would use the agent name here. The problem is that the agent name is in a build
106-
# variable, so we can't edit it's value through a ${{replace(a,b,c)}} expression, which are evaluated before
107-
# build variables are available. Because of this, we need to find another way to generate a
108-
# unique login. So instead, we'll use the the name of the platform and the python version,
109-
# which should make it unique.
110-
SG_HUMAN_LOGIN: $(python_api_human_login)-${{parameters.name}}-$(python.version)
111-
# This will give a user name like 'something macOS 2.7'
112-
SG_HUMAN_NAME: $(python_api_human_name) ${{parameters.name}} $(python.version)
113-
SG_HUMAN_PASSWORD: $(python_api_human_password)
114-
# So, first, we need to make sure that two builds running at the same time do not manipulate
115-
# the same entities, so we're sandboxing build nodes based on their name.
116-
SG_PROJECT_NAME: Python API CI - $(Agent.Name)
117-
# The entities created and then reused between tests assume that the same user is always
118-
# manipulating them. Because different builds will be assigned different agents and therefore
119-
# different projects, it means each project needs to have an entity specific to a given user.
120-
# Again, this would have been a lot simpler if we could simply have had a login based on the
121-
# agent name, but alas, the agent name has a space in it which needs to be replaced to something
122-
# else and string substitution can't be made on build variables, only template parameters.
123-
SG_ASSET_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
124-
SG_VERSION_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
125-
SG_SHOT_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
126-
SG_TASK_CONTENT: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
127-
SG_PLAYLIST_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
75+
# # The {{}} syntax is meant for the the pre-processor of Azure pipeline. Every statement inside
76+
# # a {{}} block will be evaluated and substituted before the file is parsed to create the jobs.
77+
# # So here we're inserting an extra step if the template is being invoked for Windows.
78+
# - ${{ if startsWith(parameters.vm_image, 'windows') }}:
79+
# # On Windows, we need to update the certificates, the cert store is missing the newer one
80+
# # from Amazon like some clients experienced a while back. Who would have thought Microsoft
81+
# # would have been out of date! ;)
82+
# - powershell: |
83+
# $cert_url = "https://www.amazontrust.com/repository/SFSRootCAG2.cer"
84+
# $cert_file = New-TemporaryFile
85+
# Invoke-WebRequest -Uri $cert_url -UseBasicParsing -OutFile $cert_file.FullName
86+
# Import-Certificate -FilePath $cert_file.FullName -CertStoreLocation Cert:\LocalMachine\Root
87+
# displayName: Updating OS Certificates
88+
89+
# # Runs the tests and generates test coverage. The tests results are uploaded to Azure Pipelines in the
90+
# # Tests tab of the build and each test run will be named after the --test-run-title argument to pytest,
91+
# # for example 'Windows - 2.7'
92+
# - task: Bash@3
93+
# displayName: Running tests
94+
# inputs:
95+
# targetType: inline
96+
# script: |
97+
# cp ./tests/example_config ./tests/config
98+
# pytest --durations=0 -v \
99+
# --cov shotgun_api3 --cov-report xml \
100+
# --test-run-title="${{parameters.name}}-${{ parameters.python_version }}"
101+
# env:
102+
# # Pass the values needed to authenticate with the Flow Production Tracking site and create some entities.
103+
# # Remember, on a pull request from a client or on forked repos, those variables
104+
# # will be empty!
105+
# SG_SERVER_URL: $(ci_site)
106+
# SG_SCRIPT_NAME: $(ci_site_script_name)
107+
# SG_API_KEY: $(ci_site_script_key)
108+
# # The unit tests manipulate the user and project during the tests, which can cause collisions,
109+
# # so sandbox each build variant.
110+
# # Ideally, we would use the agent name here. The problem is that the agent name is in a build
111+
# # variable, so we can't edit it's value through a ${{replace(a,b,c)}} expression, which are evaluated before
112+
# # build variables are available. Because of this, we need to find another way to generate a
113+
# # unique login. So instead, we'll use the the name of the platform and the python version,
114+
# # which should make it unique.
115+
# SG_HUMAN_LOGIN: $(python_api_human_login)-${{parameters.name}}-$(python.version)
116+
# # This will give a user name like 'something macOS 2.7'
117+
# SG_HUMAN_NAME: $(python_api_human_name) ${{parameters.name}} $(python.version)
118+
# SG_HUMAN_PASSWORD: $(python_api_human_password)
119+
# # So, first, we need to make sure that two builds running at the same time do not manipulate
120+
# # the same entities, so we're sandboxing build nodes based on their name.
121+
# SG_PROJECT_NAME: Python API CI - $(Agent.Name)
122+
# # The entities created and then reused between tests assume that the same user is always
123+
# # manipulating them. Because different builds will be assigned different agents and therefore
124+
# # different projects, it means each project needs to have an entity specific to a given user.
125+
# # Again, this would have been a lot simpler if we could simply have had a login based on the
126+
# # agent name, but alas, the agent name has a space in it which needs to be replaced to something
127+
# # else and string substitution can't be made on build variables, only template parameters.
128+
# SG_ASSET_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
129+
# SG_VERSION_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
130+
# SG_SHOT_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
131+
# SG_TASK_CONTENT: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
132+
# SG_PLAYLIST_CODE: CI-$(python_api_human_login)-${{parameters.name}}-$(python.version)
128133

129-
# Upload the code coverage result to codecov.io.
130-
- ${{ if startsWith(parameters.vm_image, 'windows') }}:
131-
- powershell: |
132-
$ProgressPreference = 'SilentlyContinue'
133-
Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
134-
.\codecov.exe -f coverage.xml
135-
displayName: Uploading code coverage
136-
- ${{ elseif startsWith(parameters.vm_image, 'macos') }}:
137-
- script: |
138-
curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov
139-
chmod +x codecov
140-
./codecov -f coverage.xml
141-
displayName: Uploading code coverage
142-
- ${{ else }}:
143-
- script: |
144-
curl -Os https://uploader.codecov.io/latest/linux/codecov
145-
chmod +x codecov
146-
./codecov -f coverage.xml
147-
displayName: Uploading code cover
134+
# # Upload the code coverage result to codecov.io.
135+
# - ${{ if startsWith(parameters.vm_image, 'windows') }}:
136+
# - powershell: |
137+
# $ProgressPreference = 'SilentlyContinue'
138+
# Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe
139+
# .\codecov.exe -f coverage.xml
140+
# displayName: Uploading code coverage
141+
# - ${{ elseif startsWith(parameters.vm_image, 'macos') }}:
142+
# - script: |
143+
# curl -Os https://uploader.codecov.io/v0.7.3/macos/codecov
144+
# chmod +x codecov
145+
# ./codecov -f coverage.xml
146+
# displayName: Uploading code coverage
147+
# - ${{ else }}:
148+
# - script: |
149+
# curl -Os https://uploader.codecov.io/latest/linux/codecov
150+
# chmod +x codecov
151+
# ./codecov -f coverage.xml
152+
# displayName: Uploading code cover

‎azure-pipelines.yml‎

Lines changed: 41 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -81,66 +81,44 @@ parameters:
8181
# This here is the list of jobs we want to run for our build.
8282
# Jobs run in parallel.
8383
jobs:
84-
# - template: build-pipeline.yml@templates
85-
# parameters:
86-
# # Python API does not follow the exact same Python version lifecycle than
87-
# # Toolkit. So we prefer to control the test execution here instead.
88-
# has_unit_tests: false
89-
90-
# has_ui_resources: false
91-
92-
# - template: azure-pipelines-templates/type_checking.yml
93-
94-
95-
# - ${{ each os_version in parameters.os_versions }}:
96-
# - ${{ each python_version in parameters.python_versions }}:
97-
# - template: azure-pipelines-templates/run-tests.yml
98-
# parameters:
99-
# name: "${{ os_version.name }} Python ${{ python_version }}"
100-
# vm_image: ${{ os_version.vm_image }}
101-
# python_version: ${{ python_version }}
102-
103-
104-
- job: ## "Run_tests" ## I don't have to ...
105-
strategy:
106-
matrix:
107-
${{ each python_version in parameters.python_versions }}:
108-
${{ each os_version in parameters.os_versions }}:
109-
"Run Tests ${{ os_version.name }} Python ${{ python_version }}":
110-
imageName: '${{ os_version.vm_image }}'
111-
python.version: '${{ python_version }}'
112-
# linux_py310:
113-
# imageName: 'ubuntu-latest'
114-
# python.version: '3.10'
115-
# mac_py310:
116-
# imageName: 'macOS-latest'
117-
# python.version: '3.10'
118-
# windows_py310 :
119-
# imageName: 'windows-latest'
120-
# python.version: '3.10'
121-
# linux_py311:
122-
# imageName: 'ubuntu-latest'
123-
# python.version: '3.11'
124-
# mac_py311:
125-
# imageName: 'macOS-latest'
126-
# python.version: '3.11'
127-
# windows_py311 :
128-
# imageName: 'windows-latest'
129-
# python.version: '3.11'
130-
maxParallel: 3
131-
# limiting the parallel jobs to 3 to avoid race condition issues between tests..... :(
132-
133-
# displayName: "Run tests_$( imageName )_$( python.version )"
134-
pool:
135-
vmImage: $(imageName)
136-
steps:
137-
- task: UsePythonVersion@0
138-
inputs:
139-
versionSpec: $(python.version)
140-
141-
- script: |
142-
python --version
143-
python -c "import sys; print(sys.platform)"
144-
145-
146-
# job: TODO_Tests_${{ matrix.imageName }}_${{ matrix.python.version }}
84+
- template: build-pipeline.yml@templates
85+
parameters:
86+
# Python API does not follow the exact same Python version lifecycle than
87+
# Toolkit. So we prefer to control the test execution here instead.
88+
has_unit_tests: false
89+
90+
has_ui_resources: false
91+
92+
- template: azure-pipelines-templates/type_checking.yml
93+
94+
- ${{ each os_version in parameters.os_versions }}:
95+
- ${{ each python_version in parameters.python_versions }}:
96+
- template: azure-pipelines-templates/run-tests.yml
97+
parameters:
98+
name: "${{ os_version.name }} Python ${{ python_version }}"
99+
vm_image: ${{ os_version.vm_image }}
100+
python_version: ${{ python_version }}
101+
102+
# - job: ## "Run_tests" ## I don't have to ...
103+
# strategy:
104+
# matrix:
105+
# ${{ each python_version in parameters.python_versions }}:
106+
# ${{ each os_version in parameters.os_versions }}:
107+
# "Run Tests ${{ os_version.name }} Python ${{ python_version }}":
108+
# imageName: '${{ os_version.vm_image }}'
109+
# python.version: '${{ python_version }}'
110+
111+
# maxParallel: 3
112+
# # limiting the parallel jobs to 3 to avoid race condition issues between tests..... :(
113+
114+
# # displayName: "Run tests_$( imageName )_$( python.version )"
115+
# pool:
116+
# vmImage: $(imageName)
117+
# steps:
118+
# - task: UsePythonVersion@0
119+
# inputs:
120+
# versionSpec: $(python.version)
121+
122+
# - script: |
123+
# python --version
124+
# python -c "import sys; print(sys.platform)"

0 commit comments

Comments
 (0)