Skip to content

Commit d198728

Browse files
EgorBoEgorBo
authored
Remove uses of CLRJIT_AZ_KEY/clrjit_key1 from SPMI (#104164)
1 parent 13d2bab commit d198728

File tree

7 files changed

+55
-53
lines changed

7 files changed

+55
-53
lines changed

‎eng/pipelines/common/templates/runtimes/run-test-job.yml‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -615,15 +615,19 @@ jobs:
615615
condition: always()
616616

617617
# Ensure the Python azure-storage-blob package is installed before doing the upload.
618-
- script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall
619-
displayName: Upgrade Pip to latest and install azure-storage-blob Python package
618+
- script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall && $(PipScript) install azure.identity==1.16.1 --force-reinstall
619+
displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages
620620
condition: always()
621621

622-
- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)
622+
- task: AzureCLI@2
623623
displayName: 'Upload SuperPMI $(CollectionName)-$(CollectionType) collection to Azure Storage'
624+
inputs:
625+
azureSubscription: 'superpmi-collect-rw'
626+
scriptType: 'pscore'
627+
scriptLocation: 'inlineScript'
628+
inlineScript: |
629+
$(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)
624630
condition: always()
625-
env:
626-
CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline
627631

628632
- task: CopyFiles@2
629633
displayName: Copying superpmi.log of all partitions

‎eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -177,13 +177,17 @@ jobs:
177177
onlyAddExtraIndex: false
178178

179179
# Ensure the Python azure-storage-blob package is installed before doing the upload.
180-
- script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall
181-
displayName: Upgrade Pip to latest and install azure-storage-blob Python package
180+
- script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall && $(PipScript) install azure.identity==1.16.1 --force-reinstall
181+
displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages
182182

183-
- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)
183+
- task: AzureCLI@2
184184
displayName: ${{ format('Upload SuperPMI {0}-{1} collection to Azure Storage', parameters.collectionName, parameters.collectionType) }}
185-
env:
186-
CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline
185+
inputs:
186+
azureSubscription: 'superpmi-collect-rw'
187+
scriptType: 'pscore'
188+
scriptLocation: 'inlineScript'
189+
inlineScript: |
190+
$(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)
187191
188192
# Always upload the available logs for diagnostics
189193
- task: CopyFiles@2

‎eng/pipelines/coreclr/templates/upload-jits-steps.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,14 @@ steps:
1010
displayName: Enable python venv
1111

1212
# Ensure the Python azure-storage-blob package is installed before doing the upload.
13-
- script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall
14-
displayName: Upgrade Pip to latest and install azure-storage-blob Python package
13+
- script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall && $(PipScript) install azure.identity==1.16.1 --force-reinstall
14+
displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages
1515

16-
- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(_BuildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) --use_latest_jit_change
17-
displayName: Upload JIT to Azure Storage
18-
env:
19-
CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline
16+
- task: AzureCLI@2
17+
displayName: 'Upload JIT to Azure Storage'
18+
inputs:
19+
azureSubscription: 'superpmi-collect-rw'
20+
scriptType: 'pscore'
21+
scriptLocation: 'inlineScript'
22+
inlineScript: |
23+
$(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(_BuildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) --use_latest_jit_change

‎eng/pipelines/libraries/superpmi-postprocess-step.yml‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,19 @@ steps:
6666
condition: always()
6767

6868
# Ensure the Python azure-storage-blob package is installed before doing the upload.
69-
- script: ${{ parameters.PipScript }} install --upgrade pip && ${{ parameters.PipScript }} install azure.storage.blob==12.5.0 --force-reinstall
70-
displayName: Upgrade Pip to latest and install azure-storage-blob Python package
69+
- script: ${{ parameters.PipScript }} install --upgrade pip && ${{ parameters.PipScript }} install azure.storage.blob==12.5.0 --force-reinstall && ${{ parameters.PipScript }} install azure.identity==1.16.1 --force-reinstall
70+
displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages
7171
condition: always()
7272

73-
- script: ${{ parameters.PythonScript }} $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch ${{ parameters.archType }} -build_type ${{ parameters.buildConfig }} -mch_files ${{ parameters.MergedMchFileLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}.${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}.mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.x64.${{ parameters.buildConfigUpper }}
73+
- task: AzureCLI@2
7474
displayName: 'Upload SuperPMI ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} collection to Azure Storage'
75+
inputs:
76+
azureSubscription: 'superpmi-collect-rw'
77+
scriptType: 'pscore'
78+
scriptLocation: 'inlineScript'
79+
inlineScript: |
80+
${{ parameters.PythonScript }} $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch ${{ parameters.archType }} -build_type ${{ parameters.buildConfig }} -mch_files ${{ parameters.MergedMchFileLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}.${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}.mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.x64.${{ parameters.buildConfigUpper }}
7581
condition: always()
76-
env:
77-
CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline
7882

7983
- task: CopyFiles@2
8084
displayName: Copying superpmi.log of all partitions

‎src/coreclr/scripts/jitrollingbuild.py‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,6 @@
108108

109109
upload_parser.add_argument("-git_hash", required=True, help=git_hash_help)
110110
upload_parser.add_argument("--use_latest_jit_change", action="store_true", help=use_latest_jit_change_help)
111-
upload_parser.add_argument("-az_storage_key", help="Key for the clrjit Azure Storage location. Default: use the value of the CLRJIT_AZ_KEY environment variable.")
112111
upload_parser.add_argument("--skip_cleanup", action="store_true", help=skip_cleanup_help)
113112

114113
# subparser for download
@@ -458,14 +457,18 @@ def upload_blob(file, blob_name):
458457

459458
try:
460459
from azure.storage.blob import BlobServiceClient
460+
from azure.identity import DefaultAzureCredential
461461

462462
except:
463463
logging.warning("Please install:")
464464
logging.warning(" pip install azure-storage-blob")
465+
logging.warning(" pip install azure-identiy")
465466
logging.warning("See also https://learn.microsoft.com/azure/storage/blobs/storage-quickstart-blobs-python")
466-
raise RuntimeError("Missing azure storage package.")
467+
raise RuntimeError("Missing azure storage or identity packages.")
467468

468-
blob_service_client = BlobServiceClient(account_url=az_blob_storage_account_uri, credential=coreclr_args.az_storage_key)
469+
default_credential = DefaultAzureCredential()
470+
471+
blob_service_client = BlobServiceClient(account_url=az_blob_storage_account_uri, credential=default_credential)
469472
blob_folder_name = "{}/{}/{}/{}/{}".format(az_builds_root_folder, jit_git_hash, coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)
470473

471474
total_bytes_uploaded = 0
@@ -623,12 +626,6 @@ def setup_spmi_location_arg(spmi_location):
623626
lambda unused: True,
624627
"Unable to set use_latest_jit_change")
625628

626-
coreclr_args.verify(args,
627-
"az_storage_key",
628-
lambda item: item is not None,
629-
"Specify az_storage_key or set environment variable CLRJIT_AZ_KEY to the key to use.",
630-
modify_arg=lambda arg: os.environ["CLRJIT_AZ_KEY"] if arg is None and "CLRJIT_AZ_KEY" in os.environ else arg)
631-
632629
coreclr_args.verify(args,
633630
"skip_cleanup",
634631
lambda unused: True,

‎src/coreclr/scripts/jitutil.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -537,13 +537,13 @@ def require_azure_storage_libraries(need_azure_storage_blob=True, need_azure_ide
537537
Once we've done it once, we don't do it again.
538538
539539
For this to work for cross-module usage, after you call this function, you need to add a line like:
540-
from jitutil import BlobClient, AzureCliCredential
540+
from jitutil import BlobClient, DefaultAzureCredential
541541
naming all the types you want to use.
542542
543543
The full set of types this function loads:
544-
BlobServiceClient, BlobClient, ContainerClient, AzureCliCredential
544+
BlobServiceClient, BlobClient, ContainerClient, DefaultAzureCredential
545545
"""
546-
global azure_storage_libraries_check, BlobServiceClient, BlobClient, ContainerClient, AzureCliCredential
546+
global azure_storage_libraries_check, BlobServiceClient, BlobClient, ContainerClient, DefaultAzureCredential
547547

548548
if azure_storage_libraries_check:
549549
return
@@ -560,7 +560,7 @@ def require_azure_storage_libraries(need_azure_storage_blob=True, need_azure_ide
560560
azure_identity_import_ok = True
561561
if need_azure_identity:
562562
try:
563-
from azure.identity import AzureCliCredential
563+
from azure.identity import DefaultAzureCredential
564564
except:
565565
azure_identity_import_ok = False
566566

@@ -608,7 +608,7 @@ def download_with_azure(uri, target_location, fail_if_not_found=True):
608608
logging.info("Download: %s -> %s", uri, target_location)
609609

610610
ok = True
611-
az_credential = AzureCliCredential()
611+
az_credential = DefaultAzureCredential()
612612
blob = BlobClient.from_blob_url(uri, credential=az_credential)
613613
with open(target_location, "wb") as my_blob:
614614
try:

‎src/coreclr/scripts/superpmi.py‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,6 @@ def add_core_root_arguments(parser, build_type_default, build_type_help):
364364
upload_parser = subparsers.add_parser("upload", description=upload_description, parents=[core_root_parser, target_parser])
365365

366366
upload_parser.add_argument("-mch_files", metavar="MCH_FILE", required=True, nargs='+', help=upload_mch_files_help)
367-
upload_parser.add_argument("-az_storage_key", help="Key for the clrjit Azure Storage location. Default: use the value of the CLRJIT_AZ_KEY environment variable.")
368367
upload_parser.add_argument("-jit_ee_version", help=jit_ee_version_help)
369368
upload_parser.add_argument("--skip_cleanup", action="store_true", help=skip_cleanup_help)
370369

@@ -3563,14 +3562,14 @@ def list_superpmi_collections_container_via_azure_api(path_filter=lambda unused:
35633562
"""
35643563

35653564
require_azure_storage_libraries()
3566-
from jitutil import ContainerClient, AzureCliCredential
3565+
from jitutil import ContainerClient, DefaultAzureCredential
35673566

35683567
superpmi_container_url = az_blob_storage_superpmi_container_uri
35693568

35703569
paths = []
35713570
ok = True
35723571
try:
3573-
az_credential = AzureCliCredential()
3572+
az_credential = DefaultAzureCredential()
35743573
container = ContainerClient.from_container_url(superpmi_container_url, credential=az_credential)
35753574
blob_name_prefix = az_collections_root_folder + "/"
35763575
blob_list = container.list_blobs(name_starts_with=blob_name_prefix, retry_total=0)
@@ -3789,8 +3788,8 @@ def upload_mch(coreclr_args):
37893788
coreclr_args (CoreclrArguments): parsed args
37903789
"""
37913790

3792-
require_azure_storage_libraries(need_azure_identity=False)
3793-
from jitutil import BlobServiceClient
3791+
require_azure_storage_libraries(need_azure_identity=True)
3792+
from jitutil import BlobServiceClient, DefaultAzureCredential
37943793

37953794
def upload_blob(file, blob_name):
37963795
blob_client = blob_service_client.get_blob_client(container=az_superpmi_container_name, blob=blob_name)
@@ -3826,7 +3825,9 @@ def upload_blob(file, blob_name):
38263825
for item in files_to_upload:
38273826
logging.info(" %s", item)
38283827

3829-
blob_service_client = BlobServiceClient(account_url=az_blob_storage_account_uri, credential=coreclr_args.az_storage_key)
3828+
default_credential = DefaultAzureCredential()
3829+
3830+
blob_service_client = BlobServiceClient(account_url=az_blob_storage_account_uri, credential=default_credential)
38303831
blob_folder_name = "{}/{}/{}/{}".format(az_collections_root_folder, coreclr_args.jit_ee_version, coreclr_args.target_os, coreclr_args.mch_arch)
38313832

38323833
total_bytes_uploaded = 0
@@ -5005,12 +5006,6 @@ def verify_base_diff_args():
50055006
verify_target_args()
50065007
verify_jit_ee_version_arg()
50075008

5008-
coreclr_args.verify(args,
5009-
"az_storage_key",
5010-
lambda item: item is not None,
5011-
"Specify az_storage_key or set environment variable CLRJIT_AZ_KEY to the key to use.",
5012-
modify_arg=lambda arg: os.environ["CLRJIT_AZ_KEY"] if arg is None and "CLRJIT_AZ_KEY" in os.environ else arg)
5013-
50145009
coreclr_args.verify(args,
50155010
"mch_files",
50165011
lambda unused: True,
@@ -5035,12 +5030,6 @@ def verify_base_diff_args():
50355030
print("Error: private store directory '" + coreclr_args.private_store + "' not found.")
50365031
sys.exit(1)
50375032

5038-
# Safety measure: don't allow CLRJIT_AZ_KEY to be set if we are uploading to a private store.
5039-
# Note that this should be safe anyway, since we're publishing something private, not public.
5040-
if "CLRJIT_AZ_KEY" in os.environ:
5041-
print("Error: environment variable CLRJIT_AZ_KEY is set, but command is `upload-private`, not `upload`. That is not allowed.")
5042-
sys.exit(1)
5043-
50445033
elif coreclr_args.mode == "download":
50455034

50465035
verify_target_args()

0 commit comments

Comments
 (0)