diff --git a/Documentation/OneLocBuild.md b/Documentation/OneLocBuild.md index 978756d402a..45a562f9e80 100644 --- a/Documentation/OneLocBuild.md +++ b/Documentation/OneLocBuild.md @@ -89,7 +89,7 @@ If you're releasing from any other branch (including servicing branches), you mu ## Filing Issues for Translation Issues -File a translation issue ticket with the localization team (see documentation [here](https://dev.azure.com/ceapex/CEINTL/_wiki/wikis/CEINTL.wiki/1361/Provide-Enough-Information-in-DevRel-Feedback-Ticket)). +File a translation issue ticket with the localization team [here](https://aka.ms/ceLocBug). ## Leaving Comments for Translators diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7f90a691fcd..ee8fe8365fd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -9,27 +9,31 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-symuploader 62ceb439e80bf0814d0ffa17f022d4624ea4aa6c + + https://github.com/dotnet/templating + + - + https://github.com/dotnet/arcade - 427c05909067bb2e484116ae2239456bb45adb85 + 200adbc809c4451973d1929a53a75502b7cada01 - + https://github.com/dotnet/arcade - 427c05909067bb2e484116ae2239456bb45adb85 + 200adbc809c4451973d1929a53a75502b7cada01 - + https://github.com/dotnet/arcade - 427c05909067bb2e484116ae2239456bb45adb85 + 200adbc809c4451973d1929a53a75502b7cada01 - + https://github.com/dotnet/arcade - 427c05909067bb2e484116ae2239456bb45adb85 + 200adbc809c4451973d1929a53a75502b7cada01 - + https://github.com/dotnet/arcade - 427c05909067bb2e484116ae2239456bb45adb85 + 200adbc809c4451973d1929a53a75502b7cada01 https://github.com/dotnet/arcade-services @@ -39,9 +43,9 @@ https://github.com/dotnet/arcade-services a5f3ed9d5f560555ff6d26b286acdcfbb7ce3b14 - + https://github.com/dotnet/xharness - ed75873fb3b4c0e8746bce1fb325bfb5061b9851 + d27acf37630aa2a03824b698aa51356e47f902c0 https://github.com/dotnet/roslyn diff --git a/eng/Versions.props b/eng/Versions.props index 2ebc749df94..0776401b9ea 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -67,8 +67,8 @@ 8.5.0 2.4.2-pre.9 2.0.3 - 7.0.0-beta.21576.4 - 7.0.0-beta.21576.4 + 7.0.0-beta.21608.1 + 7.0.0-beta.21608.1 1.22.0 1.1.2 2.0.0 @@ -79,15 +79,16 @@ 1.1.0-beta.21553.1 1.1.1-beta-21579-02 1.1.1-beta-21579-02 - 7.0.0-beta.21576.4 + 7.0.0-beta.21608.1 1.0.0-beta.21518.1 1.1.0-beta.21520.4 - 1.0.0-prerelease.21579.1 + 1.0.0-prerelease.21602.2 2.0.0-preview.1.21526.15 2.0.0-preview.1.21526.15 6.0.100-rtm.21515.10 1.0.0-preview1.1.21116.1 $(MicrosoftSourceLinkGitHubVersion) + 7.0.100-alpha.1.21601.1 diff --git a/eng/common/cross/armv6/sources.list.buster b/eng/common/cross/armv6/sources.list.buster new file mode 100644 index 00000000000..f27fc4fb346 --- /dev/null +++ b/eng/common/cross/armv6/sources.list.buster @@ -0,0 +1,2 @@ +deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi +deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6fa2c8aa551..5102245b7b5 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -99,6 +99,15 @@ while :; do __AlpineArch=armv7 __QEMUArch=arm ;; + armv6) + __BuildArch=armv6 + __UbuntuArch=armhf + __QEMUArch=arm + __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" + __CodeName=buster + __LLDB_Package="liblldb-6.0-dev" + __Keyring="/usr/share/keyrings/raspbian-archive-keyring.gpg" + ;; arm64) __BuildArch=arm64 __UbuntuArch=arm64 @@ -236,6 +245,12 @@ while :; do shift done +if [ -e "$__Keyring" ]; then + __Keyring="--keyring=$__Keyring" +else + __Keyring="" +fi + if [ "$__BuildArch" == "armel" ]; then __LLDB_Package="lldb-3.5-dev" fi @@ -337,7 +352,7 @@ elif [[ "$__CodeName" == "illumos" ]]; then wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h elif [[ -n $__CodeName ]]; then - qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo + qemu-debootstrap $__Keyring --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list chroot $__RootfsDir apt-get update chroot $__RootfsDir apt-get -f -y install diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 8128f2c3570..8cf772b3cbf 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -51,14 +51,9 @@ jobs: value: ${{ parameters.AzDOPipelineId }} - name: AzDOBuildId value: ${{ parameters.AzDOBuildId }} - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 + - template: /eng/common/templates/variables/sdl-variables.yml - name: GuardianVersion value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config pool: vmImage: windows-2019 steps: @@ -125,57 +120,11 @@ jobs: displayName: Extract Archive Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - - ${{ if ne(parameters.overrideGuardianVersion, '') }}: - - powershell: | - $content = Get-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content was:`n$content" - - $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') - $content | Set-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content updated to:`n$content" - displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} - - - task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - - task: NuGetCommand@2 - displayName: 'Install Guardian' - inputs: - restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - feedsToUse: config - nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config - externalFeedCredentials: GuardianConnect - restoreDirectory: $(Build.SourcesDirectory)\.packages - - - ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - - ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) - -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) - ${{ parameters.additionalParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - - - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - # We want to publish the Guardian results and configuration for easy diagnosis. However, the - # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default - # tooling files. Some of these files are large and aren't useful during an investigation, so - # exclude them by simply deleting them before publishing. (As of writing, there is no documented - # way to selectively exclude a dir from the pipeline artifact publish task.) - - task: DeleteFiles@1 - displayName: Delete Guardian dependencies to avoid uploading - inputs: - SourceFolder: $(Agent.BuildDirectory)/.gdn - Contents: | - c - i - condition: succeededOrFailed() - - publish: $(Agent.BuildDirectory)/.gdn - artifact: GuardianConfiguration - displayName: Publish GuardianConfiguration - condition: succeededOrFailed() + - template: /eng/common/templates/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: ${{ parameters.additionalParameters }} + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml new file mode 100644 index 00000000000..7b8ee18a28d --- /dev/null +++ b/eng/common/templates/steps/execute-sdl.yml @@ -0,0 +1,68 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - powershell: | + $content = Get-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content was:`n$content" + + $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') + $content | Set-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content updated to:`n$content" + displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- task: NuGetCommand@2 + displayName: 'Install Guardian' + inputs: + restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + feedsToUse: config + nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config + externalFeedCredentials: GuardianConnect + restoreDirectory: $(Build.SourcesDirectory)\.packages + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates/variables/sdl-variables.yml b/eng/common/templates/variables/sdl-variables.yml new file mode 100644 index 00000000000..dbdd66d4a4b --- /dev/null +++ b/eng/common/templates/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/global.json b/global.json index 1110be948b3..c94ef4b8165 100644 --- a/global.json +++ b/global.json @@ -3,7 +3,7 @@ "dotnet": "6.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21576.4", - "Microsoft.DotNet.Helix.Sdk": "7.0.0-beta.21576.4" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21608.1", + "Microsoft.DotNet.Helix.Sdk": "7.0.0-beta.21608.1" } } diff --git a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj index 63544dd9c13..32d30e285c7 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj +++ b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj @@ -84,6 +84,7 @@ $(MicrosoftDotNetXliffTasksVersion) $(MicrosoftDotNetMaestroTasksVersion) $(MicrosoftSymbolUploaderBuildTaskVersion) + $(MicrosoftTemplateEngineTasksVersion) ]]> diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Imports.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/Imports.targets index f0b6e53a912..7ab283ab033 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Imports.targets +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Imports.targets @@ -18,7 +18,7 @@ - + diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Localization.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/Localization.targets index 4eef799770d..a4c1f08916e 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Localization.targets +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Localization.targets @@ -3,11 +3,11 @@ - - + + + + true + + + + + diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj index aad388deee3..0111d6971bf 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj @@ -116,13 +116,21 @@ + + - + + + + + + + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs index 9fb636d3599..6cee3219e7e 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/model/SetupTargetFeedConfigV3.cs @@ -83,7 +83,7 @@ private static string ConvertFromBase64(string value) public override List Setup() { - return Feeds().ToList(); + return Feeds().Distinct().ToList(); } private IEnumerable Feeds() @@ -160,10 +160,8 @@ private IEnumerable Feeds() var sasUri = GetFeedSasUri(feed); if (feed != oldFeed && string.IsNullOrEmpty(key) && string.IsNullOrEmpty(sasUri)) { - Log?.LogMessage($"No keys found for {feed}, falling back to keys for {oldFeed}."); - // if we used an override, and didn't find a key, fallback to the keys for the non-override value - key = GetFeedKey(oldFeed); - sasUri = GetFeedSasUri(oldFeed); + Log?.LogWarning($"No keys found for {feed}, unable to publish to it."); + continue; } var feedType = feed.StartsWith("https://pkgs.dev.azure.com") ? FeedType.AzDoNugetFeed @@ -172,7 +170,7 @@ private IEnumerable Feeds() type, sasUri ?? feed, feedType, - key, + sasUri == null ? key : null, LatestLinkShortUrlPrefixes, spec.Assets, false, diff --git a/src/Microsoft.DotNet.Build.Tasks.Installers/build/installer.singlerid.targets b/src/Microsoft.DotNet.Build.Tasks.Installers/build/installer.singlerid.targets index 2651a4588fd..6c53ad557f9 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Installers/build/installer.singlerid.targets +++ b/src/Microsoft.DotNet.Build.Tasks.Installers/build/installer.singlerid.targets @@ -458,6 +458,7 @@ $([System.IO.File]::ReadAllText('$(LicenseFile)').Replace('%0A', '\n').Replace('"', '\"')) + /usr/share/dotnet @@ -494,7 +495,7 @@ <_CommonLinuxPackageProperty Include="short_description" String="$(_ShortDescription)" /> <_CommonLinuxPackageProperty Include="maintainer_name" String=".NET Team" /> <_CommonLinuxPackageProperty Include="maintainer_email" String="dotnetpackages@dotnetfoundation.org" /> - <_CommonLinuxPackageProperty Include="install_root" String="/usr/share/dotnet" /> + <_CommonLinuxPackageProperty Include="install_root" String="$(LinuxInstallRoot)" /> <_CommonLinuxPackageProperty Include="long_description" String=".NET is a development platform that you can use to build command-line applications, microservices and modern websites. It is open source, cross-platform and is supported by Microsoft. We hope you enjoy using it! If you do, please consider joining the active community of developers that are contributing to the project on GitHub (https://github.com/dotnet/core). We happily accept issues and PRs." /> <_CommonLinuxPackageProperty Include="homepage" String="https://github.com/dotnet/core" /> <_CommonLinuxPackageProperty Include="copyright" String="2017 Microsoft" /> diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-event-processor.py b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-event-processor.py index e4488839b1f..766ccabb798 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-event-processor.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-event-processor.py @@ -55,30 +55,30 @@ def remove_android_apps(device: str = None): print(f' Removing installed apps after unsuccessful run' + (' from ' + device if device else "")) xharness_cli_path = os.getenv('XHARNESS_CLI_PATH') - adb_path = subprocess.check_output(['dotnet', 'exec', xharness_cli_path, 'android', 'state', '--adb']).decode('utf-8').strip() + adb_args = ['dotnet', 'exec', xharness_cli_path, 'android', 'adb', '--'] # Get list of installed apps + args = ['shell', 'pm', 'list', 'packages', 'net.dot'] if device: - installed_apps = subprocess.check_output([adb_path, '-s', device, 'shell', 'pm', 'list', 'packages', 'net.dot']).decode('utf-8').splitlines() - else: - installed_apps = subprocess.check_output([adb_path, 'shell', 'pm', 'list', 'packages', 'net.dot']).decode('utf-8').splitlines() + args = ['-s', device] + args + installed_apps = subprocess.check_output(adb_args + args).decode('utf-8').splitlines() installed_apps = [app.split(':')[1] for app in installed_apps if app] # Remove all installed apps for app in installed_apps: print(f' Removing {app}') - - try: - if device: - result = subprocess.run([adb_path, '-s', device, 'uninstall', app], stdout=subprocess.PIPE) - else: - result = subprocess.run([adb_path, 'uninstall', app], stderr=subprocess.STDOUT) + args = ['uninstall', app] + if device: + args = ['-s', device] + args + + try: + result = subprocess.run(adb_args + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = result.stdout.decode('utf8') print(f' {output}') except Exception as e: - print(f' Failed to remove app: {e}') + print(f' Failed to remove app {app}: {e}') def analyze_operation(command: str, platform: str, device: str, is_device: bool, target: str, exit_code: int): """ Analyzes the result and requests retry/reboot in case of an infra failure @@ -165,6 +165,13 @@ def analyze_operation(command: str, platform: str, device: str, is_device: bool, reboot = True retry = True + # Simulators are known to slow/break down and a reboot usually helps + # This manifest by us not being able to launch the simulator/start the test run in time + if exit_code == 90: # APP_LAUNCH_TIMEOUT + print(f' Failed to start the test execution in time. {retry_message}{reboot_message}') + reboot = True + retry = True + # The JSON should be an array of objects (one per each executed XHarness command) operations = json.load(open(diagnostics_file)) @@ -223,7 +230,7 @@ def analyze_operation(command: str, platform: str, device: str, is_device: bool, app_insights.send_metric(RETRY_METRIC_NAME, retry_exit_code, properties=retry_dimensions) request_infra_retry('Requesting work item retry because an infrastructure issue was detected on this machine') - # TODO https://github.com/dotnet/core-eng/issues/15059 + # TODO https://github.com/dotnet/core-eng/issues/15059 # We need to remove testResults.xml so that it is not uploaded since this run will be discarded # This is a workaround until we make AzDO reporter not upload test results test_results = os.path.join(output_directory, "testResults.xml")