From 4d26a12fe870b9276c0efd6d7e403546f6145e20 Mon Sep 17 00:00:00 2001 From: Crystal YU Date: Fri, 4 Nov 2022 13:28:49 +0800 Subject: [PATCH] automation script to generate sdk from cadl (#32206) * cadl automation --- eng/scripts/Invoke-GenerateAndBuildV2.ps1 | 35 +++++++ .../automation/GenerateAndBuildLib.ps1 | 99 ++++++++++++++++++- .../automation/Invoke-GenerateAndBuild.ps1 | 56 ++++++++--- 3 files changed, 178 insertions(+), 12 deletions(-) diff --git a/eng/scripts/Invoke-GenerateAndBuildV2.ps1 b/eng/scripts/Invoke-GenerateAndBuildV2.ps1 index 8d4fde0a1ce0..29126ec35390 100644 --- a/eng/scripts/Invoke-GenerateAndBuildV2.ps1 +++ b/eng/scripts/Invoke-GenerateAndBuildV2.ps1 @@ -41,6 +41,7 @@ $commitid = $inputJson.headSha $repoHttpsUrl = $inputJson.repoHttpsUrl $downloadUrlPrefix = $inputJson.installInstructionInput.downloadUrlPrefix $autorestConfig = $inputJson.autorestConfig +$relatedCadlProjectFolder = $inputJson.relatedCadlProjectFolder $autorestConfigYaml = "" if ($autorestConfig) { @@ -108,6 +109,40 @@ if ($inputFileToGen) { UpdateExistingSDKByInputFiles -inputFilePaths $inputFileToGen -sdkRootPath $sdkPath -headSha $commitid -repoHttpsUrl $repoHttpsUrl -downloadUrlPrefix "$downloadUrlPrefix" -generatedSDKPackages $generatedSDKPackages } +# generate sdk from cadl file +if ($relatedCadlProjectFolder) { + foreach ($cadlRelativeFolder in $relatedCadlProjectFolder) { + $cadlFolder = Resolve-Path (Join-Path $swaggerDir $cadlRelativeFolder) + $newPackageOutput = "newPackageOutput.json" + + Push-Location $cadlFolder + trap {Pop-Location} + $cadlProjectYaml = Get-Content -Path (Join-Path "$cadlFolder" "cadl-project.yaml") -Raw + + Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module + $yml = ConvertFrom-YAML $cadlProjectYaml + $sdkFolder = $yml["emitters"]["@azure-tools/cadl-csharp"]["sdk-folder"] + $projectFolder = (Join-Path $sdkPath $sdkFolder) + # $projectFolder = $projectFolder -replace "\\", "/" + if ($projectFolder) { + $directories = $projectFolder -split "/|\\" + $count = $directories.Count + $projectFolder = $directories[0 .. ($count-2)] -join "/" + $service = $directories[-3]; + $namespace = $directories[-2]; + } + New-CADLPackageFolder -service $service -namespace $namespace -sdkPath $sdkPath -cadlInput $cadlFolder/main.cadl -outputJsonFile $newpackageoutput + $newPackageOutputJson = Get-Content $newPackageOutput -Raw | ConvertFrom-Json + $relativeSdkPath = $newPackageOutputJson.path + npm install + npx cadl compile --output-path $sdkPath --emit @azure-tools/cadl-csharp . + if ( !$?) { + Throw "Failed to generate sdk for cadl. exit code: $?" + } + GeneratePackage -projectFolder $projectFolder -sdkRootPath $sdkPath -path $relativeSdkPath -downloadUrlPrefix $downloadUrlPrefix -skipGenerate -generatedSDKPackages $generatedSDKPackages + Pop-Location + } +} $outputJson = [PSCustomObject]@{ packages = $generatedSDKPackages } diff --git a/eng/scripts/automation/GenerateAndBuildLib.ps1 b/eng/scripts/automation/GenerateAndBuildLib.ps1 index fec12401620d..979948d5f2c0 100644 --- a/eng/scripts/automation/GenerateAndBuildLib.ps1 +++ b/eng/scripts/automation/GenerateAndBuildLib.ps1 @@ -359,6 +359,100 @@ function New-MgmtPackageFolder() { return $projectFolder } +function New-CADLPackageFolder() { + param( + [string]$service, + [string]$namespace, + [string]$sdkPath = "", + [string]$cadlInput ="", + [string]$outputJsonFile = "$PWD/output.json" + ) + $serviceFolder = (Join-Path $sdkPath "sdk" $service) + $projectFolder=(Join-Path $sdkPath "sdk" $service $namespace) + $ciymlFilePath =(Join-Path $sdkPath "sdk" $service $CI_YAML_FILE) + $apifolder = (Join-Path $projectFolder "api") + Write-Host "projectFolder:$projectFolder, apifolder:$apifolder" + if ((Test-Path -Path $projectFolder) -And (Test-Path -Path $apifolder)) { + Write-Host "Path exists!" + if (Test-Path -Path $projectFolder/src/autorest.md) { + Remove-Item -Path $projectFolder/src/autorest.md + } + $projFile = (Join-Path $projectFolder "src" "$namespace.csproj") + $fileContent = Get-Content -Path $projFile + $match = ($fileContent | Select-String -Pattern "").LineNumber + if ($match.count -gt 0) { + $fileContent[$match[0] - 1] = "$cadlInput"; + } else { + $startNum = ($fileContent | Select-String -Pattern '').LineNumber[0] + $fileContent[$startNum - 2] += ([Environment]::NewLine + "$cadlInput") + } + $fileContent | Out-File $projFile + + Update-CIYmlFile -ciFilePath $ciymlFilePath -artifact $namespace + } else { + Write-Host "Path doesn't exist. create template." + dotnet new -i $sdkPath/sdk/template + Write-Host "Create project folder $projectFolder" + if (Test-Path -Path $projectFolder) { + Remove-Item -Path $projectFolder -ItemType Directory + } + + Push-Location $serviceFolder + $namespaceArray = $namespace.Split(".") + if ( $namespaceArray.Count -lt 3) { + Throw "Error: invalid namespace name." + } + + $endIndex = $namespaceArray.Count - 2 + $clientName = $namespaceArray[-1] + $groupName = $namespaceArray[1..$endIndex] -join "." + $dotnetNewCmd = "dotnet new azsdkdpg --name $namespace --clientName $clientName --groupName $groupName --serviceDirectory $service --force" + + if (Test-Path -Path $ciymlFilePath) { + Write-Host "ci.yml already exists. update it to include the new serviceDirectory." + Update-CIYmlFile -ciFilePath $ciymlFilePath -artifact $namespace + + $dotnetNewCmd = $dotnetNewCmd + " --includeCI false" + } + # dotnet new azsdkdpg --name $namespace --clientName $clientName --groupName $groupName --serviceDirectory $service --swagger $inputfile --securityScopes $securityScope --securityHeaderName $securityHeaderName --includeCI true --force + Write-Host "Invoke dotnet new command: $dotnetNewCmd" + Invoke-Expression $dotnetNewCmd + + $projFile = (Join-Path $projectFolder "src" "$namespace.csproj") + $fileContent = Get-Content -Path $projFile + $fileContent = $fileContent -replace '[^<]+', '1.0.0-beta.1' + $startNum = ($fileContent | Select-String -Pattern '').LineNumber[0] + $fileContent[$startNum - 2] += ([Environment]::NewLine + "$cadlInput") + $fileContent | Out-File $projFile + # (Get-Content $projFile) -replace "*.*.*-*.*", "1.0.0-beta.1" | -replace "*", "$cadlInput" |Set-Content $projFile + Pop-Location + # dotnet sln + Push-Location $projectFolder + if (Test-Path -Path $projectFolder/src/autorest.md) { + Remove-Item -Path $projectFolder/src/autorest.md + } + dotnet sln remove src/$namespace.csproj + dotnet sln add src/$namespace.csproj + dotnet sln remove tests/$namespace.Tests.csproj + dotnet sln add tests/$namespace.Tests.csproj + Pop-Location + } + + Push-Location $sdkPath + $relativeFolderPath = Resolve-Path $projectFolder -Relative + Pop-Location + + $outputJson = [PSCustomObject]@{ + service = $service + packageName = $namespace + projectFolder = $projectFolder + path = @($relativeFolderPath) + } + + $outputJson | ConvertTo-Json -depth 100 | Out-File $outputJsonFile + return $projectFolder +} + function Get-ResourceProviderFromReadme($readmeFile) { $readmeFile = $readmeFile -replace "\\", "/" $pathArray = $readmeFile.Split("/"); @@ -525,6 +619,7 @@ function GeneratePackage() [string]$sdkRootPath, [string]$path, [string]$downloadUrlPrefix="", + [switch]$skipGenerate, [object]$generatedSDKPackages ) @@ -542,7 +637,9 @@ function GeneratePackage() # Generate Code Write-Host "Start to generate sdk $projectFolder" $srcPath = Join-Path $projectFolder 'src' - dotnet build /t:GenerateCode $srcPath + if (!$skipGenerate) { + dotnet build /t:GenerateCode $srcPath + } if ( !$?) { Write-Error "Failed to generate sdk. exit code: $?" $result = "failed" diff --git a/eng/scripts/automation/Invoke-GenerateAndBuild.ps1 b/eng/scripts/automation/Invoke-GenerateAndBuild.ps1 index 7a25d8fdef06..5b578a3a5596 100644 --- a/eng/scripts/automation/Invoke-GenerateAndBuild.ps1 +++ b/eng/scripts/automation/Invoke-GenerateAndBuild.ps1 @@ -22,9 +22,10 @@ $downloadUrlPrefix = $inputJson.installInstructionInput.downloadUrlPrefix $autorestConfig = $inputJson.autorestConfig $autorestConfig = $inputJson.autorestConfig +$relatedCadlProjectFolder = $inputJson.relatedCadlProjectFolder $autorestConfigYaml = "" -if ($autorestConfig -ne "") { +if ($autorestConfig) { $autorestConfig | Set-Content "config.md" $autorestConfigYaml = Get-Content -Path .\config.md $range = ($autorestConfigYaml | Select-String -Pattern '```').LineNumber @@ -41,27 +42,60 @@ if ($autorestConfig -ne "") { } } -Write-Host "swaggerDir:$swaggerDir, readmeFile:$readmeFile" +$generatedSDKPackages = New-Object 'Collections.Generic.List[System.Object]' # $service, $serviceType = Get-ResourceProviderFromReadme $readmeFile $sdkPath = (Join-Path $PSScriptRoot .. .. ..) $sdkPath = Resolve-Path $sdkPath $sdkPath = $sdkPath -replace "\\", "/" -$readme = "" -if ($commitid -ne "") { - if ($repoHttpsUrl -ne "") { - $readme = "$repoHttpsUrl/blob/$commitid/$readmeFile" +if ($readmeFile) { + Write-Host "swaggerDir:$swaggerDir, readmeFile:$readmeFile" + + $readme = "" + if ($commitid -ne "") { + if ($repoHttpsUrl -ne "") { + $readme = "$repoHttpsUrl/blob/$commitid/$readmeFile" + } else { + $readme = "https://github.com/$org/azure-rest-api-specs/blob/$commitid/$readmeFile" + } } else { - $readme = "https://github.com/$org/azure-rest-api-specs/blob/$commitid/$readmeFile" + $readme = (Join-Path $swaggerDir $readmeFile) } -} else { - $readme = (Join-Path $swaggerDir $readmeFile) + Invoke-GenerateAndBuildSDK -readmeAbsolutePath $readme -sdkRootPath $sdkPath -autorestConfigYaml "$autorestConfigYaml" -downloadUrlPrefix "$downloadUrlPrefix" -generatedSDKPackages $generatedSDKPackages } -$generatedSDKPackages = New-Object 'Collections.Generic.List[System.Object]' -Invoke-GenerateAndBuildSDK -readmeAbsolutePath $readme -sdkRootPath $sdkPath -autorestConfigYaml "$autorestConfigYaml" -downloadUrlPrefix "$downloadUrlPrefix" -generatedSDKPackages $generatedSDKPackages +if ($relatedCadlProjectFolder) { + $cadlFolder = Resolve-Path (Join-Path $swaggerDir $relatedCadlProjectFolder) + $newPackageOutput = "newPackageOutput.json" + Push-Location $cadlFolder + trap {Pop-Location} + $cadlProjectYaml = Get-Content -Path (Join-Path "$cadlFolder" "cadl-project.yaml") -Raw + + Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module + $yml = ConvertFrom-YAML $cadlProjectYaml + $sdkFolder = $yml["emitters"]["@azure-tools/cadl-csharp"]["sdk-folder"] + $projectFolder = (Join-Path $sdkPath $sdkFolder) + # $projectFolder = $projectFolder -replace "\\", "/" + if ($projectFolder) { + $directories = $projectFolder -split "/|\\" + $count = $directories.Count + $projectFolder = $directories[0 .. ($count-2)] -join "/" + $service = $directories[-3]; + $namespace = $directories[-2]; + } + New-CADLPackageFolder -service $service -namespace $namespace -sdkPath $sdkPath -cadlInput $cadlFolder/main.cadl -outputJsonFile $newpackageoutput + $newPackageOutputJson = Get-Content $newPackageOutput -Raw | ConvertFrom-Json + $relativeSdkPath = $newPackageOutputJson.path + npm install + npx cadl compile --output-path $sdkPath --emit @azure-tools/cadl-csharp . + if ( !$?) { + Throw "Failed to generate sdk for cadl. exit code: $?" + } + GeneratePackage -projectFolder $projectFolder -sdkRootPath $sdkPath -path $relativeSdkPath -downloadUrlPrefix $downloadUrlPrefix -skipGenerate -generatedSDKPackages $generatedSDKPackages + Pop-Location +} $outputJson = [PSCustomObject]@{ packages = $generatedSDKPackages }