Skip to content

Commit

Permalink
Database: Create BYOS Agent (#430)
Browse files Browse the repository at this point in the history
* Add wait for agent
* Use vcanja1 pool for now
* Delete BYOS agent after tests run
* add job to setup the agent and get public IP
* Use public ip from other stages
* Setup machine with Azure CLI, Docker
* Get public Ip from stage
* Use BYOS Agent for tests
* On agent start SQL Server and Oracle XE
* Open ports
* Add test for Oracle DB
  • Loading branch information
phaser authored Jun 6, 2024
1 parent 5ce5f33 commit f89c774
Show file tree
Hide file tree
Showing 10 changed files with 552 additions and 173 deletions.
2 changes: 1 addition & 1 deletion Activities/.pipelines/docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@ services:
environment:
OFFLINE: 'true'
LICENSE_AGREEMENT: 'accept'
SQL_SERVER_HOST: test_mssql_linux
SQL_SERVER_HOST: ${SQL_SERVER_HOST}
entrypoint: "/bin/sh /input/startup.sh"
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ parameters:
robotVersion: ''
beforeTestsCustomSteps: []
afterTestsCustomSteps: []
environment: {}

jobs:
- job: "RunDatabaseRuntimeTestsPortable"
Expand Down Expand Up @@ -50,12 +51,14 @@ jobs:
displayName: 'Prepare tests'
- bash: |
env
cd $(Build.SourcesDirectory)/Activities/.pipelines/docker
export MOUNT_OUTPUT=$(System.ArtifactsDirectory)
chmod -R 777 $(System.ArtifactsDirectory)
docker-compose up -d robot
docker-compose exec -T robot dotnet /input/TestRunner/uipath.studio.codedwf.tests.runner.dll run --package $(pkgPath) --robotPath /application/UiRobot.dll --output /output
displayName: 'Run tests'
env: ${{ parameters.environment }}
- ${{ each step in parameters.afterTestsCustomSteps }}:
- ${{ step }}
Expand Down

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"DisplayName": "TestOracleServer",
"Tooltip": null,
"HelpLink": null,
"Color": null,
"OriginalIconFileName": null,
"Arguments": []
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@
"uniqueId": "f2814f99-ced0-4321-8acd-137307f286eb",
"input": [],
"output": []
},
{
"filePath": "Tests\\TestOracleServer.xaml",
"uniqueId": "0b3f761c-83b8-4b68-9325-b5f27c4fd874",
"input": [],
"output": []
}
],
"isTemplate": false,
Expand Down
66 changes: 41 additions & 25 deletions Activities/Database/azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ resources:
name: UiPath/AzurePipelinesTemplates
ref: refs/tags/uipath.waitforbuild.1.0.3
endpoint: "GitHub connection"
- repository: template-helpers
type: github
name: UiPath/AzurePipelinesTemplates
endpoint: "UiPath"
ref: refs/tags/uipath.templates.helpers.1.1.4


trigger:
batch: true
Expand Down Expand Up @@ -53,10 +59,22 @@ stages:
enableCDStages: false
hasQaPackages: false

- stage: CreateAndSetupBYOSAgent
dependsOn:
- Build
displayName: "Create and Setup BYOS Agent"
variables:
eteKvName: "ete-variables-kv"
athenaservicesAgentPrefix: "cact"
eteAgentPoolName: vcanja1
eteProjName: "Ephemeral Testing Environment"
jobs:
- template: "stage.create.byos.agent.yml"

- stage: BuildDatabaseRuntimeTests
displayName: 'Build Database Runtime Tests'
dependsOn:
- Build
- CreateAndSetupBYOSAgent
jobs:
- template: ../.pipelines/jobs/stage.build.runtime.tests.yml
parameters:
Expand All @@ -68,43 +86,41 @@ stages:

- stage: RunDatabaseRuntimeTestsWindows
displayName: 'Run Database Runtime Tests Windows'
variables:
DatabaseServicesAgentPublicIp: $[stageDependencies.CreateAndSetupBYOSAgent.ConfigureDatabaseServicesAgents.outputs['AddNsgRule.DatabaseServicesAgentPublicIp']]
dependsOn:
- BuildDatabaseRuntimeTests
- CreateAndSetupBYOSAgent
jobs:
- template: ../.pipelines/jobs/stage.run.runtime.tests.windows.yml
parameters:
robotVersion: $(TestsStudioVersion)
beforeTestsCustomSteps:
- powershell: |
$env:SQL_SERVER_HOST="localhost"
displayName: "Set env"
- task: DockerCompose@0
inputs:
dockerComposeFile: '$(Build.SourcesDirectory)/Activities/Database/UiPath.Database.Runtime.Tests/docker/docker-compose.yml'
action: 'Run a specific service'
serviceName: 'test_mssql_windows'
ports:
1433:1433
displayName: 'Run SQL Server Image'
env:
MOUNT_OUTPUT: $(System.ArtifactsDirectory)
environment:
SQL_SERVER_HOST: "$(DatabaseServicesAgentPublicIp)"

- stage: RunDatabaseRuntimeTestsPortable
displayName: 'Run Database Runtime Tests Portable'
variables:
DatabaseServicesAgentPublicIp: $[stageDependencies.CreateAndSetupBYOSAgent.ConfigureDatabaseServicesAgents.outputs['AddNsgRule.DatabaseServicesAgentPublicIp']]
dependsOn:
- BuildDatabaseRuntimeTests
- CreateAndSetupBYOSAgent
jobs:
- template: ../.pipelines/jobs/stage.run.runtime.tests.portable.yml
parameters:
robotVersion: $(TestsStudioVersion)
beforeTestsCustomSteps:
- bash: |
cd $(Build.SourcesDirectory)/Activities/Database/UiPath.Database.Runtime.Tests/docker
docker-compose --version
export MOUNT_OUTPUT=$(System.ArtifactsDirectory)
export SQL_SERVER_HOST="test_mssql_linux"
chmod -R 777 $(System.ArtifactsDirectory)
docker-compose up -d test_mssql_linux
displayName: "Start database containers"
environment:
SQL_SERVER_HOST: "$(DatabaseServicesAgentPublicIp)"

- stage: DeleteBYOSAgent
displayName: "Delete BYOS Agent"
condition: always()
variables:
athenaservicesEnvironmentIds: $[stageDependencies.CreateAndSetupBYOSAgent.CreateBYOSAgent.outputs['DeployDatabaseServicesImage.environmentIds']]
eteKvName: "ete-variables-kv"
dependsOn:
- CreateAndSetupBYOSAgent
- RunDatabaseRuntimeTestsPortable
- RunDatabaseRuntimeTestsWindows
jobs:
- template: stage.delete.byos.agent.yml
142 changes: 142 additions & 0 deletions Activities/Database/stage.create.byos.agent.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
jobs:
- job: "CreateBYOSAgent"
displayName: "Create BYOS Agent"
timeoutInMinutes: 90
pool:
vmImage: ubuntu-latest
workspace:
clean: outputs | resources | all

steps:
- checkout: none
- task: AzureKeyVault@2
inputs:
azureSubscription: "Ete-Consumer"
KeyVaultName: $(eteKvName)

- task: BYOS@1
name: DeployDatabaseServicesImage
inputs:
action: "CreateAgent"
poolName: "$(eteAgentPoolName)"
projectName: "$(eteProjName)"
ownerUsername: $(Build.RequestedForEmail)
organizationname: "uipath"
authToken: "$(System.AccessToken)"
byosServiceKey: "$(ephemeral-byos-service-key-prod)"
byosServiceUrl: "$(ephemeral-byos-service-url-prod)"
projectShortName: "$(athenaservicesAgentPrefix)"
numberOfAgents: "1"
autoLogon: "True"
agentBitness: "x64"
agentVersion: "3.220.0"
adminUsername: "$(ephemeral-env-admin-username)"
adminPassword: "$(ephemeral-env-admin-password)"
extensionType: "azurePipelinesLinuxAgent"
location: "westeurope"
sourceResourceType: "image"
vmSize: "Standard_D4s_v3"
subnetId: ""
diskSize: "512"
sourceResourceUri: "/subscriptions/3085bae6-b893-42de-ba97-87e64e81e372/resourceGroups/devops-prod-images-we-rg/providers/Microsoft.Compute/images/tas-image-2"
capabilities: '[{"capabilityName":"agentKey","capabilityValue":"$(athenaservicesAgentPrefix)-$(Build.BuildId)"}]'
debug: "True"
hostAgent: "True"
winAgents: "0"
lnxAgents: "0"
osType: "Linux"
expiryTimeMinutes: 90
removePublicIp: "False"

- template: AzurePipelines/wait-for-agent.yml@template-helpers
parameters:
capabilities: "{'agentKey': '$(athenaservicesAgentPrefix)-$(Build.BuildId)'}"
azureDevOpsToken: "$(System.AccessToken)"
poolName: "$(eteAgentPoolName)"
noOfAgents: 1

- job: ConfigureDatabaseServicesAgents
dependsOn: CreateBYOSAgent
pool:
name: "$(eteAgentPoolName)"
demands:
- agentKey -equals $(athenaservicesAgentPrefix)-$(Build.BuildId)
steps:
- checkout: none

- bash: |
sudo apt-get install -y wget apt-transport-https software-properties-common
source /etc/os-release
wget -q https://packages.microsoft.com/config/ubuntu/$VERSION_ID/packages-microsoft-prod.deb
sudo dpkg -i packages-microsoft-prod.deb
rm packages-microsoft-prod.deb
sudo apt-get update
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
curl -fsSL https://download.docker.com/linux/ubuntu/gpg |
sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian bullseye stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update -y
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
docker run --name SQLServer -d -p 1433:1433 -e ACCEPT_EULA=Y -e SA_PASSWORD=Sample123$ mcr.microsoft.com/mssql/server:2022-latest
docker run --name OracleXE_Server -d -e ORACLE_PWD=ORACLE -p 1521:1521 container-registry.oracle.com/database/express:21.3.0-xe
docker ps
displayName: "Setup"
- task: AzureCLI@2
inputs:
scriptType: "bash"
scriptLocation: "inlineScript"
inlineScript: |
prefix="$AGENT_NAME"
rgName="$prefix-rg"
nsgName="$prefix-nsg"
pipName="$prefix-pip0"
echo "Get public IP address of the VM."
publicIpAddress=$(az network public-ip show \
--resource-group $rgName \
--name $pipName \
--query ipAddress \
--output tsv)
echo "Public IP address: $publicIpAddress"
port=1433
echo "Create rule to allow all incoming traffic to port $port for NSG $nsgName."
az network nsg rule create \
--resource-group $rgName \
--nsg-name $nsgName \
--name AthenaControllerAllowIncomingPortForSqlServer \
--priority 4090 \
--source-address-prefixes '*' \
--source-port-ranges '*' \
--destination-address-prefixes '*' \
--destination-port-ranges $port \
--access Allow \
--protocol '*' \
--description "Allow all incoming traffic to port $port"
port=1521
echo "Create rule to allow all incoming traffic to port $port for NSG $nsgName."
az network nsg rule create \
--resource-group $rgName \
--nsg-name $nsgName \
--name AthenaControllerAllowIncomingPortForOracle \
--priority 4091 \
--source-address-prefixes '*' \
--source-port-ranges '*' \
--destination-address-prefixes '*' \
--destination-port-ranges $port \
--access Allow \
--protocol '*' \
--description "Allow all incoming traffic to port $port"
echo "##vso[task.setvariable variable=DatabaseServicesAgentPublicIp;isOutput=true]$publicIpAddress"
azureSubscription: "BYOS-EA"
displayName: "AddNsgRule"
name: "AddNsgRule"
24 changes: 24 additions & 0 deletions Activities/Database/stage.delete.byos.agent.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
jobs:
- job: DeleteEteHost
condition: always()
pool:
vmImage: "ubuntu-latest"
steps:
- checkout: none

- task: AzureKeyVault@2
inputs:
azureSubscription: "Ete-Consumer"
KeyVaultName: $(eteKvName)

- task: BYOS@1
displayName: "Delete agent"
inputs:
action: "DeleteAgent"
projectName: "$(System.TeamProject)"
organizationname: "uipath"
authToken: "$(System.AccessToken)"
buildId: $(Build.BuildId)
environmentId: "$(athenaservicesEnvironmentIds)"
byosServiceKey: "$(ephemeral-byos-service-key-prod)"
byosServiceUrl: "$(ephemeral-byos-service-url-prod)"

0 comments on commit f89c774

Please sign in to comment.