Merge pull request #664 from atlassian/DCA-1261-create-prepare-data-bamboo

Bamboo prepare_data and related API client

Author: SergeyMoroz0703

app/bamboo.yml

@@ -0,0 +1,89 @@
+---
+settings:
+  artifacts-dir: results/bamboo/%Y-%m-%d_%H-%M-%S
+  aggregator: consolidator
+  verbose: false
+  env:
+    application_hostname: test-bamboo.atlassian.com    # Bamboo DC hostname without protocol and port e.g. test-bamboo.atlassian.com or localhost
+    application_protocol: http      # http or https
+    application_port: 8085            # 80, 443, 8080, etc
+    secure: True                    # Set False to allow insecure connections, e.g. when using self-signed SSL certificate
+    application_postfix:            # e.g. /bamboo in case of url like http://localhost/bamboo
+    admin_login: admin
+    admin_password: admin
+    load_executor: jmeter            
+    concurrency: 1000               # number of Bamboo users for Selenium actions
+    test_duration: 45m
+
+    # 1 node scenario parameters
+    ramp-up: 20s                    # time to spin all concurrent threads
+    total_actions_per_hour: 180000  # number of total JMeter actions per hour
+
+    # 2 nodes scenario parameters
+    # ramp-up: 10s                    # time to spin all concurrent threads
+    # total_actions_per_hour: 360000  # number of total JMeter actions per hour
+
+    # 4 nodes scenario parameters
+    # ramp-up: 5s                     # time to spin all concurrent threads
+    # total_actions_per_hour: 720000  # number of total JMeter actions per hour
+
+    JMETER_VERSION: 5.2.1
+    LANGUAGE: en_US.utf8
+    allow_analytics: No            # Allow sending basic run analytics to Atlassian. These analytics help us to understand how the tool is being used and help us to continue to invest in this tooling. For more details please see our README.
+services:
+  - module: shellexec
+    prepare:
+      - python util/pre_run/environment_checker.py
+      - python util/data_preparation/bamboo_prepare_data.py
+#    shutdown:
+#      - python util/post_run/jmeter_post_check.py
+#      - python util/jtl_convertor/jtls-to-csv.py kpi.jtl
+#    post-process:
+#      - python util/analytics/analytics.py bamboo
+#      - python util/post_run/cleanup_results_dir.py
+execution:
+  - scenario: ${load_executor}
+    executor: ${load_executor}
+    concurrency: ${concurrency}
+    hold-for: ${test_duration}
+    ramp-up: ${ramp-up}
+scenarios:
+  jmeter:
+    script: jmeter/bamboo.jmx
+    properties:
+      application_hostname: ${application_hostname}
+      application_protocol: ${application_protocol}
+      application_port: ${application_port}
+      application_postfix: ${application_postfix}
+      # Workload model
+      total_actions_per_hr: ${total_actions_per_hour}
+modules:
+  consolidator:
+    rtimes-len: 0 # CONFSRVDEV-7631 reduce sampling
+    percentiles: [] # CONFSRVDEV-7631 disable all percentiles due to Taurus's excessive memory usage
+  jmeter:
+    version: ${JMETER_VERSION}
+    detect-plugins: true
+    memory-xmx: 8G  # allow JMeter to use up to 8G of memory
+    plugins:
+      - bzm-parallel=0.4
+      - bzm-random-csv=0.6
+      - jpgc-casutg=2.5
+      - jpgc-dummy=0.2
+      - jpgc-ffw=2.0
+      - jpgc-fifo=0.2
+      - jpgc-functions=2.1
+      - jpgc-json=2.6
+      - jpgc-perfmon=2.1
+      - jpgc-prmctl=0.4
+      - jpgc-tst=2.4
+      - jpgc-wsc=0.3
+      - tilln-sshmon=1.0
+      - jpgc-synthesis=2.2
+    system-properties:
+      server.rmi.ssl.disable: true
+      java.rmi.server.hostname: localhost
+      httpsampler.ignore_failed_embedded_resources: "true"
+reporting:
+- data-source: sample-labels
+  module: junit-xml

app/util/api/bamboo_clients.py

@@ -0,0 +1,63 @@
+from util.api.abstract_clients import RestClient
+
+BATCH_SIZE_SEARCH = 500
+
+
+class BambooClient(RestClient):
+
+    def get_build_plans(self, start=0, max_result=100):
+        loop_count = max_result // BATCH_SIZE_SEARCH + 1
+        content = list()
+        last_loop_remainder = max_result % BATCH_SIZE_SEARCH
+        max_result = BATCH_SIZE_SEARCH if max_result > BATCH_SIZE_SEARCH else max_result
+
+        while loop_count > 0:
+            if not max_result:
+                break
+            api_url = (
+                    self.host + f'/rest/api/latest/search/plans?start-index={start}'
+                                f'&max-result={max_result}'
+            )
+            request = self.get(api_url, "Could not retrieve build plans")
+            if request.json()['start-index'] != start:
+                break
+            content.extend(request.json()['searchResults'])
+
+            loop_count -= 1
+            if loop_count == 1:
+                max_result = last_loop_remainder
+
+            start += len(request.json()['searchResults'])
+
+        return content
+
+    def get_users(self, limit):
+        """
+        Retrieve a page of users. The authenticated user must have restricted
+        administrative permission or higher to use this resource.
+        :param start: The starting index of the returned users. Base index: 0.
+        :param limit: The maximum number of users to return per page. Default: 25.
+        """
+        request = self.get(f'{self.host}/rest/api/latest/admin/users?limit={limit}',
+                           error_msg="Can not retrieve users")
+        content = request.json()
+        return content['results']
+
+    def create_user(self, name, password):
+        """
+        Create a new user. The authenticated user must have restricted administrative
+        permission or higher to use this resource.
+        :param name: username to create.
+        :param fullName: full user name.
+        :email: email address.
+        :password: password.
+        :passwordConfirm: confirm password.
+        """
+        api_url = f'{self.host}/rest/api/latest/admin/users'
+        payload = {"name": name,
+                   "fullName": name,
+                   "email": f'{name}@example.com',
+                   "password": password,
+                   "passwordConfirm": password}
+        self.post(api_url, body=payload, error_msg="Could not create user")
+        return {'name': name}

app/util/conf.py

@@ -1,6 +1,6 @@
 import yaml
 
-from util.project_paths import JIRA_YML, CONFLUENCE_YML, BITBUCKET_YML, JSM_YML, CROWD_YML
+from util.project_paths import JIRA_YML, CONFLUENCE_YML, BITBUCKET_YML, JSM_YML, CROWD_YML, BAMBOO_YML
 
 TOOLKIT_VERSION = '4.3.0'
 
@@ -94,8 +94,18 @@ def __init__(self, config_yml):
         self.ramp_up = self.get_property('ramp-up')
 
 
+class BambooSettings(BaseAppSettings):
+
+    def __init__(self, config_yml):
+        super().__init__(config_yml)
+        self.concurrency = self.get_property('concurrency')
+        self.total_actions_per_hour = self.get_property('total_actions_per_hour')
+        self.ramp_up = self.get_property('ramp-up')
+
+
 JIRA_SETTINGS = JiraSettings(config_yml=JIRA_YML)
 CONFLUENCE_SETTINGS = ConfluenceSettings(config_yml=CONFLUENCE_YML)
 BITBUCKET_SETTINGS = BitbucketSettings(config_yml=BITBUCKET_YML)
 JSM_SETTINGS = JsmSettings(config_yml=JSM_YML)
 CROWD_SETTINGS = CrowdSettings(config_yml=CROWD_YML)
+BAMBOO_SETTINGS = BambooSettings(config_yml=BAMBOO_YML)

app/util/data_preparation/bamboo_prepare_data.py

@@ -0,0 +1,74 @@
+import random
+import string
+
+import urllib3
+
+from util.conf import BAMBOO_SETTINGS
+from util.api.bamboo_clients import BambooClient
+from util.project_paths import BAMBOO_BUILD_PLANS, BAMBOO_USERS
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+BUILD_PLANS = 'plans'
+USERS = 'users'
+DEFAULT_PASSWORD = 'password'
+
+
+def generate_random_string(length=20):
+    return "".join([random.choice(string.ascii_lowercase) for _ in range(length)])
+
+
+def get_users(client, users_count):
+    existing_users = client.get_users(users_count)
+    users_to_generate = 0
+    users = []
+    if len(existing_users) < users_count:
+        users_to_generate = users_count - len(existing_users)
+    users.extend(existing_users)
+    if users_to_generate:
+        for i in range(0, users_to_generate):
+            username = f'performance_user_{generate_random_string(5)}'
+            password = DEFAULT_PASSWORD
+            generated_user = client.create_user(name=username, password=password)
+            users.append(generated_user)
+    return users
+
+
+def __create_dataset(client):
+    dataset = dict()
+    dataset[BUILD_PLANS] = client.get_build_plans(max_result=2000)
+    dataset[USERS] = get_users(client, BAMBOO_SETTINGS.concurrency)
+    return dataset
+
+
+def __write_to_file(file_path, items):
+    with open(file_path, 'w') as f:
+        for item in items:
+            f.write(f"{item}\n")
+
+
+def write_test_data_to_files(dataset):
+    build_plans = [f"{build_plan['searchEntity']['projectName']},{build_plan['id']}" for
+                   build_plan in dataset[BUILD_PLANS]]
+    __write_to_file(BAMBOO_BUILD_PLANS, build_plans)
+    users = [f"{user['name']},{DEFAULT_PASSWORD}" for user in dataset[USERS] if user['name'] != 'admin']
+    __write_to_file(BAMBOO_USERS, users)
+
+
+def main():
+    print("Started preparing data")
+
+    url = BAMBOO_SETTINGS.server_url
+    print("Server url: ", url)
+
+    client = BambooClient(url, BAMBOO_SETTINGS.admin_login, BAMBOO_SETTINGS.admin_password,
+                          verify=BAMBOO_SETTINGS.secure)
+
+    dataset = __create_dataset(client)
+    write_test_data_to_files(dataset)
+
+    print("Finished preparing data")
+
+
+if __name__ == "__main__":
+    main()

app/util/project_paths.py

@@ -55,6 +55,14 @@ def __get_crowd_dataset(file_name):
     return __get_crowd_datasets() / file_name
 
 
+def __get_bamboo_yml():
+    return Path(__file__).parents[1] / "bamboo.yml"
+
+
+def __get_bamboo_datasets():
+    return __get_datasets() / "bamboo"
+
+
 def __get_confluence_datasets():
     return __get_datasets() / "confluence"
 
@@ -63,6 +71,10 @@ def __get_confluence_dataset(file_name):
     return __get_confluence_datasets() / file_name
 
 
+def __get_bamboo_dataset(file_name):
+    return __get_bamboo_datasets() / file_name
+
+
 def __get_bitbucket_dataset(file_name):
     return __get_bitbucket_datasets() / file_name
 
@@ -121,6 +133,11 @@ def __get_default_test_actions():
 CROWD_DATASETS = __get_crowd_datasets()
 CROWD_USERS = __get_crowd_dataset('users.csv')
 
+BAMBOO_YML = __get_bamboo_yml()
+BAMBOO_DATASETS = __get_bamboo_datasets()
+BAMBOO_BUILD_PLANS = __get_bamboo_dataset('build_plans.csv')
+BAMBOO_USERS = __get_bamboo_dataset('users.csv')
+
 
 DEFAULT_TEST_ACTIONS = __get_default_test_actions()
 ENV_TAURUS_ARTIFACT_DIR = __get_taurus_artifacts_dir()