Mgmt: Changelog tool (Azure#18612)

* add change log tools

* add change log in generation

* install requests

* rename all get method

* format pom

* simplify check return type

* add comments for calc definition stages for methods

* use tuple for return type

* split generate and compile, for changelog execute

* update changelog file path

* fix OLD_JAR and NEW_JAR

* fix generation

* refactor name for calc stage

eng/mgmt/automation/changelog.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+import os
+import sys
+import logging
+import argparse
+
+pwd = os.getcwd()
+os.chdir(os.path.abspath(os.path.dirname(sys.argv[0])))
+from parameters import *
+import generate
+os.chdir(pwd)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-s', '--service', required = True)
+    parser.add_argument('--suffix')
+    parser.add_argument('-c', '--compile', action = 'store_true')
+    return parser.parse_args()
+
+
+def main():
+    args = vars(parse_args())
+    sdk_root = os.path.abspath(
+        os.path.join(os.path.dirname(sys.argv[0]), SDK_ROOT))
+    service = args['service']
+    generate.update_parameters(args.get('suffix'))
+
+    if args.get('compile'):
+        generate.compile_package(sdk_root, service)
+
+    versions = generate.get_version(sdk_root, service).split(';')
+    stable_version = versions[1]
+    current_version = versions[2]
+    generate.compare_with_maven_package(sdk_root, service, stable_version,
+                                        current_version)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level = logging.INFO,
+        format = '%(asctime)s %(levelname)s %(message)s',
+        datefmt = '%Y-%m-%d %X',
+    )
+    main()

eng/mgmt/automation/generate.py

@@ -8,9 +8,12 @@
 import shutil
 import logging
 import argparse
+import requests
+import tempfile
 import subprocess
 import collections
 import urllib.parse
+from typing import Tuple
 
 pwd = os.getcwd()
 os.chdir(os.path.abspath(os.path.dirname(sys.argv[0])))
@@ -34,7 +37,6 @@ def generate(
     use: str,
     tag: str = None,
     version: str = None,
-    compile: bool = True,
     **kwargs,
 ):
     module = ARTIFACT_FORMAT.format(service)
@@ -71,15 +73,108 @@ def generate(
     update_root_pom(sdk_root, service)
     update_version(sdk_root, service)
 
-    if compile:
-        if os.system('mvn clean verify package -f {0}/pom.xml -pl {1}:{2} -am'.
-                     format(sdk_root, GROUP_ID, module)) != 0:
-            logging.error('[GENERATE] Maven build fail')
-            return False
+    return True
+
 
+def compile_package(sdk_root, service):
+    module = ARTIFACT_FORMAT.format(service)
+    if os.system(
+            'mvn clean verify package -f {0}/pom.xml -pl {1}:{2} -am'.format(
+                sdk_root, GROUP_ID, module)) != 0:
+        logging.error('[COMPILE] Maven build fail')
+        return False
     return True
 
 
+def generate_changelog_and_breaking_change(
+    sdk_root,
+    old_jar,
+    new_jar,
+    **kwargs,
+) -> Tuple[bool, str]:
+    logging.info('[CHANGELOG] changelog jar: {0} -> {1}'.format(
+        old_jar, new_jar))
+    stdout = subprocess.run(
+        'mvn clean compile exec:java -q -f {0}/eng/mgmt/changelog/pom.xml -DOLD_JAR="{1}" -DNEW_JAR="{2}"'
+        .format(sdk_root, old_jar, new_jar),
+        stdout = subprocess.PIPE,
+        shell = True,
+    ).stdout
+    logging.info('[CHANGELOG] changelog output: {0}'.format(stdout))
+
+    config = json.loads(stdout)
+    return (config.get('breaking', False), config.get('changelog', ''))
+
+
+def update_changelog(changelog_file, changelog):
+    version_pattern = '^## (\d+\.\d+\.\d+(?:-[\w\d\.]+)?) \((.*?)\)'
+    with open(changelog_file, 'r') as fin:
+        old_changelog = fin.read()
+
+    first_version = re.search(version_pattern, old_changelog, re.M)
+    if not first_version:
+        logging.error(
+            '[Changelog][Skip] Cannot read first version from {}'.format(
+                changelog_file))
+        return
+
+    left = old_changelog[first_version.end():]
+    second_version = re.search(version_pattern, left, re.M)
+    if not second_version:
+        logging.error(
+            '[Changelog][Skip] Cannot read second version from {}'.format(
+                changelog_file))
+        return
+
+    first_version_part = old_changelog[:first_version.end() +
+                                       second_version.start()]
+    first_version_part = re.sub('\s+$', '', first_version_part)
+    first_version_part += '\n\n' + changelog.strip() + '\n\n'
+
+    with open(changelog_file, 'w') as fout:
+        fout.write(first_version_part +
+                   old_changelog[first_version.end() + second_version.start():])
+
+    logging.info('[Changelog][Success] Write to changelog')
+
+
+def compare_with_maven_package(sdk_root, service, stable_version,
+                               current_version):
+    if stable_version == current_version:
+        logging.info('[Changelog][Skip] no previous version')
+        return
+
+    module = ARTIFACT_FORMAT.format(service)
+    r = requests.get(
+        MAVEN_URL.format(group_id = GROUP_ID.replace('.', '/'),
+                         artifact_id = module,
+                         version = stable_version))
+    r.raise_for_status()
+    old_jar_fd, old_jar = tempfile.mkstemp('.jar')
+    try:
+        with os.fdopen(old_jar_fd, 'wb') as tmp:
+            tmp.write(r.content)
+        new_jar = os.path.join(
+            sdk_root,
+            JAR_FORMAT.format(service = service,
+                              artifact_id = module,
+                              version = current_version))
+        if not os.path.exists(new_jar):
+            raise Exception('Cannot found built jar in {0}'.format(new_jar))
+        breaking, changelog = generate_changelog_and_breaking_change(
+            sdk_root, old_jar, new_jar)
+        if changelog and changelog.strip() != '':
+            changelog_file = os.path.join(
+                sdk_root,
+                CHANGELOG_FORMAT.format(service = service,
+                                        artifact_id = module))
+            update_changelog(changelog_file, changelog)
+        else:
+            logging.error('[Changelog][Skip] Cannot get changelog')
+    finally:
+        os.remove(old_jar)
+
+
 def add_module_to_modules(modules: str, module: str) -> str:
     post_module = re.search(r'([^\S\n\r]*)</modules>', modules)
     indent = post_module.group(1)
@@ -95,7 +190,7 @@ def add_module_to_modules(modules: str, module: str) -> str:
     return '<modules>\n' + ''.join(all_module) + post_module.group()
 
 
-def add_module_to_default_profile(pom: str, module: str) -> (bool, str):
+def add_module_to_default_profile(pom: str, module: str) -> Tuple[bool, str]:
     for profile in re.finditer(r'<profile>[\s\S]*?</profile>', pom):
         profile_value = profile.group()
         if re.search(r'<id>default</id>', profile_value):
@@ -119,7 +214,7 @@ def add_module_to_default_profile(pom: str, module: str) -> (bool, str):
     return (False, '')
 
 
-def add_module_to_pom(pom: str, module: str) -> (bool, str):
+def add_module_to_pom(pom: str, module: str) -> Tuple[bool, str]:
     if pom.find('<module>{0}</module>'.format(module)) >= 0:
         logging.info('[POM][Skip] pom already has module {0}'.format(module))
         return (True, pom)
@@ -207,7 +302,9 @@ def update_service_ci_and_pom(sdk_root: str, service: str):
         with open(pom_xml_file, 'r') as fin:
             pom_xml = fin.read()
     else:
-        pom_xml = POM_FORMAT.format(service = service, group_id = GROUP_ID, artifact_id = module)
+        pom_xml = POM_FORMAT.format(service = service,
+                                    group_id = GROUP_ID,
+                                    artifact_id = module)
 
     logging.info('[POM][Process] dealing with pom.xml')
     success, pom_xml = add_module_to_pom(pom_xml, module)
@@ -217,6 +314,26 @@ def update_service_ci_and_pom(sdk_root: str, service: str):
         logging.info('[POM][Success] Write to pom.xml')
 
 
+def get_version(
+    sdk_root: str,
+    service: str,
+) -> str:
+    version_file = os.path.join(sdk_root, 'eng/versioning/version_client.txt')
+    module = ARTIFACT_FORMAT.format(service)
+    project = '{0}:{1}'.format(GROUP_ID, module)
+
+    with open(version_file, 'r') as fin:
+        for line in fin.readlines():
+            version_line = line.strip()
+            if version_line.startswith('#'):
+                continue
+            versions = version_line.split(';')
+            if versions[0] == project:
+                return version_line
+    logging.error('Cannot get version of {0}'.format(project))
+    return None
+
+
 def update_version(sdk_root: str, service: str):
     pwd = os.getcwd()
     try:
@@ -254,13 +371,13 @@ def write_version(
         fout.write('\n')
 
 
-def set_or_increase_version_and_generate(
+def set_or_increase_version(
     sdk_root: str,
     service: str,
     preview = True,
     version = None,
     **kwargs,
-):
+) -> Tuple[str, str]:
     version_file = os.path.join(sdk_root, 'eng/versioning/version_client.txt')
     module = ARTIFACT_FORMAT.format(service)
     project = '{0}:{1}'.format(GROUP_ID, module)
@@ -305,13 +422,12 @@ def set_or_increase_version_and_generate(
     # version is given, set and return
     if version:
         if not stable_version:
-            stable_version = current_version
+            stable_version = version
         logging.info(
             '[VERSION][Set] set to given version "{0}"'.format(version))
         write_version(version_file, lines, version_index, project,
-                      stable_version, current_version)
-        generate(sdk_root, service, version = version, **kwargs)
-        return
+                      stable_version, version)
+        return stable_version, version
 
     current_versions = list(re.findall(version_pattern, current_version)[0])
     stable_versions = re.findall(version_pattern, stable_version)
@@ -328,7 +444,6 @@ def set_or_increase_version_and_generate(
 
         write_version(version_file, lines, version_index, project,
                       stable_version, current_version)
-        generate(sdk_root, service, version = current_version, **kwargs)
     else:
         # TODO: auto-increase for stable version and beta version if possible
         current_version = version_format.format(*current_versions)
@@ -340,7 +455,8 @@ def set_or_increase_version_and_generate(
 
         write_version(version_file, lines, version_index, project,
                       stable_version, current_version)
-        generate(sdk_root, service, version = current_version, **kwargs)
+
+    return stable_version, current_version
 
 
 def parse_args() -> argparse.Namespace:
@@ -375,11 +491,6 @@ def parse_args() -> argparse.Namespace:
         default = AUTOREST_CORE_VERSION,
         help = 'Autorest version',
     )
-    parser.add_argument(
-        '--compile',
-        action = 'store_true',
-        help = 'Do compile after generation or not',
-    )
     parser.add_argument('--suffix', help = 'Suffix for namespace and artifact')
     parser.add_argument(
         '--auto-commit-external-change',
@@ -414,7 +525,7 @@ def valid_service(service: str):
     return re.sub('[^a-z0-9_]', '', service.lower())
 
 
-def read_api_specs(api_specs_file: str) -> (str, dict):
+def read_api_specs(api_specs_file: str) -> Tuple[str, dict]:
     # return comment and api_specs
 
     with open(api_specs_file) as fin:
@@ -517,14 +628,20 @@ def sdk_automation(input_file: str, output_file: str):
                     else:
                         tag = 'package-resources-2020-10'
 
-            set_or_increase_version_and_generate(
+            stable_version, current_version = set_or_increase_version(
+                sdk_root,
+                service,
+            )
+            generate(
                 sdk_root,
                 service,
                 spec_root = config['specFolder'],
                 readme = readme,
                 autorest = AUTOREST_CORE_VERSION,
                 use = AUTOREST_JAVA,
-                tag = tag)
+                tag = tag,
+            )
+            compile_package(sdk_root, service)
 
             generated_folder = OUTPUT_FOLDER_FORMAT.format(service)
             packages.append({
@@ -587,7 +704,13 @@ def main():
     service = get_and_update_service_from_api_specs(api_specs_file, spec,
                                                     args['service'])
     args['service'] = service
-    set_or_increase_version_and_generate(sdk_root, **args)
+    stable_version, current_version = set_or_increase_version(sdk_root, **args)
+    args['version'] = current_version
+    generate(sdk_root, **args)
+
+    compile_package(sdk_root, service)
+    compare_with_maven_package(sdk_root, service, stable_version,
+                               current_version)
 
     if args.get('auto_commit_external_change') and args.get(
             'user_name') and args.get('user_email'):
@@ -606,7 +729,7 @@ def main():
 
 if __name__ == '__main__':
     logging.basicConfig(
-        level = logging.DEBUG,
+        level = logging.INFO,
         format = '%(asctime)s %(levelname)s %(message)s',
         datefmt = '%Y-%m-%d %X',
     )

eng/mgmt/automation/generation.yml

@@ -18,7 +18,7 @@ steps:
 - bash: |
     sudo apt-get install -y --upgrade python3-pip python3-setuptools
     pip3 install --upgrade wheel
-    pip3 install --upgrade PyYAML
+    pip3 install --upgrade PyYAML requests
   displayName: Update python
 
 - task: NodeTool@0

eng/mgmt/automation/init.sh

@@ -2,7 +2,7 @@
 
 sudo apt-get install -y --upgrade python3-pip python3-setuptools
 pip3 install --upgrade wheel
-pip3 install --upgrade PyYAML
+pip3 install --upgrade PyYAML requests
 
 cat << EOF > $2
 {"envs": {"PATH": "$JAVA_HOME_11_X64/bin:$PATH", "JAVA_HOME": "$JAVA_HOME_11_X64"}}

eng/mgmt/automation/parameters.py

@@ -12,6 +12,8 @@
 OUTPUT_FOLDER_FORMAT = None
 
 # Constant parameters
+MAVEN_URL = 'https://repo1.maven.org/maven2/{group_id}/{artifact_id}/{version}/{artifact_id}-{version}.jar'
+
 SDK_ROOT = '../../../'  # related to file dir
 AUTOREST_CORE_VERSION = '3.0.6350'
 AUTOREST_JAVA = '@autorest/java@4.0.9'
@@ -22,6 +24,8 @@
 CI_FILE_FORMAT = 'sdk/{0}/ci.yml'
 POM_FILE_FORMAT = 'sdk/{0}/pom.xml'
 README_FORMAT = 'specification/{0}/resource-manager/readme.md'
+JAR_FORMAT = 'sdk/{service}/{artifact_id}/target/{artifact_id}-{version}.jar'
+CHANGELOG_FORMAT = 'sdk/{service}/{artifact_id}/CHANGELOG.md'
 
 MODELERFOUR_ARGUMENTS = '--pipeline.modelerfour.additional-checks=false --pipeline.modelerfour.lenient-model-deduplication=true'
 FLUENTLITE_ARGUMENTS = '--java {0} --azure-arm --verbose --sdk-integration --fluent=lite --java.fluent=lite --java.license-header=MICROSOFT_MIT_SMALL'.format(