Skip to content

Commit

Permalink
[auto-issue-status]auto-assign and refactor (#20722)
Browse files Browse the repository at this point in the history
* release_iseus_status_auto_reply

* issue_aoto_close_revert

* Update main.py

* Update main.py

* Update update_issue_body.py

* Update reply_generator.py

* Update reply_generator.py

* Update update_issue_body.py

* Update main.py

* Update update_issue_body.py

* Update update_issue_body.py

* Update main.py

* Update reply_generator.py

* Update main.py

* Update update_issue_body.py

* Update main.py

* Update scripts/release_issue_status/update_issue_body.py

* Update update_issue_body.py

* Update main.py

* Update reply_generator.py

* Update update_issue_body.py

* Update main.py

* Update scripts/release_issue_status/main.py

* Update main.py

* Update scripts/release_issue_status/main.py

* Update scripts/release_issue_status/main.py

* Update scripts/release_issue_status/main.py

* Update scripts/release_issue_status/update_issue_body.py

* Update scripts/release_issue_status/main.py

* Update scripts/release_issue_status/update_issue_body.py

* Update reply_generator.py

* Update main.py

* Update scripts/release_issue_status/update_issue_body.py

Co-authored-by: msyyc <70930885+msyyc@users.noreply.github.com>

* Update scripts/release_issue_status/update_issue_body.py

Co-authored-by: msyyc <70930885+msyyc@users.noreply.github.com>

* Update reply_generator.py

* Update update_issue_body.py

* Update main.py

* Update main.py

* Update update_issue_body.py

* Update main.py

* Update scripts/release_issue_status/main.py

* Update main.py

* Update reply_generator.py

* Update main.py

* Update main.py

* Update main.py

* Update main.py

* Update main.py

* Update main.py

* Update main.py

* Update main.py

* Update main.py

* Update update_issue_body.py

* Update main.py

* Update update_issue_body.py

* Update reply_generator.py

* Update update_issue_body.py

* Update main.py

* Update update_issue_body.py

* Update update_issue_body.py

* Update update_issue_body.py

* Update update_issue_body.py

* Update update_issue_body.py

* Update main.py

* Update main.py

* Update release_issue_status.yml for Azure Pipelines

* Update main.py

* Update reply_generator.py

* Add files via upload

* Update reply_generator.py

* Update update_issue_body.py

* Update reply_generator.py

* Update auto_pipeline_run.py

* Update auto_pipeline_run.py

* add auto-close

* Update release_issue_status.yml for Azure Pipelines

* Update auto_close.py

* Update main.py

* Update release_issue_status.yml for Azure Pipelines

* Update auto_pipeline_run.py

* fix bug

* Update main.py

* Update auto_close.py

* Update auto_close.py

* Update main.py

* Update reply_generator.py

* Update main.py

* Update main.py

* Update auto_close.py

* Update main.py

* Update auto-close

* Update auto_pipeline_run.py

* Update update_issue_body.py

* Update auto_pipeline_run.py

* Update release_issue_status.yml for Azure Pipelines

* Update auto_pipeline_run.py

* Update release_issue_status.yml for Azure Pipelines

* Update auto_pipeline_run.py

* Update auto_pipeline_run.py

* Add pipeline link

* Update pipeline link

* Update auto_pipeline_run.py

* Update release_issue_status.yml for Azure Pipelines

* Update auto_pipeline_run.py

* Update requirement.txt

* Update auto_pipeline_run.py

* Add get_python_pipeline

* Update auto_pipeline_run.py

* test

* test

* add outputfolder

* add label

* Update main.py

* Update get_python_pipeline.py

* Update main.py

* Update auto_pipeline_run.py

* Add utils

* Update main.py

* Update main.py

* Delete old py

* Update reply_generator.py

* Update main.py

* Update main.py

* Update main.py

* Update main.py

* Update main.py

* Update utils.py

* Update utils.py

* Update main.py

* Update main.py

* Update reply_generator.py

* Update utils.py

* Add get_changelog function

* Update utils.py

* Update reply_generator.py

* Update main.py

* Update reply_generator.py

* Update function

* del useless code

* Update utils.py

* Update main.py

* Update reply_generator.py

* Update main.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update reply_generator.py

* Update main.py

* Update main.py

* Update main.py

* Update release_issue_status.yml for Azure Pipelines

* Update main.py

* Update release_issue_status.yml for Azure Pipelines

* Update release_issue_status.yml for Azure Pipelines

* Update main.py

* Update utils.py

Co-authored-by: Zed <601306339@qq.com>
Co-authored-by: Zed Lei <59104634+RAY-316@users.noreply.github.com>
Co-authored-by: msyyc <70930885+msyyc@users.noreply.github.com>
  • Loading branch information
4 people authored Sep 27, 2021
1 parent 6bba532 commit 55526df
Show file tree
Hide file tree
Showing 8 changed files with 279 additions and 303 deletions.
36 changes: 0 additions & 36 deletions scripts/release_issue_status/auto_close.py

This file was deleted.

47 changes: 0 additions & 47 deletions scripts/release_issue_status/auto_pipeline_run.py

This file was deleted.

27 changes: 0 additions & 27 deletions scripts/release_issue_status/get_python_pipeline.py

This file was deleted.

64 changes: 33 additions & 31 deletions scripts/release_issue_status/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,22 @@
from datetime import date, datetime
import subprocess as sp
import traceback
import logging

from github import Github
from azure.storage.blob import BlobClient

import reply_generator as rg
from update_issue_body import update_issue_body, find_readme_and_output_folder
from auto_close import auto_close_issue
from get_python_pipeline import get_python_pipelines, get_pipeline_url
from utils import update_issue_body, get_readme_and_output_folder, \
get_python_pipelines, get_pipeline_url, auto_close_issue


_NULL = ' '
_FILE_OUT = 'release_issue_status.csv'
_FILE_OUT_PYTHON = 'release_python_status.md'
_PYTHON_SDK_ADMINISTRATORS = {'msyyc', 'RAY-316', 'BigCat20196'}
logging.basicConfig(level=logging.INFO,
format='[auto-reply log] - %(funcName)s[line:%(lineno)d] - %(levelname)s: %(message)s')

def my_print(cmd):
print('==' + cmd + ' ==\n')
Expand Down Expand Up @@ -127,46 +129,42 @@ def _latest_comment_time(comments, delay_from_create_date):


def auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python_piplines):
print("==========new issue number: {}".format(item.issue_object.number))
if 'Configured' in item.labels:
item.labels.remove('Configured')
logging.info("new issue number: {}".format(item.issue_object.number))

if 'auto-link' not in item.labels:
item.labels.append('auto-link')
item.issue_object.set_labels(*item.labels)
item.issue_object.add_to_labels('auto-link')
try:
package_name, readme_link, output_folder = update_issue_body(request_repo, rest_repo, item.issue_object.number)
print("pkname, readme", package_name, readme_link)
logging.info("pkname, readme", package_name, readme_link)
item.package = package_name
key = ('Python', item.package)
duplicated_issue[key] = duplicated_issue.get(key, 0) + 1
except Exception as e:
item.bot_advice = 'failed to modify the body of the new issue. Please modify manually'
item.labels.append('attention')
item.issue_object.set_labels(*item.labels)
print(e)
item.issue_object.add_to_labels('attention')
logging.info(e)
raise
else:
try:
readme_link, output_folder = find_readme_and_output_folder(request_repo, rest_repo, item.issue_object.number)
readme_link, output_folder = get_readme_and_output_folder(request_repo, rest_repo, item.issue_object.number)
except Exception as e:
print('Issue: {} updates body failed'.format(item.issue_object.number))
item.bot_advice = 'failed to find Readme link, Please check !!'
item.labels.append('attention')
item.issue_object.set_labels(*item.labels)
logging.info('Issue: {} get pkname and output folder failed'.format(item.issue_object.number))
item.bot_advice = 'failed to find Readme link and output folder. Please check !!'
item.issue_object.add_to_labels('attention')
logging.info(e)
raise
try:
print("*********************")
print(python_piplines)
logging.info(python_piplines)
pipeline_url = get_pipeline_url(python_piplines, output_folder)
rg.begin_reply_generate(item=item, rest_repo=rest_repo, readme_link=readme_link,
sdk_repo=sdk_repo, pipeline_url=pipeline_url)
if 'Configured' in item.labels:
item.issue_object.remove_from_labels('Configured')
except Exception as e:
item.bot_advice = 'auto reply failed, Please intervene manually !!'
print('Error from auto reply ========================')
print('Issue:{}'.format(item.issue_object.number))
print(traceback.format_exc())
print('==============================================')
logging.info('Error from auto reply')
logging.info('Issue:{}'.format(item.issue_object.number))
logging.info(traceback.format_exc())


def main():
Expand Down Expand Up @@ -220,10 +218,19 @@ def main():
# rule6: if delay from created date is over 30 days and owner never reply, close it.
# rule7: if delay from created date is over 15 days and owner never reply, remind owner to handle it.
for item in issue_status:
if item.language == 'Python':
issue_status_python.append(item)
if item.status == 'release':
item.bot_advice = 'better to release asap.'
elif (item.comment_num == 0 or 'Configured' in item.labels) and 'Python' in item.labels:
item.bot_advice = 'new issue and better to confirm quickly.'
if 'assigned' not in item.labels:
time.sleep(0.1)
assign_count = int(str(time.time())[-1]) % 2
if assign_count == 1:
item.issue_object.remove_from_assignees(*['RAY-316'])
item.issue_object.add_to_assignees(*['BigCat20196'])
item.issue_object.add_to_labels('assigned')
try:
auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python_piplines)
except Exception as e:
Expand All @@ -237,25 +244,20 @@ def main():
auto_close_issue(request_repo, item)
except Exception as e:
item.bot_advice = 'auto-close failed, please check!'
print(f"=====issue: {item.issue_object.number}, {e}")
logging.info(f"=====issue: {item.issue_object.number}, {e}")

if item.days_from_latest_commit >= 30 and item.language == 'Python' and '30days attention' not in item.labels:
item.labels.append('30days attention')
item.issue_object.set_labels(*item.labels)
item.issue_object.add_to_labels('30days attention')
item.issue_object.create_comment(f'hi @{item.author}, the issue is closed since there is no reply for a long time. Please reopen it if necessary or create new one.')
item.issue_object.edit(state='close')
elif item.days_from_latest_commit >= 15 and item.language == 'Python' and '15days attention' not in item.labels:
item.issue_object.create_comment(f'hi @{item.author}, this release-request has been delayed more than 15 days,'
' please deal with it ASAP. We will close the issue if there is still no response after 15 days!')
item.labels.append('15days attention')
item.issue_object.set_labels(*item.labels)
item.issue_object.add_to_labels('15days attention')

# judge whether there is duplicated issue for same package
if item.package != _NULL and duplicated_issue.get((item.language, item.package)) > 1:
item.bot_advice = f'Warning:There is duplicated issue for {item.package}. ' + item.bot_advice

if item.language == 'Python':
issue_status_python.append(item)

# output result
output_python_md(issue_status_python)
Expand Down
3 changes: 2 additions & 1 deletion scripts/release_issue_status/release_issue_status.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ jobs:
export TOKEN=$(USR_TOKEN)
export HEADERS=$(PIPELINE_HEADERS)
export URL=$(PIPELINE_URL)
export PIPELINE_TOKEN = $(PIPELINE_TOKEN)
export PIPELINE_TOKEN=$(USR_PIPELINE_TOKEN)
export COOKIE=$(USR_TOKEN)
# create virtual env
python -m venv venv-sdk
Expand Down
87 changes: 31 additions & 56 deletions scripts/release_issue_status/reply_generator.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import auto_pipeline_run as apr
from utils import run_pipeline
import re
import logging

issue_object_rg = None
logging.basicConfig(level=logging.INFO,
format='[auto-reply log] - %(funcName)s[line:%(lineno)d] - %(levelname)s: %(message)s')


def weather_change_readme(rest_repo, link_dict, labels):
def readme_comparison(rest_repo, link_dict, labels):
# to see whether need change readme
contents = str(rest_repo.get_contents(link_dict['readme_path']).decoded_content)
pattern_tag = re.compile(r'tag: package-[\w+-.]+')
Expand All @@ -13,6 +16,8 @@ def weather_change_readme(rest_repo, link_dict, labels):
whether_multi_api = 'multi-api' in readme_python_contents
whether_same_tag = link_dict['readme_tag'] in package_tag
whether_change_readme = not whether_same_tag or whether_multi_api and not 'MultiAPI' in labels
if 'Configured' in labels:
whether_change_readme = False
return whether_change_readme


Expand Down Expand Up @@ -47,76 +52,46 @@ def get_latest_pr_from_readme(rest_repo, link_dict):
return latest_pr_number_int[-1]


def latest_pr_parse(rest_repo, latest_pr_number):
latest_pr = rest_repo.get_issue(latest_pr_number)
latest_pr_comments = latest_pr.get_comments()
b = [i for i in latest_pr_comments]
for comment in latest_pr_comments:
if '<h3>Swagger Generation Artifacts</h3>' in comment.body:
return swagger_generator_parse(comment.body, latest_pr_number)


def swagger_generator_parse(context, latest_pr_number):
track1_info_model = ''
try:
if '<b> azure-sdk-for-python</b>' in context:
pattern_python_t1 = re.compile('<b> azure-sdk-for-python</b>.+?</details>', re.DOTALL)
python_t1 = re.search(pattern_python_t1, context).group()
prttern_python_track1 = re.compile('<ul>\s+?<li>\s+?<a.+</ul>', re.DOTALL)
python_track1_info = re.search(prttern_python_track1, python_t1).group()
track1_info_model = '<details open><summary><b> python-track1</b></summary>{} </details>'.format(
python_track1_info)
except Exception as e:
print('track1 generate error')
pattern_python = re.compile('<b> azure-sdk-for-python-track2</b>.+?</details>', re.DOTALL)
python = re.search(pattern_python, context).group()
# the way that reply not contains [Release SDK Changes]
# pattern_python_track2 = re.compile('<ul>\s*?<li>\s*?<a.*</ul>', re.DOTALL)
pattern_python_track2 = re.compile('<b>track2_.*</ul>', re.DOTALL)
python_track2_info = re.search(pattern_python_track2, python).group()
track2_info_model = '<details open><summary><b> python-track2</b></summary>{} </details>'.format(
python_track2_info)
pattern_sdk_changes = re.compile('/azure-sdk-for-python/pull/\d*">Release SDK Changes</a>', re.DOTALL)
sdk_link = re.search(pattern_sdk_changes, python_track2_info).group()
sdk_link_number = re.search(re.compile('[0-9]+'), sdk_link).group()
info_model = 'hi @{} Please check the package whether works well and the changelog info is as below:\n' \
'{}\n{}\n' \
'\n* (The version of the package is only a temporary version for testing)\n' \
'\nhttps://github.com/Azure/azure-rest-api-specs/pull/{}\n' \
.format(issue_object_rg.user.login, track1_info_model, track2_info_model, str(latest_pr_number))

return info_model, sdk_link_number


def reply_owner(reply_content):
issue_object_rg.create_comment(reply_content)


def add_label(label_name, labels):
if label_name not in labels:
labels.append(label_name)
issue_object_rg.set_labels(*labels)

def get_reply_and_sdk_number_from_readme(rest_repo, link_dict):
commits = rest_repo.get_commits(path=link_dict['resource_manager'])
latest_commit = commits[0]
check_run_id = latest_commit.get_check_runs(check_name='SDK azure-sdk-for-python-track2')[0].id
latest_pr_number = latest_commit.get_pulls()[0].number
details = rest_repo.get_check_run(check_run_id).output.text
sdk_link_number = re.findall(r'/azure-sdk-for-python/pull/(\d*)">Release SDK Changes</a>', details)[0]
changelog = '<details open><summary><b> python-track2</b></summary> track2_azure-mgmt-{} </details>'.format(
details.split('</code><b>track2_azure-mgmt-')[-1])
info_model = 'hi @{} Please check the package whether works well and the changelog info ' \
'is as below:\n{}\n' \
'\n* (The version of the package is only a temporary version for testing)\n' \
'\nhttps://github.com/Azure/azure-rest-api-specs/pull/{}\n' \
.format(issue_object_rg.user.login, changelog, str(latest_pr_number))
return info_model, sdk_link_number


def begin_reply_generate(item, rest_repo, readme_link, sdk_repo, pipeline_url):
global issue_object_rg
issue_object_rg = item.issue_object
link_dict = get_links(readme_link)
labels = item.labels
whether_change_readme = weather_change_readme(rest_repo, link_dict, labels)
whether_change_readme = readme_comparison(rest_repo, link_dict, labels)

if not whether_change_readme:
latest_pr_number = get_latest_pr_from_readme(rest_repo, link_dict)
reply_content, sdk_link_number = latest_pr_parse(rest_repo, latest_pr_number)
run_pipeline = apr.run_pipeline(issue_link=issue_object_rg.html_url,
reply_content, sdk_link_number = get_reply_and_sdk_number_from_readme(rest_repo, link_dict)
res_run = run_pipeline(issue_link=issue_object_rg.html_url,
sdk_issue_object=sdk_repo.get_pull(int(sdk_link_number)),
pipeline_url=pipeline_url
)
if run_pipeline:
print(f'{issue_object_rg.number} run pipeline successfully')
if res_run:
logging.info(f'{issue_object_rg.number} run pipeline successfully')
else:
print(f'{issue_object_rg.number} run pipeline fail')
logging.info(f'{issue_object_rg.number} run pipeline fail')
reply_owner(reply_content)
add_label('auto-ask-check', labels)
issue_object_rg.add_to_labels('auto-ask-check')
else:
print('issue {} need config readme***********'.format(issue_object_rg.number))
logging.info('issue {} need config readme'.format(issue_object_rg.number))
Loading

0 comments on commit 55526df

Please sign in to comment.