Skip to content

Commit

Permalink
Add support for git project handling
Browse files Browse the repository at this point in the history 
Automatically switches over from OBS api calls to git
when specifing a git repository on CLI
  • Loading branch information
@adrianschroeter
adrianschroeter committed May 7, 2024
1 parent 9b449f9 commit cf36ceb
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 48 deletions.
7 changes: 4 additions & 3 deletions pkglistgen/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ def do_handle_update_repos(self, subcmd, opts, project):
@cmdln.option('-f', '--force', action='store_true', help='continue even if build is in progress')
@cmdln.option('-d', '--dry', help='no modifications uploaded')
@cmdln.option('-p', '--project', help='target project')
@cmdln.option('-g', '--git-url', help='git repository for target project')
@cmdln.option('-s', '--scope', help='scope on which to operate ({}, staging:$letter)'.format(', '.join(SCOPES)))
@cmdln.option('--no-checkout', action='store_true', help='reuse checkout in cache')
@cmdln.option('--stop-after-solve', action='store_true', help='only create group files')
Expand Down Expand Up @@ -93,13 +94,13 @@ def solve_project(project, scope: str):
self.tool.reset()
self.tool.dry_run = self.options.dry
return self.tool.update_and_solve_target(api, target_project, target_config, main_repo,
project=project, scope=scope, force=opts.force,
no_checkout=opts.no_checkout,
git_url=opts.git_url, project=project, scope=scope,
force=opts.force, no_checkout=opts.no_checkout,
only_release_packages=opts.only_release_packages,
stop_after_solve=opts.stop_after_solve,
custom_cache_tag=opts.custom_cache_tag)
except MismatchedRepoException:
logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again.")
logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again!")
# for stagings we have to be strict on the exit value
if scope == 'staging':
return 1
Expand Down
8 changes: 6 additions & 2 deletions pkglistgen/group.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,11 +330,15 @@ def tocompose(self, prefix, arch, ignore_broken=False, comment=None):
continue
if name in missing:
if ignore_broken and name not in self.required:
content += prefix + "#- " + name + " is missing\n"
msg = ' {} not found on {}'.format(name, ','.join(sorted(missing[name])))
content += prefix + "#- " + msg + "\n"
self.logger.error(msg)
continue
if name in unresolvable:
if ignore_broken and name not in self.required:
content += prefix + "#- " + name + " is unresolvable: " + unresolvable[name] + "\n"
msg = ' {} uninstallable: {}'.format(name, unresolvable[name])
content += prefix + "#- " + msg + "\n"
self.logger.error(msg)
continue
content += prefix + "- " + name
if comment:
Expand Down
134 changes: 91 additions & 43 deletions pkglistgen/tool.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))

PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product'
PRODUCT_COMPOSE_SEPERATOR_LINE = '# The following is generated by openSUSE-release-tools'

# share header cache with repochecker
CACHEDIR = CacheManager.directory('repository-meta')
Expand Down Expand Up @@ -141,7 +142,7 @@ def write_productcompose(self):
archs = ['*'] + self.all_architectures
# a single file covering all builds via multibuild flavors
with open(os.path.join(self.productcompose_dir, 'default.productcompose'), 'a') as opfh:
opfh.write("# The following is generated by openSUSE-release-tools\n")
opfh.write(PRODUCT_COMPOSE_SEPERATOR_LINE + "\n")
for name in self.groups:
group = self.groups[name]
if not group.solved:
Expand Down Expand Up @@ -656,6 +657,7 @@ def update_and_solve_target(
target_project: str,
target_config: Mapping[str, Any],
main_repo: str,
git_url: str,
project: str,
scope: str,
force: bool,
Expand All @@ -675,43 +677,60 @@ def update_and_solve_target(
release = target_config.get('pkglistgen-release', '000release-packages')
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')

url = api.makeurl(['source', project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, product, 'revive')
# TODO disable build.
logging.info('{} undeleted, skip dvd until next cycle'.format(product))
return
elif not force:
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
repository=[main_repo], multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
logging.info('{}/{} build in progress'.format(project, product))
return

drop_list = api.item_exists(project, oldrepos)
checkout_list = [group, product, release]
if drop_list and not only_release_packages:
checkout_list.append(oldrepos)

if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, release, 'revive')
logging.info('{} undeleted, skip dvd until next cycle'.format(release))
return

# Cache dir specific to hostname and project.
host = urlparse(api.apiurl).hostname
prefix_dir = 'pkglistgen'
if custom_cache_tag:
prefix_dir += '-' + custom_cache_tag
cache_dir = CacheManager.directory(prefix_dir, host, project)

if not no_checkout:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
drop_list = []
checkout_list = [group, product, release]
if not force:
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
repository=[main_repo], multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
logging.info('{}/{} build in progress'.format(project, product))
return
if git_url:
if os.path.exists(cache_dir + "/.git"):
# reset and update existing clone
logging.debug(subprocess.check_output(
['git', 'reset', '--hard'], cwd=cache_dir, encoding='utf-8'))
logging.debug(subprocess.check_output(
['git', 'pull', '--rebase'], cwd=cache_dir, encoding='utf-8'))
else:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
logging.debug(subprocess.check_output(
['git', 'clone', '--recurse-submodules', git_url, cache_dir], encoding='utf-8'))
if os.path.exists(cache_dir + '/' + '000update-repos'):
logging.error('No support for 000update-repos in git projects atm')
return
else:
url = api.makeurl(['source', project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, product, 'revive')
# TODO disable build.
logging.info('{} undeleted, skip dvd until next cycle'.format(product))
return

drop_list = api.item_exists(project, oldrepos)
if drop_list and not only_release_packages:
checkout_list.append(oldrepos)

if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, release, 'revive')
logging.info('{} undeleted, skip dvd until next cycle'.format(release))
return

if not no_checkout:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)

group_dir = os.path.join(cache_dir, group)
product_dir = os.path.join(cache_dir, product)
Expand All @@ -722,36 +741,41 @@ def update_and_solve_target(
self.input_dir = group_dir
self.output_dir = product_dir

for package in checkout_list:
if no_checkout:
logging.debug('Skipping checkout of {}/{}'.format(project, package))
continue
checkout_package(api.apiurl, project, package, expand_link=True,
prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))
if not no_checkout and not git_url:
logging.debug('Skipping checkout of {}'.format(project))
for package in checkout_list:
checkout_package(api.apiurl, project, package, expand_link=True,
prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))

file_utils.unlink_all_except(release_dir, ['weakremovers.inc', '*.changes'])
if not only_release_packages:
file_utils.unlink_all_except(product_dir)
ignore_list = ['supportstatus.txt', 'summary-staging.txt', 'package-groups.changes']
ignore_list = ['supportstatus.txt', 'summary-staging.txt', 'package-groups.changes',
'default.productcompose.in']
ignore_list += self.group_input_files()
file_utils.copy_directory_contents(group_dir, product_dir, ignore_list)
file_utils.change_extension(product_dir, '.spec.in', '.spec')
file_utils.change_extension(product_dir, '.product.in', '.product')
fn = os.path.join(group_dir, 'default.productcompose')
fn = os.path.join(group_dir, 'default.productcompose.in')
if os.path.isfile(fn):
if not os.path.isdir(self.productcompose_dir):
os.mkdir(self.productcompose_dir) # currently always creating it local at least
raise Exception('default.productcompose.in exists, but output directory is missing in git!')
lines = open(fn).readlines()
new_lines = []
for line in lines:
if line.startswith('# The following is generated by openSUSE-release-tools'):
if line.startswith(PRODUCT_COMPOSE_SEPERATOR_LINE):
break
new_lines.append(line)
open(os.path.join(self.productcompose_dir, 'default.productcompose'), 'w').write(''.join(new_lines))
<<<<<<< HEAD
if git_url:
logging.debug(subprocess.check_output(
['git', 'add', self.productcompose_dir, f"{self.productcompose_dir}/default.productcompose"],
cwd=cache_dir, encoding='utf-8'))
=======
if os.path.isfile(os.path.join(group_dir, 'supportstatus.txt')):
shutil.copy(os.path.join(group_dir, 'supportstatus.txt'), self.productcompose_dir)
>>>>>>> 8c7c16b8 (git cleanup)
self.skip_productcompose = False
else:
# No template file, so we don't create one either
Expand Down Expand Up @@ -834,7 +858,11 @@ def update_and_solve_target(
"- automatically generated by openSUSE-release-tools/pkglistgen\n\n"
)

self.commit_package(release_dir)
if git_url:
logging.debug(subprocess.check_output(
'git add *.spec', cwd=release_dir, shell=True, encoding='utf-8'))
else:
self.commit_package(release_dir)

if only_release_packages:
return
Expand All @@ -854,7 +882,27 @@ def update_and_solve_target(
for line in sorted(output):
f.write(line + '\n')

self.commit_package(product_dir)
if git_url:
<<<<<<< HEAD
if os.path.isfile(f"{productcompose_dir}/default.productcompose"):
logging.debug(subprocess.check_output(
['git', 'add', productcompose_dir, f"{productcompose_dir}/default.productcompose"],
cwd=cache_dir, encoding='utf-8'))
if os.path.isdir(product_dir):
=======
if product_dir and os.path.isdir(product_dir):
>>>>>>> 8c7c16b8 (git cleanup)
logging.debug(subprocess.check_output(
['git', 'add', productcompose_dir, product_dir],
cwd=cache_dir, encoding='utf-8'))

logging.debug(subprocess.check_output(
'git commit -m "Update by pkglistgen of openSUSE-release-tool"', cwd=cache_dir, shell=True, encoding='utf-8'))
if not self.dry_run:
logging.debug(subprocess.check_output(
'git push', cwd=cache_dir, shell=True, encoding='utf-8'))
elif not self.dry_run:
self.commit_package(product_dir)

if os.path.isfile(reference_summary):
return self.comment.handle_package_diff(project, reference_summary, summary_file)

0 comments on commit cf36ceb

Please sign in to comment.