Skip to content

[pull] master from aurutils:master #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pull merged 2 commits into from
Aug 28, 2025
Merged

[pull] master from aurutils:master #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
897fc49
examples/sync-rebuild: use aur-fetch
AladW Aug 27, 2025
c4ad5dd
Merge pull request #1209 from aurutils/fetch-rebuild
AladW Aug 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions examples/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ clients of the local repository.
Assuming a list of rebuild targets is known, `sync-rebuild` performs the
following steps:

1. Retrieve and inspect all source files with `aur sync`
1. Retrieve and update all source files with `aur fetch --existing`
2. Retrieve all versions of packages in the local repository
3. If `pkgver` matches the local repository version, set `pkgrel` in the
`PKGBUILD` to the local repository version, incremented by `0.1`. Otherwise,
Expand All @@ -126,22 +126,33 @@ in case the incremented version is lost, or otherwise restored (e.g. with
`git-reset`). The fractional part is always increased; for example, a `pkgrel`
of `35.9` is increased to `35.10`, not `36`.

> **Note**
> If one dependency fails to build, `sync-rebuild` will try rebuilding the package
> that depends on it anyway. To avoid this, use the `--fail-fast` option.

`aur-repo` can be used to retrieve a list of packages that depend on specific
package. For example:

```bash
aur repo --search '^python.*' --search-by depends --list
```

`aur-sync` can be used to inspect new PKGBUILD revisions beforehand
and retrieve any new dependencies. For example:

```bash
aur sync --no-build --no-ver-argv <targets...>
```

> **Note**
> By default, AUR packages are rebuilt in dependency order. When using
> `--no-sync` or non-AUR packages, targets are rebuilt in sequential order. In
> this case, `arch-rebuild-order` can be used as follows:
> AUR packages are rebuilt in command-line order. The rebuild order can be
> retrieved with `arch-rebuild-order` as follows:
>
> `$ arch-rebuild-order --repos=custom <targets...>`
>
> If one dependency fails to build, `sync-rebuild` will try rebuilding the package
> that depends on it anyway. To avoid this, use the `--fail-fast` option.
> This ignores any build dependencies specified in `optdepends`; for
> AUR targets, `aur depends --optdepends` may be used (although these
> dependencies need manual installation.)

## view-delta

Expand Down
86 changes: 46 additions & 40 deletions examples/sync-rebuild
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ from pwd import getpwnam
from decimal import Decimal
#from pyalpm import vercmp
from srcinfo.parse import parse_srcinfo
ARGV0 = 'sync-rebuild'
ARGV0 = 'rebuild'

def xdg_cache_home(user=None):
"""Retrieve XDG_CACHE_HOME from the XDG Base Directory specification
Expand Down Expand Up @@ -119,7 +119,8 @@ def update_pkgrel(buildscript, pkgrel=None, increment=0.1):


# TODO: use vercmp to ensure rebuilds, abort reverse depends when depends fails (sync--ninja)
def rebuild_packages(repo_targets, db_name, start_dir, pkgver=False, fail_fast=False, user=None, *build_args):
def rebuild_packages(repo_targets, db_name, start_dir, pkgver=False,
fail_fast=False, user=None, *build_args):
"""Rebuild a series of packages in successive order.
"""
build_cmd = ['aur', 'build'] + list(*build_args)
Expand All @@ -146,7 +147,7 @@ def rebuild_packages(repo_targets, db_name, start_dir, pkgver=False, fail_fast=F

# Run pkgver() function for VCS packages
if pkgver:
print(f'{ARGV0}: updating pkgver with aur-srcver', file=sys.stderr)
print(f'{ARGV0}: {pkgname}: updating pkgver with aur-srcver', file=sys.stderr)
for n, pkg_str in enumerate(run_readline(srcver_cmd, cwd=src_dir)):
if n > 0:
raise RuntimeError('ambiguous aur-srcver output')
Expand All @@ -172,13 +173,13 @@ def rebuild_packages(repo_targets, db_name, start_dir, pkgver=False, fail_fast=F
new_pkgrel = update_pkgrel(buildscript, pkgrel=float(pkgrel), increment=0.1)

# Print bumped pkgrel to standard error
print(f'{ARGV0}: {pkgname}: {pkgver}-{pkgrel} -> {pkgver}-{new_pkgrel}',
print(f'{ARGV0}: {pkgbase}: {pkgver}-{pkgrel} -> {pkgver}-{new_pkgrel}',
file=sys.stderr)
else:
print(f'{ARGV0}: source and local repository version differ', file=sys.stderr)
print(f'{ARGV0}: using existing pkgver', file=sys.stderr)
print(f'{ARGV0}: {pkgbase}: source and local repository version differ', file=sys.stderr)
print(f'{ARGV0}: {pkgbase}: using existing pkgver', file=sys.stderr)

failed_rebuilds = {}
failed = []

# Build package with modified pkgrel
try:
Expand All @@ -204,21 +205,21 @@ def rebuild_packages(repo_targets, db_name, start_dir, pkgver=False, fail_fast=F
except subprocess.CalledProcessError:
# Build process failed, revert to unmodified PKGBUILD
if buildscript_backup is not None:
print(f'{ARGV0}: build failed, reverting PKGBUILD', file=sys.stderr)
print(f'{ARGV0}: {pkgbase}: build failed, reverting PKGBUILD', file=sys.stderr)
os.replace(buildscript_backup, buildscript)

# --fail-fast: if a package failed to build, also consider
# remaining targets as failed
# --fail-fast: if a package failed to build, consider remaining targets as failed
if fail_fast:
print(f'{ARGV0}: {pkgbase}: build failed, exiting', file=sys.stderr)
# XXX: Preserve original order of inputs
return rebuilds, list(set(repo_targets) - set(rebuilds))

# Mark rebuild as failure for later reporting to the user
failed_rebuilds[pkgname] = pkgbase
failed.append(pkgbase)

rebuilds[pkgname] = pkgbase

return rebuilds, failed_rebuilds
return rebuilds, failed


def print_cached_packages(pkgnames):
Expand All @@ -232,26 +233,26 @@ def print_cached_packages(pkgnames):
p2.communicate()


def main(targets, db_name, start_dir, pkgver, fail_fast, run_sync, chroot, user):
def main(targets, db_name, start_dir, pkgver, fail_fast, run_fetch, chroot, user):
# Ensure all sources are available. Only packages are cloned that are
# already available in the local repository.
sync_cmd = ['aur', 'sync', '--no-build', '--no-ver-argv']
repo_cmd = ['aur', 'repo', '--jsonl']
# XXX: Does not retrieve or handle new dependencies.
fetch_cmd = ['aur', 'fetch', '--existing']
repo_cmd = ['aur', 'repo', '--jsonl']

if user is not None:
sync_cmd = ['runuser', '-u', user, '--'] + sync_cmd
repo_cmd = ['runuser', '-u', user, '--'] + repo_cmd
fetch_cmd = ['runuser', '-u', user, '--'] + fetch_cmd
repo_cmd = ['runuser', '-u', user, '--'] + repo_cmd

if db_name is not None:
sync_cmd.extend(('--database', db_name))
repo_cmd.extend(('--database', db_name))

if chroot:
build_args = ['--chroot']
else:
build_args = ['--syncdeps', '--rmdeps', '--noconfirm']

repo_targets = {}
repo_targets_tmp = {}

# Read repository contents line by line to handle potentially large databases
for pkg_str in run_readline(repo_cmd):
Expand All @@ -260,35 +261,40 @@ def main(targets, db_name, start_dir, pkgver, fail_fast, run_sync, chroot, user)

# Restrict to packages specified on the command-line
if pkgname in targets:
repo_targets[pkgname] = {
'PackageBase': pkg['PackageBase'], 'Version' : pkg['Version']
repo_targets_tmp[pkgname] = {
'PackageBase': pkg['PackageBase'], 'Version': pkg['Version']
}

# Restore order of command-line targets
repo_targets = {}
for target in targets:
if target in repo_targets_tmp:
repo_targets[target] = repo_targets_tmp[target]

# Clone targets that are part of the local repository
# TODO: handle "new" AUR targets as usual
if len(repo_targets) > 0:
sync_cmd.extend(list(repo_targets.keys()))
fetch_cmd.extend(list(repo_targets.keys()))

if run_sync:
if run_fetch:
repo_targets_ordered = {} # `dict` preserves order since python >=3.6

# Temporary file for dependency order
with tempfile.NamedTemporaryFile() as sync_queue:
with tempfile.NamedTemporaryFile() as fetch_results:
# Read access to build user
if user is not None:
shutil.chown(sync_queue.name, user=user)
shutil.chown(fetch_results.name, user=user)

# Clone AUR targets and retrieve dependency order. Dependencies
# not in the local repository already will be added as targets.
# XXX: requires at least one valid AUR target
subprocess.run([*sync_cmd, '--save', sync_queue.name], check=True)
# Clone AUR targets. Dependency order is taken from the command-line.
subprocess.run([*fetch_cmd, '--results', fetch_results.name],
cwd=start_dir, check=True)

with open(sync_queue.name, 'r') as f:
# Retrieve names of cloned/fetched packages.
with open(fetch_results.name, 'r') as f:
for line in f.readlines():
name = os.path.basename(line.rstrip())
name = os.path.basename(line.split(':')[-1].rstrip())
repo_targets_ordered[name] = repo_targets[name]

# Local repository targets not retrieved by `aur-sync` are missing from AUR
# Local repository targets not retrieved by `aur-fetch` are missing from AUR
# XXX: append to queue if target directories are available
not_aur = list(set(repo_targets.keys()) - set(repo_targets_ordered.keys()))

Expand All @@ -308,9 +314,9 @@ def main(targets, db_name, start_dir, pkgver, fail_fast, run_sync, chroot, user)

if len(failed) > 0:
print(f'{ARGV0}: the following targets failed to build:', end=' ', file=sys.stderr)
print(' '.join(failed.keys()), file=sys.stderr)
print(' '.join(failed), file=sys.stderr)

rest = list(set(targets) - set(rebuilds.keys()) - set(failed.keys()) - set(not_aur))
rest = list(set(targets) - set(rebuilds.keys()) - set(failed) - set(not_aur))
else:
rest = list(targets)

Expand All @@ -333,7 +339,7 @@ if __name__ == '__main__':
parser.add_argument('-U', '--user')
parser.add_argument('--pkgver', action='store_true')
parser.add_argument('--fail-fast', action='store_true')
parser.add_argument('--no-sync', action='store_false')
parser.add_argument('--no-fetch', action='store_false')
parser.add_argument('targets', nargs='+')
args = parser.parse_args()

Expand All @@ -352,9 +358,9 @@ if __name__ == '__main__':
# Get the path to user-specific cache files
# Note: this only retrieves `AURDEST` from the current user environment.
if 'AURDEST' in os.environ:
aurdest = os.getenv('AURDEST')
start_dir = os.getenv('AURDEST')
else:
aurdest = os.path.join(xdg_cache_home(args.user), 'aurutils/sync')
start_dir = os.path.join(xdg_cache_home(args.user), 'aurutils/sync')

main({i:1 for i in args.targets}, args.database, aurdest,
args.pkgver, args.fail_fast, args.no_sync, args.chroot, args.user)
main({i:1 for i in args.targets}, args.database, start_dir,
args.pkgver, args.fail_fast, args.no_fetch, args.chroot, args.user)