Add --clean-loose-per-pack to verdi storage maintain

environment.yml

@@ -12,7 +12,7 @@ dependencies:
 - circus~=0.19.0
 - click-spinner~=0.1.8
 - click<8.3,>=8.1.0
-- disk-objectstore~=1.4.0
+- disk-objectstore@ git+https://github.com/GeigerJ2/disk-objectstore@4871edea58dd20fd62ec9f10fce0d53f63797aa4
 - docstring_parser
 - get-annotations~=0.1
 - python-graphviz~=0.19

pyproject.toml

@@ -35,7 +35,8 @@ dependencies = [
   'circus~=0.19.0',
   'click-spinner~=0.1.8',
   'click>=8.1.0,<8.3',
-  'disk-objectstore~=1.4.0',
+  # 'disk-objectstore~=1.4.0',
+  'disk-objectstore @ git+https://github.com/GeigerJ2/disk-objectstore@4871edea58dd20fd62ec9f10fce0d53f63797aa4',
   'docstring-parser',
   'get-annotations~=0.1;python_version<"3.10"',
   'graphviz~=0.19',
@@ -317,6 +318,9 @@ exclude = [
   'tests/'
 ]
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.mypy]
 disallow_any_generics = false
 disallow_incomplete_defs = false

src/aiida/cmdline/commands/cmd_storage.py

@@ -152,9 +152,15 @@ def storage_info(detailed):
 @click.option(
     '--compress', is_flag=True, default=False, help='Use compression if possible when carrying out maintenance tasks.'
 )
+@click.option(
+    '--clean-loose-per-pack/--no-clean-loose-per-pack',
+    is_flag=True,
+    default=True,
+    help='Delete corresponding loose files immediately after each pack creation when running a `full maintenance`.',
+)
 @decorators.with_dbenv()
 @click.pass_context
-def storage_maintain(ctx, full, no_repack, force, dry_run, compress):
+def storage_maintain(ctx, full, no_repack, force, dry_run, compress, clean_loose_per_pack):
     """Performs maintenance tasks on the repository."""
     from aiida.common.exceptions import LockingProfileError
     from aiida.common.progress_reporter import set_progress_bar_tqdm, set_progress_reporter
@@ -198,9 +204,15 @@ def storage_maintain(ctx, full, no_repack, force, dry_run, compress):
 
     try:
         if full and no_repack:
-            storage.maintain(full=full, dry_run=dry_run, do_repack=False, compress=compress)
+            storage.maintain(
+                full=full,
+                dry_run=dry_run,
+                do_repack=False,
+                compress=compress,
+                clean_loose_per_pack=clean_loose_per_pack,
+            )
         else:
-            storage.maintain(full=full, dry_run=dry_run, compress=compress)
+            storage.maintain(full=full, dry_run=dry_run, compress=compress, clean_loose_per_pack=clean_loose_per_pack)
     except LockingProfileError as exception:
         echo.echo_critical(str(exception))
     echo.echo_success('Requested maintenance procedures finished.')

src/aiida/repository/backend/disk_object_store.py

@@ -154,6 +154,7 @@ def maintain(  # type: ignore[override]
         clean_storage: t.Optional[bool] = None,
         do_vacuum: t.Optional[bool] = None,
         compress: bool = False,
+        clean_loose_per_pack: bool = True,
     ) -> dict:
         """Performs maintenance operations.
 
@@ -163,6 +164,8 @@ def maintain(  # type: ignore[override]
         :param clean_storage: flag for forcing the cleaning of soft-deleted files from the repository.
         :param do_vacuum: flag for forcing the vacuuming of the internal database when cleaning the repository.
         :param compress: flag for compressing the data when packing loose files. Set to ``Compress.AUTO`` if ``True``.
+        :param clean_loose_per_pack: if True, the loose files that went into a `pack` are deleted immediately after this
+            `pack` is created.
         :return: a dictionary with information on the operations performed.
         """
         from disk_objectstore import CompressMode
@@ -196,7 +199,9 @@ def maintain(  # type: ignore[override]
                 if not dry_run:
                     with get_progress_reporter()(total=1) as progress:
                         callback = create_callback(progress)
-                        container.pack_all_loose(compress=compress, callback=callback)
+                        container.pack_all_loose(
+                            compress=compress, callback=callback, clean_loose_per_pack=clean_loose_per_pack
+                        )
 
             if do_repack:
                 files_numb = container.count_objects().packed