Feat delete empty folders

.github/workflows/release.yml

@@ -29,7 +29,7 @@ jobs:
 
       - name: Test with pytest
         run: |
-          python3 -m pytest -s --color=yes -vv tests
+          python3 -m pytest -s --color=yes -vv tests artifactory_cleanup
 
       - name: Build package
         run: python -m build

CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+## 0.4.0
+
+- Remove `--remove-empty-folder` option. Artifactory provides corresponding built-in functionality already
+- Change the `delete_empty_folder` rule to not depend on an external plugin, but directly delete files from this script
+
+## 0.3.4
+
+* Previous versions do not yet have a changelog

README.md

@@ -14,6 +14,7 @@
   - [Available Rules](#available-rules)
   - [Artifact cleanup policies](#artifact-cleanup-policies)
   - [Docker Container Usage](#docker-container-usage)
+- [FAQ](#faq)
 - [Release](#release)
 
 <!-- tocstop -->
@@ -41,7 +42,7 @@ You should take the following steps:
 2. Сreate a python file, for example, `reponame.py` with the following contents:
 ```python
 from artifactory_cleanup import rules
-from artifactory_cleanup.rules import CleanupPolicy
+from artifactory_cleanup import CleanupPolicy
 
 RULES = [
 
@@ -69,11 +70,6 @@ artifactory-cleanup --destroy --user user --password password --artifactory-serv
 # debug run - only print founded artifacts. it do not delete
 artifactory-cleanup --user user --password password --artifactory-server https://repo.example.com/artifactory --config reponame.py
 
-# Clean up empty folder
-# --remove-empty-folder
-# You need to use the plugin https://github.com/jfrog/artifactory-user-plugins/tree/master/cleanup/deleteEmptyDirs to delete empty folders
-artifactory-cleanup --remove-empty-folder --user user --password password --artifactory-server https://repo.example.com/artifactory
-
 # Debug run only for policytestname. Find any *policytestname*
 # debug run - only print founded artifacts. it do not delete
 artifactory-cleanup --policy-name policytestname --user user --password password --artifactory-server https://repo.example.com/artifactory --config reponame.py
@@ -97,7 +93,7 @@ To add a cleaning policy you need:
 
 ```python
 from artifactory_cleanup import rules
-from artifactory_cleanup.rules import CleanupPolicy
+from artifactory_cleanup import CleanupPolicy
 
 RULES = [
 
@@ -151,10 +147,40 @@ To build the container image locally run the following command in the folder of
 ```bash
 docker build . --tag artifactory-cleanup
 ```
+# FAQ
+
+## How to clean up Conan repository?
+The idea came from https://github.com/devopshq/artifactory-cleanup/issues/47
+
+```python
+from artifactory_cleanup import rules
+from artifactory_cleanup import CleanupPolicy
+RULES = [
+    # ------ ALL REPOS --------
+    CleanupPolicy(
+       'Delete files older than 60 days',
+        rules.repo('conan-testing'),
+        rules.delete_not_used_since(days=60),
+        # Make sure to keep conan metadata. See also
+        # https://github.com/devopshq/artifactory-cleanup/issues/47
+        rules.exclude_filename(['.timestamp', 'index.json']),
+    ),
+    CleanupPolicy(
+       'Delete empty folders',
+        rules.repo('conan-testing'),
+        rules.delete_empty_folder(),
+        # Exclude metadata files
+        # If a folder only contains these files, consider it as empty
+        rules.exclude_filename(['.timestamp', 'index.json']),
+    ),
+]
+```
+
 
 # Release
 
 In order to provide a new release of `artifactory-cleanup`, there are two steps involved.
 
 1. Bump the version in the [setup.py](setup.py)
 2. Create a Git release tag (e.g. `v0.3.3`) by creating a release on Github
+

artifactory_cleanup/__init__.py

@@ -1 +1,2 @@
 from artifactory_cleanup.artifactorycleanup import ArtifactoryCleanup  # noqa
+from artifactory_cleanup.rules.base import CleanupPolicy  # noqa

artifactory_cleanup/artifactorycleanup.py

@@ -10,7 +10,6 @@
 from requests.auth import HTTPBasicAuth
 from artifactory_cleanup.context_managers import get_context_managers
 from artifactory_cleanup.rules.base import CleanupPolicy
-from artifactory_cleanup.rules.delete import delete_empty_folder
 
 
 requests.packages.urllib3.disable_warnings()
@@ -61,10 +60,6 @@ class ArtifactoryCleanup(cli.Application):
         mandatory=False,
     )
 
-    _remove_empty_folder = cli.Flag(
-        "--remove-empty-folder", help="Cleaning up empty folders in local repositories"
-    )
-
     _days_in_future = cli.SwitchAttr(
         "--days-in-future",
         help="Simulate future behaviour",
@@ -83,21 +78,13 @@ def _destroy_or_verbose(self):
     def main(self):
         # remove trailing slash
         self._artifactory_server = self._artifactory_server.rstrip("/")
-        if self._remove_empty_folder:
-            rules = [
-                CleanupPolicy(
-                    "Cleaning up empty folders in local repositories",
-                    delete_empty_folder(),
-                )
-            ]
-        else:
-            try:
-                self._config = self._config.replace(".py", "")
-                sys.path.append(".")
-                rules = getattr(importlib.import_module(self._config), "RULES")
-            except ImportError as error:
-                print("Error: {}".format(error))
-                exit(1)
+        try:
+            self._config = self._config.replace(".py", "")
+            sys.path.append(".")
+            rules = getattr(importlib.import_module(self._config), "RULES")
+        except ImportError as error:
+            print("Error: {}".format(error))
+            exit(1)
 
         self._destroy_or_verbose()
 

artifactory_cleanup/rules/base.py

@@ -200,7 +200,12 @@ def filter(self, artifacts):
         return artifacts
 
     def delete(self, artifact, destroy):
-        artifact_path = quote("{repo}/{path}/{name}".format(**artifact))
+        if artifact["path"] == ".":
+            artifact_path = "{repo}/{name}".format(**artifact)
+        else:
+            artifact_path = "{repo}/{path}/{name}".format(**artifact)
+
+        artifact_path = quote(artifact_path)
         if destroy:
             print("DESTROY MODE - delete {}".format(artifact_path))
             delete_url = "{}/{}".format(self.artifactory_url, artifact_path)

artifactory_cleanup/rules/delete.py

@@ -1,5 +1,6 @@
 from datetime import timedelta
 
+from artifactory_cleanup.rules import utils
 from artifactory_cleanup.rules.base import Rule
 
 
@@ -85,44 +86,19 @@ def _aql_add_filter(self, aql_query_list):
 
 class delete_empty_folder(Rule):
     """
-    Clean up empty folders in local repositories. A special rule that runs separately on all repositories.
+    Remove empty folders.
 
-    Refers to the plugin
-    https://github.com/jfrog/artifactory-user-plugins/tree/master/cleanup/deleteEmptyDirs
+    If you just want to clean up empty folders - Artifactory must do it automatically.
+    We use the rule to help with some specific cases - look at README.md "FAQ: How to clean up Conan repository"
     """
 
     def _aql_add_filter(self, aql_query_list):
-        update_dict = {
-            "repo": {
-                "$match": "deleteEmptyFolder",
-            }
-        }
-        aql_query_list.append(update_dict)
+        # Get list of all files and folders
+        all_files_dict = {"path": {"$match": "**"}, "type": {"$eq": "any"}}
+        aql_query_list.append(all_files_dict)
         return aql_query_list
 
-    def _filter_result(self, result_artifact):
-        r = self.artifactory_session.get(
-            "{}/api/repositories?type=local".format(self.artifactory_server)
-        )
-        r.raise_for_status()
-        repositories = r.json()
-
-        for count, repository in enumerate(repositories, start=1):
-            if repository["packageType"] == "GitLfs":
-                # GitLfs should be handled by the jfrog cli: https://jfrog.com/blog/clean-up-your-git-lfs-repositories-with-jfrog-cli/
-                print(
-                    f"Skipping '{repository['key']}' because it is a Git LFS repository"
-                )
-                continue
-
-            url = "{}/api/plugins/execute/deleteEmptyDirsPlugin?params=paths={}".format(
-                self.artifactory_server, repository["key"]
-            )
-
-            print(
-                f"Deleting empty folders for '{repository['key']}' - {count} of {len(repositories)}"
-            )
-            r = self.artifactory_session.post(url)
-            r.raise_for_status()
-
-        return []
+    def _filter_result(self, result_artifacts):
+        repositories = utils.build_repositories(result_artifacts)
+        folders = utils.get_empty_folders(repositories)
+        return folders