[Backport 2.x] Add CI bundle pattern to distribution download (#5573)

* Add CI bundle pattern to distribution download (#5348) * Add CI bundle pattern for ivy repo Signed-off-by: Zelin Hao <zelinhao@amazon.com> * Gradle update Signed-off-by: Zelin Hao <zelinhao@amazon.com> * Extract path Signed-off-by: Zelin Hao <zelinhao@amazon.com> * Change with customDistributionDownloadType Signed-off-by: Zelin Hao <zelinhao@amazon.com> * Add default for exception handle Signed-off-by: Zelin Hao <zelinhao@amazon.com> * Add documentations Signed-off-by: Zelin Hao <zelinhao@amazon.com> Signed-off-by: Zelin Hao <zelinhao@amazon.com> (cherry picked from commit fe8fd67) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * Reorder logics to fix tests Signed-off-by: Zelin Hao <zelinhao@amazon.com> Signed-off-by: Zelin Hao <zelinhao@amazon.com> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Zelin Hao <zelinhao@amazon.com>
opensearch-project · Dec 19, 2022 · 6ac361f · 6ac361f
1 parent 994802a
commit 6ac361f
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Adding support to register settings dynamically ([#5495](https://github.com/opensearch-project/OpenSearch/pull/5495))
 - Adding auto release workflow ([#5582](https://github.com/opensearch-project/OpenSearch/pull/5582))
 - Added experimental support for extensions ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347)), ([#5518](https://github.com/opensearch-project/OpenSearch/pull/5518))
+- Add CI bundle pattern to distribution download ([#5348](https://github.com/opensearch-project/OpenSearch/pull/5348))
 
 
 ### Dependencies

diff --git a/TESTING.md b/TESTING.md
@@ -383,6 +383,10 @@ Use -Dtest.class and -Dtests.method to run a specific bwcTest test. For example
      -Dtests.class=org.opensearch.upgrades.RecoveryIT \
      -Dtests.method=testHistoryUUIDIsGenerated
 
+Use `-PcustomDistributionDownloadType=bundle` to run the bwcTest against the test cluster with latest CI distribution bundle set up for the specified version; this property is default to min and exclusive choices between `bundle` and `min`:
+
+    ./gradlew bwcTest -PcustomDistributionDownloadType=bundle
+
 When running `./gradlew check`, minimal bwc checks are also run against compatible versions that are not yet released.
 
 ## BWC Testing against a specific remote/branch

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java
@@ -80,6 +80,8 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     private static final String RELEASE_PATTERN_LAYOUT = "/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]";
     private static final String SNAPSHOT_PATTERN_LAYOUT =
         "/snapshots/core/opensearch/[revision]/[module]-min-[revision](-[classifier])-latest.[ext]";
+    private static final String BUNDLE_PATTERN_LAYOUT =
+        "/ci/dbc/distribution-build-opensearch/[revision]/latest/linux/x64/tar/dist/opensearch/[module]-[revision](-[classifier]).[ext]";
 
     private NamedDomainObjectContainer<OpenSearchDistribution> distributionsContainer;
     private NamedDomainObjectContainer<DistributionResolution> distributionsResolutionStrategiesContainer;
@@ -196,24 +198,45 @@ private static void setupDownloadServiceRepo(Project project) {
             return;
         }
         Object customDistributionUrl = project.findProperty("customDistributionUrl");
-        // checks if custom Distribution Url has been passed by user from plugins
+        Object customDistributionDownloadType = project.findProperty("customDistributionDownloadType");
+        // distributionDownloadType is default min if is not specified; download the distribution from CI if is bundle
+        String distributionDownloadType = customDistributionDownloadType != null
+            && customDistributionDownloadType.toString().equals("bundle") ? "bundle" : "min";
+
+        addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
+        addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
+
         if (customDistributionUrl != null) {
             addIvyRepo(project, DOWNLOAD_REPO_NAME, customDistributionUrl.toString(), FAKE_IVY_GROUP, "");
             addIvyRepo(project, SNAPSHOT_REPO_NAME, customDistributionUrl.toString(), FAKE_SNAPSHOT_IVY_GROUP, "");
-        } else {
-            addIvyRepo(
-                project,
-                DOWNLOAD_REPO_NAME,
-                "https://artifacts.opensearch.org",
-                FAKE_IVY_GROUP,
-                "/releases" + RELEASE_PATTERN_LAYOUT,
-                "/release-candidates" + RELEASE_PATTERN_LAYOUT
-            );
-            addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
+            return;
+        }
+        switch (distributionDownloadType) {
+            case "bundle":
+                addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://ci.opensearch.org", FAKE_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
+                addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://ci.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
+                break;
+            case "min":
+                addIvyRepo(
+                    project,
+                    DOWNLOAD_REPO_NAME,
+                    "https://artifacts.opensearch.org",
+                    FAKE_IVY_GROUP,
+                    "/releases" + RELEASE_PATTERN_LAYOUT,
+                    "/release-candidates" + RELEASE_PATTERN_LAYOUT
+                );
+                addIvyRepo(
+                    project,
+                    SNAPSHOT_REPO_NAME,
+                    "https://artifacts.opensearch.org",
+                    FAKE_SNAPSHOT_IVY_GROUP,
+                    SNAPSHOT_PATTERN_LAYOUT
+                );
+                break;
+            default:
+                throw new IllegalArgumentException("Unsupported property argument: " + distributionDownloadType);
         }
 
-        addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
-        addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
     }
 
     /**