[1.3] Remove zips and download from remote at build time (#337) (#422)

* Remove zips and download from remote at build time (#337)

Signed-off-by: Joshua Li <joshuali925@gmail.com>

* Remove odfe bwc test (#408)

Signed-off-by: Shenoy Pratik <sgguruda@amazon.com>

Signed-off-by: Joshua Li <joshuali925@gmail.com>
Signed-off-by: Shenoy Pratik <sgguruda@amazon.com>
Co-authored-by: Shenoy Pratik <sgguruda@amazon.com>

.github/workflows/reports-scheduler-test-and-build-workflow.yml

@@ -25,13 +25,6 @@ jobs:
       - name: Checkout Reports Scheduler
         uses: actions/checkout@v2
 
-      - name: RunBackwards Compatibility Tests
-        run: |
-          cd reports-scheduler
-          echo "Running backwards compatibility tests ..."
-          ./gradlew bwcTestSuite
-
-
       - name: Build with Gradle
         run: |
           cd reports-scheduler
@@ -54,4 +47,3 @@ jobs:
         with:
           name: reports-scheduler
           path: reports-scheduler-builds
-

reports-scheduler/build.gradle

@@ -240,6 +240,8 @@ Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
 integTest.dependsOn(bundle)
 integTest.getClusters().forEach{c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))}
 
+String jobSchedulerURL = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_version.replace("-SNAPSHOT", "") + "/latest/linux/x64/builds/opensearch/plugins/opensearch-job-scheduler-" + opensearch_build.replace("-SNAPSHOT", "") + ".zip"
+
 testClusters.integTest {
     testDistribution = "INTEG_TEST"
     // need to install job-scheduler first, need to assemble job-scheduler first
@@ -249,6 +251,14 @@ testClusters.integTest {
             return new RegularFile() {
                 @Override
                 File getAsFile() {
+                    File dir = new File("src/test/resources/job-scheduler")
+                    if (!dir.exists()) {
+                        dir.mkdirs()
+                    }
+                    File file = new File(dir, "opensearch-job-scheduler-" + opensearch_build + ".zip")
+                    if (!file.exists()) {
+                        new URL(jobSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+                    }
                     return fileTree("src/test/resources/job-scheduler").getSingleFile()
                 }
             }
@@ -274,19 +284,29 @@ testClusters.integTest {
 String bwcVersion = "1.13.0.0"
 String baseName = "reportsSchedulerBwcCluster"
 String bwcFilePath = "src/test/resources/bwc"
+String bwcJobSchedulerURL = "https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-job-scheduler/opendistro-job-scheduler-" + bwcVersion + ".zip"
+String bwcReportsSchedulerURL = "https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-reports-scheduler/opendistro-reports-scheduler-" + bwcVersion + ".zip"
 
 2.times {i ->
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = ["7.10.2","1.3.5-SNAPSHOT"]
+            versions = ["7.10.2", opensearch_version]
             numberOfNodes = 3
             plugin(provider(new Callable<RegularFile>(){
                 @Override
                 RegularFile call() throws Exception {
                     return new RegularFile() {
                         @Override
                         File getAsFile() {
+                            File dir = new File(bwcFilePath + "/job-scheduler/" + bwcVersion)
+                            if (!dir.exists()) {
+                                dir.mkdirs()
+                            }
+                            File file = new File(dir, "opendistro-job-scheduler-" + bwcVersion + ".zip")
+                            if (!file.exists()) {
+                                new URL(bwcJobSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+                            }
                             return fileTree(bwcFilePath + "/job-scheduler/" + bwcVersion).getSingleFile()
                         }
                     }
@@ -298,6 +318,14 @@ String bwcFilePath = "src/test/resources/bwc"
                     return new RegularFile() {
                         @Override
                         File getAsFile() {
+                            File dir = new File(bwcFilePath + "/reports-scheduler/" + bwcVersion)
+                            if (!dir.exists()) {
+                                dir.mkdirs()
+                            }
+                            File file = new File(dir, "opendistro-reports-scheduler-" + bwcVersion + ".zip")
+                            if (!file.exists()) {
+                                new URL(bwcReportsSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+                            }
                             return fileTree(bwcFilePath + "/reports-scheduler/" + bwcVersion).getSingleFile()
                         }
                     }
@@ -322,6 +350,14 @@ task prepareBwcTests {
                         return new RegularFile() {
                             @Override
                             File getAsFile() {
+                                File dir = new File(bwcFilePath + "/job-scheduler/" + project.version)
+                                if (!dir.exists()) {
+                                    dir.mkdirs()
+                                }
+                                File file = new File(dir, "opendistro-reports-scheduler-" + project.version + ".zip")
+                                if (!file.exists()) {
+                                    new URL(jobSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+                                }
                                 return fileTree(bwcFilePath + "/job-scheduler/" + project.version).getSingleFile()
                             }
                         }

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/TestHelpers.kt

@@ -93,4 +93,4 @@ internal fun createReportDefinitionDetails(): ReportDefinitionDetails {
         listOf(),
         reportDefinition = createReportDefinitionObject()
     )
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/CreateReportDefinitionRequestTests.kt

@@ -98,4 +98,4 @@ internal class CreateReportDefinitionRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { CreateReportDefinitionRequest(it) }
         Assertions.assertEquals(reportDefinitionObject, recreatedObject.reportDefinition)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/CreateReportDefinitionResponseTests.kt

@@ -68,4 +68,4 @@ internal class CreateReportDefinitionResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { CreateReportDefinitionResponse.parse(it) }
         Assertions.assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/DeleteReportDefinitionRequestTests.kt

@@ -56,4 +56,4 @@ internal class DeleteReportDefinitionRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { DeleteReportDefinitionRequest.parse(it) }
         assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/DeleteReportDefinitionResponseTests.kt

@@ -64,4 +64,4 @@ internal class DeleteReportDefinitionResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { DeleteReportDefinitionResponse.parse(it) }
         assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportDefinitionsRequestTests.kt

@@ -69,4 +69,4 @@ internal class GetAllReportDefinitionsRequestTests {
         assertEquals(0, recreatedObject.fromIndex)
         assertEquals(defaultMaxItems, recreatedObject.maxItems)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportDefinitionsResponseTests.kt

@@ -132,4 +132,4 @@ internal class GetAllReportDefinitionsResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetAllReportDefinitionsResponse(it) }
         Assertions.assertEquals(reportDefinitionDetailsSearchResults.objectList, recreatedObject.reportDefinitionList.objectList)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportInstancesRequestTests.kt

@@ -68,4 +68,4 @@ internal class GetAllReportInstancesRequestTests {
         assertEquals(0, recreatedObject.fromIndex)
         assertEquals(defaultMaxItems, recreatedObject.maxItems)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportInstancesResponseTests.kt

@@ -103,4 +103,4 @@ internal class GetAllReportInstancesResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetAllReportInstancesResponse(it) }
         Assertions.assertEquals(reportInstanceSearchResults.objectList, recreatedObject.reportInstanceList.objectList)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportDefinitionRequestTests.kt

@@ -67,4 +67,4 @@ internal class GetReportDefinitionRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetReportDefinitionRequest.parse(it) }
         assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportDefinitionResponseTests.kt

@@ -111,4 +111,4 @@ internal class GetReportDefinitionResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetReportDefinitionResponse(it) }
         assertEquals(reportDefinitionDetails, recreatedObject.reportDefinitionDetails)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportInstanceRequestTests.kt

@@ -67,4 +67,4 @@ internal class GetReportInstanceRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetReportInstanceRequest.parse(it) }
         Assertions.assertEquals(reportInstanceId, recreatedObject.reportInstanceId)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportInstanceResponseTests.kt

@@ -84,4 +84,4 @@ internal class GetReportInstanceResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetReportInstanceResponse(it) }
         Assertions.assertEquals(reportInstance, recreatedObject.reportInstance)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/InContextReportCreateRequestTests.kt

@@ -80,4 +80,4 @@ internal class InContextReportCreateRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { InContextReportCreateRequest(it) }
         verify(request, recreatedObject)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/InContextReportCreateResponseTests.kt

@@ -84,4 +84,4 @@ internal class InContextReportCreateResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { InContextReportCreateResponse(it) }
         Assertions.assertEquals(reportInstance, recreatedObject.reportInstance)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/OnDemandReportCreateRequestTests.kt

@@ -56,4 +56,4 @@ internal class OnDemandReportCreateRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { OnDemandReportCreateRequest.parse(it) }
         Assertions.assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/OnDemandReportCreateResponseTests.kt

@@ -83,4 +83,4 @@ internal class OnDemandReportCreateResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { OnDemandReportCreateResponse(it) }
         Assertions.assertEquals(reportInstance, recreatedObject.reportInstance)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportDefinitionRequestTests.kt

@@ -100,4 +100,4 @@ internal class UpdateReportDefinitionRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { UpdateReportDefinitionRequest(it) }
         Assertions.assertEquals(reportDefinitionRequest.reportDefinition, recreatedObject.reportDefinition)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportDefinitionResponseTests.kt

@@ -66,4 +66,4 @@ internal class UpdateReportDefinitionResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { UpdateReportDefinitionResponse.parse(it) }
         Assertions.assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportInstanceStatusRequestTests.kt

@@ -68,4 +68,4 @@ internal class UpdateReportInstanceStatusRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { UpdateReportInstanceStatusRequest.parse(it) }
         verify(request, recreatedObject)
     }
-}
+}

reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportInstanceStatusResponseTests.kt

@@ -67,4 +67,4 @@ internal class UpdateReportInstanceStatusResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { UpdateReportInstanceStatusResponse.parse(it) }
         Assertions.assertEquals(reportInstanceId, recreatedObject.reportInstanceId)
     }
-}
+}

reports-scheduler/src/test/resources/bwc/job-scheduler/.gitignore

@@ -0,0 +1,4 @@
+# ignore artifacts in this directory
+*
+*/
+!.gitignore

reports-scheduler/src/test/resources/bwc/reports-scheduler/.gitignore

@@ -0,0 +1,4 @@
+# ignore artifacts in this directory
+*
+*/
+!.gitignore

reports-scheduler/src/test/resources/job-scheduler/.gitignore

@@ -0,0 +1,4 @@
+# ignore artifacts in this directory
+*
+*/
+!.gitignore