Add an integration test for HTTP inputSource (#10620)

apache · Dec 3, 2020 · ae6c43d · ae6c43d
1 parent 2cd017b
commit ae6c43d
Show file tree

Hide file tree

Showing 3 changed files with 241 additions and 0 deletions.
diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHttpInputSourceTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHttpInputSourceTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.tests.indexer;
+
+import org.apache.druid.testing.guice.DruidTestModuleFactory;
+import org.apache.druid.tests.TestNGGroup;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.UUID;
+
+@Test(groups = TestNGGroup.INPUT_SOURCE)
+@Guice(moduleFactory = DruidTestModuleFactory.class)
+public class ITHttpInputSourceTest extends AbstractITBatchIndexTest
+{
+  private static final String INDEX_TASK = "/indexer/wikipedia_http_inputsource_task.json";
+  private static final String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_http_inputsource_queries.json";
+
+  @Test
+  public void doTest() throws IOException
+  {
+    final String indexDatasource = "wikipedia_http_inputsource_test_" + UUID.randomUUID();
+    try (final Closeable ignored1 = unloader(indexDatasource + config.getExtraDatasourceNameSuffix())) {
+      doIndexTest(
+          indexDatasource,
+          INDEX_TASK,
+          INDEX_QUERIES_RESOURCE,
+          false,
+          true,
+          true
+      );
+    }
+  }
+}
diff --git a/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_queries.json b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_queries.json
@@ -0,0 +1,98 @@
+[
+    {
+        "description": "timeseries, 1 agg, all",
+        "query":{
+            "queryType" : "timeBoundary",
+            "dataSource": "%%DATASOURCE%%"
+        },
+        "expectedResults":[
+            {
+                "timestamp" : "2016-06-27T00:00:11.000Z",
+                "result" : {
+                    "minTime" : "2016-06-27T00:00:11.000Z",
+                    "maxTime" : "2016-06-27T21:31:02.000Z"
+                }
+            }
+        ]
+    },
+    {
+        "description": "timeseries, datasketch aggs, all",
+        "query":{
+            "queryType" : "timeseries",
+            "dataSource": "%%DATASOURCE%%",
+            "granularity":"day",
+            "intervals":[
+                "2016-06-27/P1D"
+            ],
+            "filter":null,
+            "aggregations":[
+                {
+                    "type": "HLLSketchMerge",
+                    "name": "approxCountHLL",
+                    "fieldName": "HLLSketchBuild",
+                    "lgK": 12,
+                    "tgtHllType": "HLL_4",
+                    "round": true
+                },
+                {
+                    "type":"thetaSketch",
+                    "name":"approxCountTheta",
+                    "fieldName":"thetaSketch",
+                    "size":16384,
+                    "shouldFinalize":true,
+                    "isInputThetaSketch":false,
+                    "errorBoundsStdDev":null
+                },
+                {
+                    "type":"quantilesDoublesSketch",
+                    "name":"quantilesSketch",
+                    "fieldName":"quantilesDoublesSketch",
+                    "k":128
+                }
+            ]
+        },
+        "expectedResults":[
+            {
+                "timestamp" : "2016-06-27T00:00:00.000Z",
+                "result" : {
+                    "quantilesSketch":48866,
+                    "approxCountTheta":7862.0,
+                    "approxCountHLL":7961
+                }
+            }
+        ]
+    },
+    {
+        "description": "timeseries, stringFirst/stringLast aggs, all",
+        "query":{
+            "queryType" : "timeseries",
+            "dataSource": "%%DATASOURCE%%",
+            "granularity":"day",
+            "intervals":[
+                "2016-06-27/P1D"
+            ],
+            "filter":null,
+            "aggregations":[
+                {
+                    "type": "stringFirst",
+                    "name": "first_user",
+                    "fieldName": "user"
+                },
+                {
+                    "type":"stringLast",
+                    "name":"last_user",
+                    "fieldName":"user"
+                }
+            ]
+        },
+        "expectedResults":[
+            {
+                "timestamp" : "2016-06-27T00:00:00.000Z",
+                "result" : {
+                    "first_user":"Lsjbot",
+                    "last_user":"EmausBot"
+                }
+            }
+        ]
+    }
+]
diff --git a/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_task.json b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_task.json
@@ -0,0 +1,90 @@
+{
+  "type": "index_parallel",
+  "spec": {
+    "dataSchema": {
+      "dataSource": "%%DATASOURCE%%",
+      "timestampSpec": {
+        "column": "timestamp"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          "page",
+          {"type": "string", "name": "language", "createBitmapIndex": false},
+          "user",
+          "unpatrolled",
+          "newPage",
+          "robot",
+          "anonymous",
+          "namespace",
+          "continent",
+          "country",
+          "region",
+          "city"
+        ]
+      },
+      "metricsSpec": [
+        {
+          "type": "count",
+          "name": "count"
+        },
+        {
+          "type": "doubleSum",
+          "name": "added",
+          "fieldName": "added"
+        },
+        {
+          "type": "doubleSum",
+          "name": "deleted",
+          "fieldName": "deleted"
+        },
+        {
+          "type": "doubleSum",
+          "name": "delta",
+          "fieldName": "delta"
+        },
+        {
+          "name": "thetaSketch",
+          "type": "thetaSketch",
+          "fieldName": "user"
+        },
+        {
+          "name": "quantilesDoublesSketch",
+          "type": "quantilesDoublesSketch",
+          "fieldName": "delta"
+        },
+        {
+          "name": "HLLSketchBuild",
+          "type": "HLLSketchBuild",
+          "fieldName": "user"
+        }
+      ],
+      "granularitySpec": {
+        "segmentGranularity": "DAY",
+        "queryGranularity": "second",
+        "intervals" : [ "2016-06/P1M" ]
+      }
+    },
+    "ioConfig": {
+      "type": "index_parallel",
+      "inputSource": {
+        "type": "http",
+        "uris": ["https://druid.apache.org/data/wikipedia.json.gz", "https://druid.apache.org/data/wikipedia.json.gz"]
+      },
+      "inputFormat": {
+        "type": "json"
+      }
+    },
+    "tuningConfig": {
+      "type": "index_parallel",
+      "maxNumConcurrentSubTasks": 10,
+      "partitionsSpec": {
+        "type": "hashed"
+      },
+      "forceGuaranteedRollup": true,
+      "splitHintSpec": {
+        "type": "maxSize",
+        "maxNumFiles": 1
+      }
+    }
+  }
+}