From 97856478ec4f952486786633d10e02d0682a0f94 Mon Sep 17 00:00:00 2001 From: Joe McDonnell Date: Tue, 6 Oct 2020 11:03:33 -0700 Subject: [PATCH] IMPALA-10198 (part 1): Unify Java in a single java/ directory This changes all existing Java code to be submodules under a single root pom. The root pom is impala-parent/pom.xml with minor changes to add submodules. This avoids most of the weird CMake/maven interactions, because there is now a single maven invocation for all the Java code. This moves all the Java projects other than fe into a top level java directory. fe is left where it is to avoid disruption (but still is compiled via the java directory's root pom). Various pieces of code that reference the old locations are updated. Based on research, there are two options for dealing with the shaded dependencies. The first is to have an entirely separate Maven project with a separate Maven invocation. In this case, the consumers of the shaded jars will see the reduced set of transitive dependencies. The second is to have the shaded dependencies as modules with a single Maven invocation. The consumer would see all of the original transitive dependencies and need to exclude them all. See MSHADE-206/MNG-5899. This chooses the second. This only moves code around and does not focus on version numbers or making "mvn versions:set" work. Testing: - Ran a core job - Verified existing maven commands from fe/ directory still work - Compared the *-classpath.txt files from fe and executor-deps and verified they are the same except for paths Change-Id: I08773f4f9d7cb269b0491080078d6e6f490d8d7a Reviewed-on: http://gerrit.cloudera.org:8080/16500 Reviewed-by: Joe McDonnell Tested-by: Joe McDonnell --- CMakeLists.txt | 9 +------- bin/clean-cmake.sh | 4 ++-- bin/clean.sh | 2 +- bin/create_testdata.sh | 6 +++-- buildall.sh | 4 ++-- common/thrift/CMakeLists.txt | 2 +- common/yarn-extras/CMakeLists.txt | 23 ------------------- docker/CMakeLists.txt | 2 +- docker/setup_build_context.py | 2 +- ext-data-source/CMakeLists.txt | 23 ------------------- fe/CMakeLists.txt | 22 ------------------ fe/pom.xml | 2 +- {impala-parent => java}/CMakeLists.txt | 2 +- {testdata => java}/TableFlattener/.gitignore | 0 {testdata => java}/TableFlattener/README | 0 {testdata => java}/TableFlattener/pom.xml | 1 - .../infra/tableflattener/FileMigrator.java | 0 .../infra/tableflattener/FlattenedSchema.java | 0 .../impala/infra/tableflattener/Main.java | 0 .../infra/tableflattener/SchemaFlattener.java | 0 .../infra/tableflattener/SchemaUtil.java | 0 {testdata => java/datagenerator}/pom.xml | 1 - .../datagenerator/CsvToHBaseConverter.java | 0 .../datagenerator/JsonToParquetConverter.java | 0 .../RandomNestedDataGenerator.java | 0 .../datagenerator/TestDataGenerator.java | 0 {mvn-deps => java}/executor-deps/pom.xml | 13 ++++++++++- .../ext-data-source}/.gitignore | 0 .../ext-data-source}/api/pom.xml | 0 .../util/SerializationUtils.java | 0 .../extdatasource/v1/ExternalDataSource.java | 0 .../ext-data-source}/pom.xml | 1 - .../ext-data-source}/sample/pom.xml | 0 .../extdatasource/sample/EchoDataSource.java | 0 .../ext-data-source}/test/pom.xml | 0 .../extdatasource/AllTypesDataSource.java | 0 {impala-parent => java}/pom.xml | 13 +++++++++++ .../query-event-hook-api}/pom.xml | 1 - .../impala/hooks/QueryCompleteContext.java | 0 .../apache/impala/hooks/QueryEventHook.java | 0 .../shaded-deps/hive-exec/pom.xml | 2 +- .../shaded-deps/s3a-aws-sdk/pom.xml | 2 +- {tests => java}/test-hive-udfs/pom.xml | 1 - .../impala/ImportsNearbyClassesUdf.java | 0 .../apache/impala/IncompatibleUdfTest.java | 0 .../java/org/apache/impala/JavaGcUdfTest.java | 0 .../java/org/apache/impala/JavaUdfTest.java | 0 .../org/apache/impala/ReplaceStringUdf.java | 0 .../main/java/org/apache/impala/TestUdf.java | 0 .../org/apache/impala/TestUdfException.java | 0 .../java/org/apache/impala/TestUpdateUdf.java | 0 .../java/org/apache/impala/UnresolvedUdf.java | 0 .../java/org/apache/impala/UtilForUdf.java | 0 {common => java}/yarn-extras/.gitignore | 0 {common => java}/yarn-extras/README.txt | 0 {common => java}/yarn-extras/pom.xml | 1 - .../resource/ResourceWeights.java | 0 .../fair/AllocationConfiguration.java | 0 .../AllocationConfigurationException.java | 0 .../fair/AllocationFileLoaderService.java | 0 .../scheduler/fair/FSQueueType.java | 0 .../fair/FairSchedulerConfiguration.java | 0 .../scheduler/fair/QueuePlacementPolicy.java | 0 .../scheduler/fair/QueuePlacementRule.java | 0 .../scheduler/fair/SchedulingPolicy.java | 0 .../yarn/server/utils/BuilderUtils.java | 0 mvn-deps/executor-deps/CMakeLists.txt | 20 ---------------- mvn-deps/shaded-deps/hive-exec/CMakeLists.txt | 20 ---------------- .../shaded-deps/s3a-aws-sdk/CMakeLists.txt | 20 ---------------- query-event-hook-api/CMakeLists.txt | 22 ------------------ testdata/bin/copy-data-sources.sh | 2 +- testdata/bin/copy-udfs-udas.sh | 8 +++++-- testdata/bin/create-load-data.sh | 2 +- testdata/bin/generate-load-nested.sh | 6 ++--- tests/custom_cluster/test_coordinators.py | 2 +- tests/query_test/test_udfs.py | 4 ++-- tests/run-tests.py | 2 +- 77 files changed, 57 insertions(+), 190 deletions(-) delete mode 100644 common/yarn-extras/CMakeLists.txt delete mode 100644 ext-data-source/CMakeLists.txt delete mode 100644 fe/CMakeLists.txt rename {impala-parent => java}/CMakeLists.txt (93%) rename {testdata => java}/TableFlattener/.gitignore (100%) rename {testdata => java}/TableFlattener/README (100%) rename {testdata => java}/TableFlattener/pom.xml (97%) rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java (100%) rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java (100%) rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java (100%) rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java (100%) rename {testdata => java}/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java (100%) rename {testdata => java/datagenerator}/pom.xml (98%) rename {testdata => java/datagenerator}/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java (100%) rename {testdata => java/datagenerator}/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java (100%) rename {testdata => java/datagenerator}/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java (100%) rename {testdata => java/datagenerator}/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java (100%) rename {mvn-deps => java}/executor-deps/pom.xml (96%) rename {ext-data-source => java/ext-data-source}/.gitignore (100%) rename {ext-data-source => java/ext-data-source}/api/pom.xml (100%) rename {ext-data-source => java/ext-data-source}/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java (100%) rename {ext-data-source => java/ext-data-source}/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java (100%) rename {ext-data-source => java/ext-data-source}/pom.xml (96%) rename {ext-data-source => java/ext-data-source}/sample/pom.xml (100%) rename {ext-data-source => java/ext-data-source}/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java (100%) rename {ext-data-source => java/ext-data-source}/test/pom.xml (100%) rename {ext-data-source => java/ext-data-source}/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java (100%) rename {impala-parent => java}/pom.xml (96%) rename {query-event-hook-api => java/query-event-hook-api}/pom.xml (97%) rename {query-event-hook-api => java/query-event-hook-api}/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java (100%) rename {query-event-hook-api => java/query-event-hook-api}/src/main/java/org/apache/impala/hooks/QueryEventHook.java (100%) rename {mvn-deps => java}/shaded-deps/hive-exec/pom.xml (98%) rename {mvn-deps => java}/shaded-deps/s3a-aws-sdk/pom.xml (99%) rename {tests => java}/test-hive-udfs/pom.xml (98%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java (100%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java (100%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java (100%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java (100%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java (100%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java (100%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java (100%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java (100%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java (100%) rename {tests => java}/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java (100%) rename {common => java}/yarn-extras/.gitignore (100%) rename {common => java}/yarn-extras/README.txt (100%) rename {common => java}/yarn-extras/pom.xml (98%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java (100%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java (100%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java (100%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java (100%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java (100%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (100%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java (100%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java (100%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java (100%) rename {common => java}/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java (100%) delete mode 100644 mvn-deps/executor-deps/CMakeLists.txt delete mode 100644 mvn-deps/shaded-deps/hive-exec/CMakeLists.txt delete mode 100644 mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt delete mode 100644 query-event-hook-api/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f88a68dabd..ddfc26b1b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -425,17 +425,10 @@ endif() add_subdirectory(common/function-registry) add_subdirectory(common/thrift) add_subdirectory(common/fbs) -add_subdirectory(common/yarn-extras) add_subdirectory(common/protobuf) add_subdirectory(be) add_subdirectory(docker) -add_subdirectory(mvn-deps/shaded-deps/hive-exec) -add_subdirectory(mvn-deps/shaded-deps/s3a-aws-sdk) -add_subdirectory(mvn-deps/executor-deps) -add_subdirectory(fe) -add_subdirectory(impala-parent) -add_subdirectory(ext-data-source) -add_subdirectory(query-event-hook-api) +add_subdirectory(java) # Build target for all generated files which most backend code depends on add_custom_target(gen-deps ALL DEPENDS thrift-deps proto-deps fb-deps diff --git a/bin/clean-cmake.sh b/bin/clean-cmake.sh index aa1ea96aac..bd9cc77406 100755 --- a/bin/clean-cmake.sh +++ b/bin/clean-cmake.sh @@ -29,8 +29,8 @@ if [[ -z "${IMPALA_HOME}" || ! -d "${IMPALA_HOME}" ]]; then fi # Remove trailing / ROOT_DIR=${IMPALA_HOME%%/} -for loc in "${ROOT_DIR}/ -maxdepth 1" "${ROOT_DIR}/be/" "${ROOT_DIR}/fe/" "${ROOT_DIR}/common/"\ - "${ROOT_DIR}/ext-data-source/"; do +for loc in "${ROOT_DIR}/ -maxdepth 1" "${ROOT_DIR}/be/" "${ROOT_DIR}/java/" \ + "${ROOT_DIR}/common/" ; do find ${loc} \( -iname CMakeCache.txt -o -iname CMakeFiles \ -o -iname CTestTestfile.cmake -o -iname cmake_install.cmake \) -exec rm -Rf {} + done diff --git a/bin/clean.sh b/bin/clean.sh index 34781f97f0..259648a232 100755 --- a/bin/clean.sh +++ b/bin/clean.sh @@ -30,7 +30,7 @@ setup_report_build_error "${MAKE_CMD:-make}" clean || : # clean the external data source project -pushd "${IMPALA_HOME}/ext-data-source" +pushd "${IMPALA_HOME}/java/ext-data-source" rm -rf api/generated-sources/* ${IMPALA_HOME}/bin/mvn-quiet.sh clean popd diff --git a/bin/create_testdata.sh b/bin/create_testdata.sh index 5b81d9ec5f..9c703621a9 100755 --- a/bin/create_testdata.sh +++ b/bin/create_testdata.sh @@ -29,13 +29,13 @@ bin=`cd "$bin"; pwd` DATALOC=$IMPALA_HOME/testdata/target # regenerate the test data generator -cd $IMPALA_HOME/testdata +cd $IMPALA_HOME/java/datagenerator ${IMPALA_HOME}/bin/mvn-quiet.sh clean ${IMPALA_HOME}/bin/mvn-quiet.sh package # find jars CP="" -JARS=`find target/*.jar 2> /dev/null || true` +JARS=`find ${IMPALA_HOME}/java/datagenerator/target/*.jar 2> /dev/null || true` for i in $JARS; do if [ -n "$CP" ]; then CP=${CP}:${i} @@ -47,6 +47,8 @@ done # run test data generator echo $DATALOC mkdir -p $DATALOC +pushd $IMPALA_HOME/testdata "$JAVA" -cp $CP org.apache.impala.datagenerator.TestDataGenerator $DATALOC "$JAVA" -cp $CP org.apache.impala.datagenerator.CsvToHBaseConverter echo "SUCCESS, data generated into $DATALOC" +popd diff --git a/buildall.sh b/buildall.sh index 1440e324f5..2a7eade581 100755 --- a/buildall.sh +++ b/buildall.sh @@ -424,7 +424,7 @@ bootstrap_dependencies() { # Build the Impala frontend and its dependencies. build_fe() { generate_cmake_files $CMAKE_BUILD_TYPE - ${MAKE_CMD} ${IMPALA_MAKE_FLAGS} fe + ${MAKE_CMD} ${IMPALA_MAKE_FLAGS} java } # Build all components. The build type is specified as the first argument, and the @@ -449,7 +449,7 @@ build_all_components() { MAKE_TARGETS="impalad statestored catalogd fesupport loggingsupport ImpalaUdf \ udasample udfsample" if (( build_independent_targets )); then - MAKE_TARGETS+=" cscope fe tarballs" + MAKE_TARGETS+=" cscope java tarballs" fi fi ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} ${IMPALA_MAKE_FLAGS} ${MAKE_TARGETS} diff --git a/common/thrift/CMakeLists.txt b/common/thrift/CMakeLists.txt index 9e7be7759e..9b14ab58dc 100644 --- a/common/thrift/CMakeLists.txt +++ b/common/thrift/CMakeLists.txt @@ -177,7 +177,7 @@ set(THRIFT_INCLUDE_DIR_OPTION -I ${THRIFT_CONTRIB_DIR} -I $ENV{HIVE_METASTORE_TH set(BE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/be/generated-sources) set(FE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/fe/generated-sources) # TODO: avoid duplicating generated java classes -set(EXT_DS_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/ext-data-source/api/generated-sources) +set(EXT_DS_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/java/ext-data-source/api/generated-sources) set(PYTHON_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/shell/) set(THRIFT11_PYTHON_OUTPUT_DIR ${PYTHON_OUTPUT_DIR}/build/thrift-11-gen) MESSAGE("Found output dir: " ${PYTHON_OUTPUT_DIR}) diff --git a/common/yarn-extras/CMakeLists.txt b/common/yarn-extras/CMakeLists.txt deleted file mode 100644 index 4f46ba517d..0000000000 --- a/common/yarn-extras/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# The dependency on ext-data-source here is fictional, but Maven does not like -# concurrent invocations. These lead to opaque, non-deterministic errors due to -# races in how Maven handles its ~/.m2/repository doc. -add_custom_target(yarn-extras ALL DEPENDS impala-parent ext-data-source - COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests -) diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt index 44f132937a..dfb6a336d6 100644 --- a/docker/CMakeLists.txt +++ b/docker/CMakeLists.txt @@ -48,7 +48,7 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED") # with symlinks add_custom_target(impala_base_build_context_${build_type} COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args} - DEPENDS daemons fe ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py + DEPENDS daemons java ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py COMMENT "Creating impala base build context build_type=${build_type}." VERBATIM ) diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py index f359328788..f41a75744d 100755 --- a/docker/setup_build_context.py +++ b/docker/setup_build_context.py @@ -137,7 +137,7 @@ def strip_debug_symbols(src_file, dst_dirs): # Impala Executor dependencies. dep_classpath = file(os.path.join(IMPALA_HOME, - "mvn-deps/executor-deps/target/build-executor-deps-classpath.txt")).read() + "java/executor-deps/target/build-executor-deps-classpath.txt")).read() for jar in dep_classpath.split(":"): assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar) symlink_file_into_dir(jar, EXEC_LIB_DIR) diff --git a/ext-data-source/CMakeLists.txt b/ext-data-source/CMakeLists.txt deleted file mode 100644 index 0f88dd2e93..0000000000 --- a/ext-data-source/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# The dependency on shaded-deps is only added to avoid parallel downloads -# of dependencies. For more details see IMPALA-7051, which was a similar issue. -add_custom_target(ext-data-source ALL DEPENDS gen-deps impala-parent - shaded-deps-hive-exec shaded-deps-s3a-aws-sdk - COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests -) diff --git a/fe/CMakeLists.txt b/fe/CMakeLists.txt deleted file mode 100644 index 2d52adeccd..0000000000 --- a/fe/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -add_custom_target(fe ALL DEPENDS - executor-deps shaded-deps-hive-exec shaded-deps-s3a-aws-sdk thrift-deps fb-deps - yarn-extras function-registry ext-data-source query-event-hook-api impala-parent - COMMAND ${CMAKE_SOURCE_DIR}/bin/mvn-quiet.sh -B install -DskipTests -) diff --git a/fe/pom.xml b/fe/pom.xml index 3f81c5d04b..d84e16d91d 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.impala impala-parent 0.1-SNAPSHOT - ../impala-parent/pom.xml + ../java/pom.xml 4.0.0 org.apache.impala diff --git a/impala-parent/CMakeLists.txt b/java/CMakeLists.txt similarity index 93% rename from impala-parent/CMakeLists.txt rename to java/CMakeLists.txt index fdd6e98183..08feb30ea0 100644 --- a/impala-parent/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -15,6 +15,6 @@ # specific language governing permissions and limitations # under the License. -add_custom_target(impala-parent ALL +add_custom_target(java ALL DEPENDS gen-deps function-registry COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests ) diff --git a/testdata/TableFlattener/.gitignore b/java/TableFlattener/.gitignore similarity index 100% rename from testdata/TableFlattener/.gitignore rename to java/TableFlattener/.gitignore diff --git a/testdata/TableFlattener/README b/java/TableFlattener/README similarity index 100% rename from testdata/TableFlattener/README rename to java/TableFlattener/README diff --git a/testdata/TableFlattener/pom.xml b/java/TableFlattener/pom.xml similarity index 97% rename from testdata/TableFlattener/pom.xml rename to java/TableFlattener/pom.xml index cb232d434d..40a404cb1f 100644 --- a/testdata/TableFlattener/pom.xml +++ b/java/TableFlattener/pom.xml @@ -23,7 +23,6 @@ org.apache.impala impala-parent 0.1-SNAPSHOT - ../../impala-parent/pom.xml 4.0.0 org.apache.impala diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java similarity index 100% rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FileMigrator.java diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java similarity index 100% rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/FlattenedSchema.java diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java similarity index 100% rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/Main.java diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java similarity index 100% rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaFlattener.java diff --git a/testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java b/java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java similarity index 100% rename from testdata/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java rename to java/TableFlattener/src/main/java/org/apache/impala/infra/tableflattener/SchemaUtil.java diff --git a/testdata/pom.xml b/java/datagenerator/pom.xml similarity index 98% rename from testdata/pom.xml rename to java/datagenerator/pom.xml index 80e0e38e60..03195da285 100644 --- a/testdata/pom.xml +++ b/java/datagenerator/pom.xml @@ -24,7 +24,6 @@ under the License. org.apache.impala impala-parent 0.1-SNAPSHOT - ../impala-parent/pom.xml 4.0.0 diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java similarity index 100% rename from testdata/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java rename to java/datagenerator/src/main/java/org/apache/impala/datagenerator/CsvToHBaseConverter.java diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java similarity index 100% rename from testdata/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java rename to java/datagenerator/src/main/java/org/apache/impala/datagenerator/JsonToParquetConverter.java diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java similarity index 100% rename from testdata/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java rename to java/datagenerator/src/main/java/org/apache/impala/datagenerator/RandomNestedDataGenerator.java diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java b/java/datagenerator/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java similarity index 100% rename from testdata/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java rename to java/datagenerator/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java diff --git a/mvn-deps/executor-deps/pom.xml b/java/executor-deps/pom.xml similarity index 96% rename from mvn-deps/executor-deps/pom.xml rename to java/executor-deps/pom.xml index 1b51fa887f..f9e8897029 100644 --- a/mvn-deps/executor-deps/pom.xml +++ b/java/executor-deps/pom.xml @@ -35,7 +35,6 @@ under the License. org.apache.impala impala-parent 0.1-SNAPSHOT - ../../impala-parent/pom.xml 4.0.0 org.apache.impala @@ -107,6 +106,12 @@ under the License. org.apache.impala impala-minimal-s3a-aws-sdk ${project.version} + + + * + * + + @@ -159,6 +164,12 @@ under the License. org.apache.impala impala-minimal-hive-exec ${project.version} + + + * + * + + diff --git a/ext-data-source/.gitignore b/java/ext-data-source/.gitignore similarity index 100% rename from ext-data-source/.gitignore rename to java/ext-data-source/.gitignore diff --git a/ext-data-source/api/pom.xml b/java/ext-data-source/api/pom.xml similarity index 100% rename from ext-data-source/api/pom.xml rename to java/ext-data-source/api/pom.xml diff --git a/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java b/java/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java similarity index 100% rename from ext-data-source/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java rename to java/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/util/SerializationUtils.java diff --git a/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java b/java/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java similarity index 100% rename from ext-data-source/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java rename to java/ext-data-source/api/src/main/java/org/apache/impala/extdatasource/v1/ExternalDataSource.java diff --git a/ext-data-source/pom.xml b/java/ext-data-source/pom.xml similarity index 96% rename from ext-data-source/pom.xml rename to java/ext-data-source/pom.xml index b4b004508d..8b4081df27 100644 --- a/ext-data-source/pom.xml +++ b/java/ext-data-source/pom.xml @@ -23,7 +23,6 @@ org.apache.impala impala-parent 0.1-SNAPSHOT - ../impala-parent/pom.xml 4.0.0 org.apache.impala diff --git a/ext-data-source/sample/pom.xml b/java/ext-data-source/sample/pom.xml similarity index 100% rename from ext-data-source/sample/pom.xml rename to java/ext-data-source/sample/pom.xml diff --git a/ext-data-source/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java b/java/ext-data-source/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java similarity index 100% rename from ext-data-source/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java rename to java/ext-data-source/sample/src/main/java/org/apache/impala/extdatasource/sample/EchoDataSource.java diff --git a/ext-data-source/test/pom.xml b/java/ext-data-source/test/pom.xml similarity index 100% rename from ext-data-source/test/pom.xml rename to java/ext-data-source/test/pom.xml diff --git a/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java b/java/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java similarity index 100% rename from ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java rename to java/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java diff --git a/impala-parent/pom.xml b/java/pom.xml similarity index 96% rename from impala-parent/pom.xml rename to java/pom.xml index 3baac4d7db..5c8649e1aa 100644 --- a/impala-parent/pom.xml +++ b/java/pom.xml @@ -219,4 +219,17 @@ under the License. + + datagenerator + executor-deps + ext-data-source + ../fe + query-event-hook-api + shaded-deps/hive-exec + shaded-deps/s3a-aws-sdk + TableFlattener + test-hive-udfs + yarn-extras + + diff --git a/query-event-hook-api/pom.xml b/java/query-event-hook-api/pom.xml similarity index 97% rename from query-event-hook-api/pom.xml rename to java/query-event-hook-api/pom.xml index 8466e0b111..4e9cb9925f 100644 --- a/query-event-hook-api/pom.xml +++ b/java/query-event-hook-api/pom.xml @@ -23,7 +23,6 @@ org.apache.impala impala-parent 0.1-SNAPSHOT - ../impala-parent/pom.xml 4.0.0 org.apache.impala diff --git a/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java b/java/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java similarity index 100% rename from query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java rename to java/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryCompleteContext.java diff --git a/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryEventHook.java b/java/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryEventHook.java similarity index 100% rename from query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryEventHook.java rename to java/query-event-hook-api/src/main/java/org/apache/impala/hooks/QueryEventHook.java diff --git a/mvn-deps/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml similarity index 98% rename from mvn-deps/shaded-deps/hive-exec/pom.xml rename to java/shaded-deps/hive-exec/pom.xml index 33cb897382..ea28a6b60a 100644 --- a/mvn-deps/shaded-deps/hive-exec/pom.xml +++ b/java/shaded-deps/hive-exec/pom.xml @@ -28,7 +28,7 @@ the same dependencies org.apache.impala impala-parent 0.1-SNAPSHOT - ../../impala-parent/pom.xml + ../../pom.xml 4.0.0 org.apache.impala diff --git a/mvn-deps/shaded-deps/s3a-aws-sdk/pom.xml b/java/shaded-deps/s3a-aws-sdk/pom.xml similarity index 99% rename from mvn-deps/shaded-deps/s3a-aws-sdk/pom.xml rename to java/shaded-deps/s3a-aws-sdk/pom.xml index 392ea10092..aebf3d8644 100644 --- a/mvn-deps/shaded-deps/s3a-aws-sdk/pom.xml +++ b/java/shaded-deps/s3a-aws-sdk/pom.xml @@ -26,7 +26,7 @@ though some of them might not be necessary. The exclusions are sorted alphabetic org.apache.impala impala-parent 0.1-SNAPSHOT - ../../impala-parent/pom.xml + ../../pom.xml 4.0.0 org.apache.impala diff --git a/tests/test-hive-udfs/pom.xml b/java/test-hive-udfs/pom.xml similarity index 98% rename from tests/test-hive-udfs/pom.xml rename to java/test-hive-udfs/pom.xml index e51d598d49..dc3b431e3d 100644 --- a/tests/test-hive-udfs/pom.xml +++ b/java/test-hive-udfs/pom.xml @@ -23,7 +23,6 @@ under the License. org.apache.impala impala-parent 0.1-SNAPSHOT - ../../impala-parent/pom.xml 4.0.0 diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/ImportsNearbyClassesUdf.java diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java b/java/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/IncompatibleUdfTest.java diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java b/java/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/JavaGcUdfTest.java diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java b/java/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/JavaUdfTest.java diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/ReplaceStringUdf.java diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/TestUdf.java diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java b/java/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/TestUdfException.java diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/TestUpdateUdf.java diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/UnresolvedUdf.java diff --git a/tests/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java b/java/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java similarity index 100% rename from tests/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java rename to java/test-hive-udfs/src/main/java/org/apache/impala/UtilForUdf.java diff --git a/common/yarn-extras/.gitignore b/java/yarn-extras/.gitignore similarity index 100% rename from common/yarn-extras/.gitignore rename to java/yarn-extras/.gitignore diff --git a/common/yarn-extras/README.txt b/java/yarn-extras/README.txt similarity index 100% rename from common/yarn-extras/README.txt rename to java/yarn-extras/README.txt diff --git a/common/yarn-extras/pom.xml b/java/yarn-extras/pom.xml similarity index 98% rename from common/yarn-extras/pom.xml rename to java/yarn-extras/pom.xml index 9222ef6239..ce2d7f84d3 100644 --- a/common/yarn-extras/pom.xml +++ b/java/yarn-extras/pom.xml @@ -23,7 +23,6 @@ org.apache.impala impala-parent 0.1-SNAPSHOT - ../../impala-parent/pom.xml 4.0.0 org.apache.impala diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/resource/ResourceWeights.java diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationConfigurationException.java diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java diff --git a/common/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java b/java/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java similarity index 100% rename from common/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java rename to java/yarn-extras/src/main/java/org/apache/impala/yarn/server/utils/BuilderUtils.java diff --git a/mvn-deps/executor-deps/CMakeLists.txt b/mvn-deps/executor-deps/CMakeLists.txt deleted file mode 100644 index 2f80db40d4..0000000000 --- a/mvn-deps/executor-deps/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -add_custom_target(executor-deps ALL DEPENDS shaded-deps-hive-exec shaded-deps-s3a-aws-sdk - COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests -) diff --git a/mvn-deps/shaded-deps/hive-exec/CMakeLists.txt b/mvn-deps/shaded-deps/hive-exec/CMakeLists.txt deleted file mode 100644 index 7d8b6b125a..0000000000 --- a/mvn-deps/shaded-deps/hive-exec/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -add_custom_target(shaded-deps-hive-exec ALL DEPENDS impala-parent - COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests -) diff --git a/mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt b/mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt deleted file mode 100644 index 956f5eb462..0000000000 --- a/mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -add_custom_target(shaded-deps-s3a-aws-sdk ALL DEPENDS impala-parent - COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests -) diff --git a/query-event-hook-api/CMakeLists.txt b/query-event-hook-api/CMakeLists.txt deleted file mode 100644 index 7352ed2169..0000000000 --- a/query-event-hook-api/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# The dependency on yarn-extras is only added to avoid parallel downloads -# of dependencies. For more details see IMPALA-7051, which was a similar issue. -add_custom_target(query-event-hook-api ALL DEPENDS gen-deps impala-parent yarn-extras - COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests -) diff --git a/testdata/bin/copy-data-sources.sh b/testdata/bin/copy-data-sources.sh index 1782aca3dc..c218e15ddb 100755 --- a/testdata/bin/copy-data-sources.sh +++ b/testdata/bin/copy-data-sources.sh @@ -28,5 +28,5 @@ setup_report_build_error hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}/test-warehouse/data-sources/ hadoop fs -put -f \ - ${IMPALA_HOME}/ext-data-source/test/target/impala-data-source-test-*.jar \ + ${IMPALA_HOME}/java/ext-data-source/test/target/impala-data-source-test-*.jar \ ${FILESYSTEM_PREFIX}/test-warehouse/data-sources/test-data-source.jar diff --git a/testdata/bin/copy-udfs-udas.sh b/testdata/bin/copy-udfs-udas.sh index def78121ac..438eff4a9b 100755 --- a/testdata/bin/copy-udfs-udas.sh +++ b/testdata/bin/copy-udfs-udas.sh @@ -51,7 +51,7 @@ then pushd "${IMPALA_HOME}" "${MAKE_CMD:-make}" ${IMPALA_MAKE_FLAGS} "-j${IMPALA_BUILD_THREADS:-4}" \ TestUdas TestUdfs test-udfs-ir udfsample udasample udf-sample-ir uda-sample-ir - cd "${IMPALA_HOME}/tests/test-hive-udfs" + cd "${IMPALA_HOME}/java/test-hive-udfs" "${IMPALA_HOME}/bin/mvn-quiet.sh" package cp target/test-hive-udfs-1.0.jar "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar" # Change one of the Java files to make a new jar for testing purposes, then change it @@ -64,9 +64,13 @@ then bash -c "sed s/'ReplaceStringUdf'/'NewReplaceStringUdf'/g '{}'" \; \ > src/main/java/org/apache/impala/NewReplaceStringUdf.java "${IMPALA_HOME}/bin/mvn-quiet.sh" package + cp target/test-hive-udfs-1.0.jar \ + "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs-modified.jar" find . -type f -name 'TestUpdateUdf.java' -execdir \ bash -c "sed -i s/'Text(\"New UDF\")'/'Text(\"Old UDF\")'/g '{}'" \; rm src/main/java/org/apache/impala/NewReplaceStringUdf.java + # Rebuild with the original version + "${IMPALA_HOME}/bin/mvn-quiet.sh" package popd fi @@ -91,7 +95,7 @@ ln -s "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so" \ mkdir "${UDF_TMP_DIR}/udf_test" ln -s "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so" "${UDF_TMP_DIR}/udf_test" ln -s "${HIVE_HOME}/lib/hive-exec-"*.jar "${UDF_TMP_DIR}/hive-exec.jar" -ln -s "${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar" \ +ln -s "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar" \ "${UDF_TMP_DIR}/impala-hive-udfs.jar" ln -s "${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll" "${UDF_TMP_DIR}" ln -s "${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so" "${UDF_TMP_DIR}" diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh index 312bd3aa1c..c2dfe336ac 100755 --- a/testdata/bin/create-load-data.sh +++ b/testdata/bin/create-load-data.sh @@ -479,7 +479,7 @@ function load-custom-data { function build-and-copy-hive-udfs { # Build the test Hive UDFs - pushd ${IMPALA_HOME}/tests/test-hive-udfs + pushd ${IMPALA_HOME}/java/test-hive-udfs ${IMPALA_HOME}/bin/mvn-quiet.sh clean ${IMPALA_HOME}/bin/mvn-quiet.sh package popd diff --git a/testdata/bin/generate-load-nested.sh b/testdata/bin/generate-load-nested.sh index cceea1ac55..b773ed0fc4 100755 --- a/testdata/bin/generate-load-nested.sh +++ b/testdata/bin/generate-load-nested.sh @@ -55,7 +55,7 @@ while getopts ":n:e:i:f" OPTION; do done if $FLATTEN; then - mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/testdata/TableFlattener/pom.xml" package + mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/TableFlattener/pom.xml" package fi RANDOM_SCHEMA_GENERATOR=${IMPALA_HOME}/testdata/bin/random_avro_schema.py; @@ -79,12 +79,12 @@ do do FILE_NAME=$(basename ${AVRO_SCHEMA_PATH}) TABLE_NAME="${FILE_NAME%.*}" - mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/testdata/pom.xml" exec:java \ + mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/datagenerator/pom.xml" exec:java \ -Dexec.mainClass="org.apache.impala.datagenerator.RandomNestedDataGenerator" \ -Dexec.args="${AVRO_SCHEMA_PATH} ${NUM_ELEMENTS} ${NUM_FIELDS} ${DB_DIR}/${TABLE_NAME}/${TABLE_NAME}.parquet"; if $FLATTEN; then - mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/testdata/TableFlattener/pom.xml" \ + mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/TableFlattener/pom.xml" \ exec:java -Dexec.mainClass=org.apache.impala.infra.tableflattener.Main \ -Dexec.arguments="file://${DB_DIR}/${TABLE_NAME}/${TABLE_NAME}.parquet,file://${DB_DIR}/${TABLE_NAME}_flat,-sfile://${AVRO_SCHEMA_PATH}"; fi diff --git a/tests/custom_cluster/test_coordinators.py b/tests/custom_cluster/test_coordinators.py index 7a4336b007..6e3b2965fc 100644 --- a/tests/custom_cluster/test_coordinators.py +++ b/tests/custom_cluster/test_coordinators.py @@ -145,7 +145,7 @@ def test_executor_only_lib_cache(self): old_src_path = os.path.join( os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs.jar') new_src_path = os.path.join( - os.environ['IMPALA_HOME'], 'tests/test-hive-udfs/target/test-hive-udfs-1.0.jar') + os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs-modified.jar') tgt_dir = get_fs_path('/test-warehouse/{0}.db'.format(db_name)) tgt_path = tgt_dir + "/tmp.jar" diff --git a/tests/query_test/test_udfs.py b/tests/query_test/test_udfs.py index 3d73918324..4d2177d617 100644 --- a/tests/query_test/test_udfs.py +++ b/tests/query_test/test_udfs.py @@ -517,7 +517,7 @@ def test_udf_update_via_drop(self, vector, unique_database): old_udf = os.path.join( os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs.jar') new_udf = os.path.join( - os.environ['IMPALA_HOME'], 'tests/test-hive-udfs/target/test-hive-udfs-1.0.jar') + os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs-modified.jar') udf_dst = get_fs_path( '/test-warehouse/{0}.db/impala-hive-udfs.jar'.format(unique_database)) @@ -550,7 +550,7 @@ def test_udf_update_via_create(self, vector, unique_database): old_udf = os.path.join( os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs.jar') new_udf = os.path.join( - os.environ['IMPALA_HOME'], 'tests/test-hive-udfs/target/test-hive-udfs-1.0.jar') + os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs-modified.jar') udf_dst = get_fs_path( '/test-warehouse/{0}.db/impala-hive-udfs.jar'.format(unique_database)) old_function_name = "udf_update_test_create1" diff --git a/tests/run-tests.py b/tests/run-tests.py index 8f1e8d3c52..8168cd1357 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -44,7 +44,7 @@ # new dir to the list of valid test dirs above. All dirs unders tests/ must be placed # into one of these lists, otherwise the script will throw an error. This list can be # removed once IMPALA-4417 has been resolved. -TEST_HELPER_DIRS = ['aux_parquet_data_load', 'test-hive-udfs', 'comparison', 'benchmark', +TEST_HELPER_DIRS = ['aux_parquet_data_load', 'comparison', 'benchmark', 'custom_cluster', 'util', 'experiments', 'verifiers', 'common', 'performance', 'beeswax', 'aux_custom_cluster_tests', 'authorization']