Skip to content

HBASE-24170 Remove hadoop-2.0 profile #1495

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 16, 2020
Merged

HBASE-24170 Remove hadoop-2.0 profile #1495

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8e4ffcf
HBASE-24170 Remove hadoop-2.0 profile
Apache9 Apr 13, 2020
ca4aef5
comment
Apache9 Apr 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions dev-support/Jenkinsfile_GitHub
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ pipeline {
// stage works in its own subdirectory. there is an "output" under each of these
// directories, which we retrieve after the build is complete.
WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
WORKDIR_REL_JDK8_HADOOP2_CHECK = 'yetus-jdk8-hadoop2-check'
WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
}

Expand Down Expand Up @@ -149,7 +149,7 @@ pipeline {
}
}
}
stage ('yetus jdk8 Hadoop2 checks') {
stage ('yetus jdk8 Hadoop3 checks') {
agent {
node {
label 'Hadoop'
Expand All @@ -159,7 +159,7 @@ pipeline {
// customized per parallel stage
PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
SET_JAVA_HOME = '/usr/lib/jvm/java-8'
WORKDIR_REL = "${WORKDIR_REL_JDK8_HADOOP2_CHECK}"
WORKDIR_REL = "${WORKDIR_REL_JDK8_HADOOP3_CHECK}"
// identical for all parallel stages
WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
YETUSDIR = "${WORKDIR}/${YETUS_REL}"
Expand Down Expand Up @@ -259,7 +259,6 @@ pipeline {
// customized per parallel stage
PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
SET_JAVA_HOME = '/usr/lib/jvm/java-11'
HADOOP_PROFILE = '3.0'
WORKDIR_REL = "${WORKDIR_REL_JDK11_HADOOP3_CHECK}"
// identical for all parallel stages
WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
Expand Down
32 changes: 24 additions & 8 deletions dev-support/hbase-personality.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,13 +589,16 @@ function hadoopcheck_rebuild
else
hbase_hadoop2_versions="2.8.5 2.9.2 2.10.0"
fi
else
yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+/master/feature branch rules."
elif [[ "${PATCH_BRANCH}" = branch-2.* ]]; then
yetus_info "Setting Hadoop 2 versions to test based on branch-2.3+ rules."
if [[ "${QUICK_HADOOPCHECK}" == "true" ]]; then
hbase_hadoop2_versions="2.10.0"
else
hbase_hadoop2_versions="2.10.0"
fi
else
yetus_info "Setting Hadoop 2 versions to null on master/feature branch rules since we do not support hadoop 2 for hbase 3.x any more."
hbase_hadoop2_versions=""
fi
if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
yetus_info "Setting Hadoop 3 versions to test based on branch-1.x rules."
Expand Down Expand Up @@ -635,13 +638,22 @@ function hadoopcheck_rebuild

for hadoopver in ${hbase_hadoop3_versions}; do
logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
# no hbase_hadoop2_versions means hadoop 3.x only, so we do not need to
# specify hadoop.profile
# disabled because "maven_executor" needs to return both command and args
# shellcheck disable=2046
echo_and_redirect "${logfile}" \
$(maven_executor) clean install \
-DskipTests -DHBasePatchProcess \
-Dhadoop-three.version="${hadoopver}" \
-Dhadoop.profile=3.0
if [[ -n "${hbase_hadoop2_versions}" ]]; then
echo_and_redirect "${logfile}" \
$(maven_executor) clean install \
-DskipTests -DHBasePatchProcess \
-Dhadoop-three.version="${hadoopver}" \
-Dhadoop.profile=3.0
else
echo_and_redirect "${logfile}" \
$(maven_executor) clean install \
-DskipTests -DHBasePatchProcess \
-Dhadoop-three.version="${hadoopver}"
fi
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
Expand All @@ -655,7 +667,11 @@ function hadoopcheck_rebuild
fi

if [[ -n "${hbase_hadoop3_versions}" ]]; then
add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
if [[ -n "${hbase_hadoop2_versions}" ]]; then
add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions} or ${hbase_hadoop3_versions}."
else
add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop3_versions}."
fi
else
add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
fi
Expand Down
119 changes: 2 additions & 117 deletions hbase-backup/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,126 +123,11 @@
</dependency>
</dependencies>
<profiles>
<!-- profile against Hadoop 2.x: This is the default. -->
<profile>
<id>hadoop-2.0</id>
<activation>
<property>
<!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
<!--h2-->
<name>!hadoop.profile</name>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
<!--We let hadoop include guava here. It uses it
in the Configuration class -->
</exclusions>
</dependency>
<dependency>
<!--Hadoop2 Uses Guava in Configuration class at least-->
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${hadoop.guava.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<!--This module seems to need this.-->
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
<version>${netty.hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop-two.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
<version>${hadoop-two.version}</version>
</dependency>
</dependencies>
</profile>
<!--
profile for building against Hadoop 3.0.x. Activate using:
mvn -Dhadoop.profile=3.0
-->
<!-- Profile for building against Hadoop 3.0.0. Activate by default -->
<profile>
<id>hadoop-3.0</id>
<activation>
<property>
<name>hadoop.profile</name>
<value>3.0</value>
</property>
<property><name>!hadoop.profile</name></property>
</activation>
<dependencies>
<dependency>
Expand Down
74 changes: 2 additions & 72 deletions hbase-client/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,81 +191,11 @@
<surefire.skipSecondPart>true</surefire.skipSecondPart>
</properties>
</profile>

<!-- profile for building against Hadoop 2.x. This is the default -->
<profile>
<id>hadoop-2.0</id>
<activation>
<property>
<!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
<!--h2--><name>!hadoop.profile</name>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
<exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
</exclusion>
<exclusion>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>

<!--
profile for building against Hadoop 3.0.x. Activate using:
mvn -Dhadoop.profile=3.0
-->
<!-- Profile for building against Hadoop 3.0.0. Activate by default -->
<profile>
<id>hadoop-3.0</id>
<activation>
<property>
<name>hadoop.profile</name>
<value>3.0</value>
</property>
<property><name>!hadoop.profile</name></property>
</activation>
<properties>
<hadoop.version>3.0-SNAPSHOT</hadoop.version>
Expand Down
54 changes: 2 additions & 52 deletions hbase-common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,61 +266,11 @@
<surefire.skipSecondPart>true</surefire.skipSecondPart>
</properties>
</profile>


<!-- profile for building against Hadoop 2.x. This is the default -->
<profile>
<id>hadoop-2.0</id>
<activation>
<property>
<!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
<!--h2--><name>!hadoop.profile</name>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<!--FYI This pulls in hadoop's guava. Its needed for Configuration
at least-->
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>create-mrapp-generated-classpath</id>
<phase>generate-test-resources</phase>
<goals>
<goal>build-classpath</goal>
</goals>
<configuration>
<!-- needed to run the unit test for DS to generate
the required classpath that is required in the env
of the launch container in the mini mr/yarn cluster
-->
<outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>

<!--
profile for building against Hadoop 3.0.x. Activate using:
mvn -Dhadoop.profile=3.0
-->
<!-- Profile for building against Hadoop 3.0.0. Activate by default -->
<profile>
<id>hadoop-3.0</id>
<activation>
<property>
<name>hadoop.profile</name>
<value>3.0</value>
</property>
<property><name>!hadoop.profile</name></property>
</activation>
<properties>
<hadoop.version>3.0-SNAPSHOT</hadoop.version>
Expand Down
Loading